From 29e1e42c48de8fb30965706aabb92f358eb8c8f8 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sat, 3 Oct 2026 22:58:11 -0400 Subject: [PATCH 1/9] test(client): failing tests for client hardening under a two-message protocol (rows-first c0a) Tests for the client half of plan 1 phase 0a. A server that sends the summary stats separately, or not at all, leaves the first message without values the client has been assuming. BuckarooView: a change that reached the model before the effect subscribed, one initial_state with metadata decoding once, and out-of-order decodes applying the newer. Pinned rows: a valueless key shows a placeholder with its own row id while df_meta.stats.status is pending, and is omitted when not_computed. The simple tooltip returns nothing for a valueless cell. color_map is silent without bins and restyles when they arrive. An unrelated df_data_dict update keeps the in-flight indicator when the server reports df_meta.stats. One Storybook Playwright test covers pending, not_computed and complete in a real browser, and is added to the Storybook list in scripts/test_playwright_storybook.sh. Co-Authored-By: Claude Sonnet 5.5 --- .../stats-pending-pinned-rows.spec.ts | 55 +++++ .../BuckarooInfiniteWidget.flash.test.tsx | 209 +++++++++++++++++- .../DFViewerParts/gridUtils.test.ts | 69 +++++- .../src/server/BuckarooView.test.tsx | 134 ++++++++++- .../StatsPendingPinnedRows.stories.tsx | 130 +++++++++++ scripts/test_playwright_storybook.sh | 1 + 6 files changed, 590 insertions(+), 8 deletions(-) create mode 100644 packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts create mode 100644 packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx diff --git a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts new file mode 100644 index 000000000..88c60f4a4 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts @@ -0,0 +1,55 @@ +/** + * Playwright test for the client states of the two-message protocol + * (rows-first c0a). + * + * While summary stats have not arrived, `df_meta.stats.status` decides what + * the pinned area shows. The StatsPendingPinnedRows story flips the status and + * supplies the stats on "complete": + * - pending: every valueless pinned key shows a placeholder row with its own + * row id, and hovering a valueless cell does not throw + * - not_computed: valueless pinned keys are omitted + * - complete: the values appear and the color-mapped column restyles + */ +import { test, expect } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-dfviewer-statspendingpinnedrows--primary&globals=&args="; + +test("pinned rows follow df_meta.stats.status: placeholders, omitted, then values and colors", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + const bodyCellA = page.locator('.ag-center-cols-container .ag-row[row-index="0"] [col-id="a"]'); + const backgroundOf = (loc: typeof bodyCellA) => loc.evaluate((el) => getComputedStyle(el).backgroundColor); + + // pending: one placeholder per pinned key, each with its own row id. + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(bodyCellA).toHaveText("1"); + const backgroundBeforeBins = await backgroundOf(bodyCellA); + + // Hovering a valueless pinned cell shows no tooltip and throws nothing. + await page.locator('.ag-floating-top .ag-cell[col-id="a"]').first().hover(); + await page.waitForTimeout(500); + expect(pageErrors).toEqual([]); + + // not_computed: the valueless keys are omitted. + await page.getByTestId("status-not_computed").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + + // complete: the values appear, and the color-mapped column restyles now + // that its histogram bins exist. + await page.getByTestId("status-complete").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + await expect.poll(() => backgroundOf(bodyCellA), { timeout: 10_000 }).not.toBe(backgroundBeforeBins); + expect(pageErrors).toEqual([]); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index a117e9816..51a14cb41 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -7,7 +7,7 @@ * Tests assert CURRENT behavior on main (Option A in docs/rerender-test-plan.md). * Tests tagged "[captures current flash]" are tracking pain, not validating it. */ -import { render } from "@testing-library/react"; +import { render, act } from "@testing-library/react"; import { BuckarooInfiniteWidget } from "./BuckarooWidgetInfinite"; import { KeyAwareSmartRowCache } from "./DFViewerParts/SmartRowCache"; import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridSpy"; @@ -21,8 +21,14 @@ jest.mock("ag-grid-react", () => jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); // StatusBar also instantiates AgGridReact; stub it so the spy only counts the data grid. +// The stub records its props so the in-flight tests can read inFlight and drive +// setBuckarooState the way the real status bar does. +let mockStatusBarProps: any = null; jest.mock("./StatusBar", () => ({ - StatusBar: () =>
, + StatusBar: (props: any) => { + mockStatusBarProps = props; + return
; + }, })); // DFViewerInfinite-prop capture for identity-stability assertion. @@ -656,3 +662,202 @@ describe("BuckarooInfiniteWidget — flash matrix (current behavior)", () => { }); }); }); + +// Rows-first c0a: the grid has to behave while summary stats are pending, not +// computed, or arrive after the first paint. df_meta.stats.status drives it; +// absent df_meta.stats means "complete", as servers without the field send. +describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", () => { + const withStats = (status: string): DFMeta => ({ ...baseDfMeta, stats: { status } } as DFMeta); + + const pinnedConfig: DFViewerConfig = { + ...baseConfig, + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "histogram", displayer_args: { displayer: "obj" } }, + ], + }; + const pinnedDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: pinnedConfig, summary_stats_key: "summary_stats" }, + }; + + const widgetProps = (over: Record = {}) => ({ + df_data_dict: { summary_stats: [] as any[] }, + df_display_args: pinnedDisplayArgs, + df_meta: baseDfMeta, + operations: [], + on_operations: jest.fn(), + operation_results: {} as any, + command_config: { argspecs: {}, defaultArgs: {} }, + buckaroo_state: initialState, + on_buckaroo_state: jest.fn(), + buckaroo_options: baseOptions, + src: mkSrc(), + ...over, + }); + + const lastPinnedRows = (): any[] => { + const sets = getSpyCalls().setGridOption.filter(([k]) => k === "pinnedTopRowData"); + return sets[sets.length - 1][1] as any[]; + }; + const pinnedRowIds = (rows: any[]): string[] => { + const { gridOptions, context } = getSpyCalls().lastProps; + return rows.map((data) => gridOptions.getRowId({ data, level: 0, rowPinned: "top", context })); + }; + + describe("pinned rows with no value", () => { + it("pending: shows one placeholder per valueless key, each with its own row id", () => { + render(); + const rows = lastPinnedRows(); + // Each placeholder keeps its key as the row label, so the pinned area + // holds its height and the rows are told apart. + expect(rows.map((r) => r?.index)).toEqual(["dtype", "histogram"]); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: a key that has a value renders it and only the other key gets a placeholder", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + const rows = lastPinnedRows(); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]?.index).toBe("histogram"); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: placeholders turn into the real rows when stats arrive", () => { + const props = widgetProps({ df_meta: withStats("pending") }); + const { rerender } = render(); + const stats = [ + { index: "dtype", a: "int64" }, + { index: "histogram", a: [{ name: "1-5", population: 100 }] }, + ]; + rerender( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("not_computed: omits the valueless keys", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("not_computed: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("no df_meta.stats (an older server) behaves as complete: valueless keys stay undefined rows", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render(); + const rows = lastPinnedRows(); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]).toEqual({}); + }); + }); + + describe("color_map columns", () => { + const colorConfig: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + }; + const colorDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: colorConfig, summary_stats_key: "summary_stats" }, + }; + const colorProps = (stats: any[]) => + widgetProps({ df_display_args: colorDisplayArgs, df_data_dict: { summary_stats: stats } }); + const forcedRefreshes = () => getSpyCalls().refreshCells.filter((p: any) => p.force === true); + + it("restyles the color-mapped columns when histogram bins arrive after the first render", () => { + const props = colorProps([{ index: "dtype", a: "int64", b: "int64" }]); + const { rerender } = render(); + expect(forcedRefreshes()).toHaveLength(0); + + rerender( + , + ); + const refreshes = forcedRefreshes(); + expect(refreshes).toHaveLength(1); + expect(refreshes[0].columns).toEqual(["a"]); + }); + + it("does not refresh when the new stats leave the bins unchanged", () => { + const props = colorProps([{ index: "histogram_bins", a: [1, 2, 3, 4, 5] }]); + const { rerender } = render(); + rerender( + , + ); + expect(forcedRefreshes()).toHaveLength(0); + }); + }); + + describe("in-flight indicator", () => { + const inFlightAttr = () => document.querySelector('[data-testid="status-bar-stub"]')!.getAttribute("data-inflight"); + const dispatchSearch = () => + act(() => { + mockStatusBarProps.setBuckarooState({ ...initialState, quick_command_args: { search: ["x"] } }); + }); + const props = (meta: DFMeta, dict: Record) => widgetProps({ df_meta: meta, df_data_dict: dict }); + + it("a df_data_dict update alone does not clear it when the server reports df_meta.stats", () => { + const meta = withStats("pending"); + const first = props(meta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + // A stats-only update: a new df_data_dict, the same df_meta. + rerender(); + expect(inFlightAttr()).toBe("true"); + }); + + it("clears when a frame with a new df_meta and df_data_dict arrives", () => { + const first = props(withStats("pending"), { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + + rerender( + , + ); + expect(inFlightAttr()).toBe("false"); + }); + + it("without df_meta.stats a new df_data_dict still clears it, as before", () => { + const first = props(baseDfMeta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + rerender(); + expect(inFlightAttr()).toBe("false"); + }); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts index 01697ba2e..944ceddbc 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts @@ -15,7 +15,8 @@ import { import * as _ from "lodash-es"; import { DFData, DFViewerConfig, NormalColumnConfig, MultiIndexColumnConfig, PinnedRowConfig, ColumnConfig, FormatterArgs } from "./DFWhole"; import { getFormatter, getFloatFormatter, getCompactNumberFormatter, formatDuration, formatIsoDuration, getDurationFormatter } from './Displayer'; -import { ColDef, ICellRendererParams, ValueFormatterParams } from 'ag-grid-community'; +import { CellClassParams, ColDef, ICellRendererParams, ITooltipParams, ValueFormatterParams } from 'ag-grid-community'; +import { getSimpleTooltip } from './SeriesSummaryTooltip'; describe("testing utility functions in gridUtils ", () => { // mostly sanity checks to help develop gridUtils @@ -732,6 +733,70 @@ describe("testing multi index organiztion ", () => { expect(children.length).toBe(2); }); +}); + +// Rows-first c0a: while summary stats are pending or not computed, pinned +// cells have no value, and color_map has no histogram bins to read. +describe("pinned cells without stats values (rows-first c0a)", () => { + it("the simple tooltip returns nothing for a valueless pinned cell instead of throwing", () => { + const tooltip = getSimpleTooltip("a"); + // A pinned row with no value for the column: only the row label exists. + expect(() => tooltip({ data: { index: "dtype" } } as ITooltipParams)).not.toThrow(); + expect(tooltip({ data: { index: "dtype" } } as ITooltipParams)).toBeUndefined(); + // A null cell and a row that has not loaded (data undefined) are also valueless. + expect(() => tooltip({ data: { index: "mean", a: null } } as ITooltipParams)).not.toThrow(); + expect(() => tooltip({ data: undefined } as unknown as ITooltipParams)).not.toThrow(); + }); + + it("the simple tooltip still renders a cell that has a value", () => { + const tooltip = getSimpleTooltip("a"); + const el = tooltip({ data: { index: 0, a: 5 } } as ITooltipParams); + expect(el).toBeDefined(); + expect((el as any).props.children).toBe("5"); + }); + + describe("color_map without histogram bins", () => { + const config: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + }, + ], + }; + const cellStyleFor = (context: any, value: any = 3) => { + const colDef = dfToAgrid(config)[0] as ColDef; + const cellStyle = colDef.cellStyle as (p: CellClassParams) => Record; + return cellStyle({ context, data: { index: 0, a: value }, value, node: { rowPinned: undefined } } as unknown as CellClassParams); + }; - + let logSpy: jest.SpyInstance; + beforeEach(() => { logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); }); + afterEach(() => { logSpy.mockRestore(); }); + + it("returns the neutral style without logging when the stats column has no entry", () => { + expect(cellStyleFor({ histogram_stats: {} })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the entry has no histogram_bins", () => { + expect(cellStyleFor({ histogram_stats: { a: { histogram_log_bins: [1, 2] } } })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the context carries no histogram_stats", () => { + expect(cellStyleFor({})).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("colors the cell once bins exist", () => { + const style = cellStyleFor({ histogram_stats: { a: { histogram_bins: [1, 2, 3, 4, 5] } } }); + expect(style.backgroundColor).not.toBe("inherit"); + expect(style.backgroundColor).toBeDefined(); + }); + }); }); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx index 8a38aa6cf..796e1db75 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx @@ -9,18 +9,34 @@ import { render, cleanup, act } from "@testing-library/react"; import { BuckarooView } from "./BuckarooView"; import type { IModel } from "./IModel"; +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; // Stub the heavy widget surfaces — this test exercises the injection // wiring, not AG-Grid. The widget components instantiate AgGridReact which // is fragile under jsdom; the stub keeps the test focused on the model // contract. +// +// The viewer stub records the props BuckarooView hands it, so the rows-first +// tests below can see which df_meta / df_data_dict reached the widget. +const mockViewerProps: any[] = []; jest.mock("../components/BuckarooWidgetInfinite", () => ({ BuckarooInfiniteWidget: () =>
, - DFViewerInfiniteDS: () =>
, + DFViewerInfiniteDS: (props: any) => { + mockViewerProps.push(props); + return
; + }, getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), })); -function makeFakeModel(): { model: IModel; events: Map>; sent: any[] } { +// Wrap decodeDFDataDict in a jest.fn that defaults to the real decoder, so the +// existing tests run unchanged and the rows-first tests can count and delay +// decodes. +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { ...actual, decodeDFDataDict: jest.fn(actual.decodeDFDataDict) }; +}); + +function makeFakeModel(): { model: IModel; events: Map>; sent: any[]; state: Record } { const events = new Map>(); const state: Record = {}; const sent: any[] = []; @@ -35,10 +51,13 @@ function makeFakeModel(): { model: IModel; events: Map>; s }, off: (e, h) => { events.get(e)?.delete(h); }, }; - return { model, events, sent }; + return { model, events, sent, state }; } -afterEach(() => cleanup()); +afterEach(() => { + mockViewerProps.length = 0; + cleanup(); +}); describe("BuckarooView (injectable IModel — #759)", () => { it("renders the viewer widget when given a fake IModel + initialState — no WebSocket needed", async () => { @@ -102,3 +121,110 @@ describe("BuckarooView (injectable IModel — #759)", () => { expect(onMetadata).toHaveBeenCalledWith({ path: "/data/sales.parquet", rows: 42 }, "tell me about sales"); }); }); + +// Rows-first c0a: a second message can follow the first at any moment, so the +// view has to cope with changes that land while it is still wiring itself up. +describe("BuckarooView two-message hardening (rows-first c0a)", () => { + const mockDecode = decodeDFDataDict as jest.Mock; + const realDecode = jest.requireActual("../components/DFViewerParts/resolveDFData").decodeDFDataDict; + + const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, + }; + const metaWith = (total_rows: number) => ({ total_rows, columns: 1, filtered_rows: total_rows, rows_shown: total_rows }); + const emit = (events: Map>, name: string, ...args: unknown[]) => { + for (const h of Array.from(events.get(name) ?? [])) h(...args); + }; + const lastProps = () => mockViewerProps[mockViewerProps.length - 1]; + + beforeEach(() => { + mockDecode.mockReset(); + mockDecode.mockImplementation(realDecode); + }); + + it("applies a change that reached the model before the effect subscribed", async () => { + // initialState is what the host held when it built the view. A second + // initial_state then landed on the model while React was committing, + // so its change:* events had no listener yet. + const { model, state } = makeFakeModel(); + state.df_meta = metaWith(99); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_meta.total_rows).toBe(99); + }); + + it("decodes a df_data_dict that reached the model before the effect subscribed", async () => { + const { model, state } = makeFakeModel(); + const raw = { format: "mock", id: "N" }; + state.df_data_dict = { main: raw }; + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "N" }]); + }); + + it("decodes one initial_state that carries metadata once, and renders its df_data_dict once", async () => { + const { model, events, state } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + const dictsBefore = new Set(mockViewerProps.map((p) => p.df_data_dict)); + mockDecode.mockClear(); + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + + // The order WebSocketModel emits for a full frame: one change per key, + // then "metadata". The model state already holds the new values. + const frame = { + df_meta: metaWith(7), + df_data_dict: { main: { format: "mock", id: "F" } }, + df_display_args: displayArgs, + metadata: { path: "/data/f.parquet", rows: 7 }, + }; + Object.assign(state, frame); + await act(async () => { + emit(events, "change:df_meta", frame.df_meta); + emit(events, "change:df_data_dict", frame.df_data_dict); + emit(events, "change:df_display_args", frame.df_display_args); + emit(events, "metadata", frame.metadata, undefined); + }); + + expect(mockDecode).toHaveBeenCalledTimes(1); + const dictsAfter = new Set(mockViewerProps.map((p) => p.df_data_dict)); + expect(dictsAfter.size - dictsBefore.size).toBe(1); + expect(lastProps().df_meta.total_rows).toBe(7); + }); + + it("applies the newer df_data_dict when decodes complete out of order", async () => { + const { model, events } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + + const finish: Record void> = {}; + mockDecode.mockImplementation( + (dict: any) => + new Promise((resolve) => { + finish[dict.main.id] = () => resolve({ main: [{ index: 0, from: dict.main.id }] }); + }), + ); + await act(async () => { + emit(events, "change:df_data_dict", { main: { format: "mock", id: "older" } }); + emit(events, "change:df_data_dict", { main: { format: "mock", id: "newer" } }); + }); + // The newer decode finishes first, then the stale one. + await act(async () => { finish["newer"](); }); + await act(async () => { finish["older"](); }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "newer" }]); + }); +}); diff --git a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx new file mode 100644 index 000000000..caa5628c1 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx @@ -0,0 +1,130 @@ +/** + * Story for the two-message protocol's client states (rows-first c0a). + * + * The server sends the first message once the schema and row count are known, + * and the summary stats later or never. `df_meta.stats.status` says which: + * + * - "pending" pinned keys with no value show a placeholder row + * - "not_computed" pinned keys with no value are omitted + * - "complete" stats are present; also the meaning of a missing + * `df_meta.stats`, as servers without the field send + * + * Column `a` is color-mapped and has a simple tooltip, so the story also shows + * that cells restyle when the histogram bins arrive and that hovering a + * valueless pinned cell does nothing. Used by stats-pending-pinned-rows.spec.ts. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import { DFViewerInfiniteDS } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { DFMeta } from "../components/WidgetTypes"; + +type Status = "pending" | "not_computed" | "complete"; +const STATUSES: Status[] = ["pending", "not_computed", "complete"]; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, + { index: 3, a: 4, b: "w" }, + { index: 4, a: 5, b: "v" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 3, b: "N/A" }, + { index: "histogram_bins", a: [0, 1, 2, 3, 4, 5], b: [] }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + tooltip_config: { tooltip_type: "simple", val_column: "a" }, + }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const StatsPendingPinnedRowsInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { status }, + }) as DFMeta, + [status], + ); + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : ([] as DFData), + empty: [] as DFData, + }), + [status], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + df_meta.stats.status = {status} +
+
+ +
+
+ ); +}; + +const meta = { + title: "Buckaroo/DFViewer/StatsPendingPinnedRows", + component: StatsPendingPinnedRowsInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; diff --git a/scripts/test_playwright_storybook.sh b/scripts/test_playwright_storybook.sh index c0064c16d..6d835ab2a 100755 --- a/scripts/test_playwright_storybook.sh +++ b/scripts/test_playwright_storybook.sh @@ -95,6 +95,7 @@ STORYBOOK_TESTS=( "pw-tests/transcript-replayer.spec.ts" "pw-tests/outside-params.spec.ts" "pw-tests/fitcontent-height.spec.ts" + "pw-tests/stats-pending-pinned-rows.spec.ts" # "pw-tests/example.spec.ts" # Has pre-existing failures, excluded for now ) From 405f4ded5f3a767e39cfd542abe461c4b54f8417 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sat, 3 Oct 2026 23:10:59 -0400 Subject: [PATCH 2/9] fix(client): harden the client for a two-message protocol (rows-first c0a) BuckarooView reads model.get() for every key once its listeners are in place, so a change:* emitted before the effect ran reaches React. All df_data_dict values go through one decoder (makeLatestDictDecoder) that skips a dict it has already seen and applies only the newest decode, so an initial_state with metadata decodes once and out-of-order decodes cannot overwrite a newer one. standalone.tsx gets the same decoder and catch-up. df_meta.stats.status reaches the grid as stats_status. While pending, a required pinned key with no value becomes a placeholder row that carries its key as index (unique row id, empty value cells). When not_computed it is omitted. A missing df_meta.stats, or any other status, keeps the old behaviour. The simple tooltip returns nothing for a cell with no value or no row data. color_map no longer logs when bins are missing, and the grid refreshes the color-mapped columns when their bins change after the first render. That needs RenderApiModule, which was not registered, so api.refreshCells logged AG Grid error 200 and did nothing. BuckarooInfiniteWidget keeps inFlight when only df_data_dict changed and the server reports df_meta.stats; the answer to the dispatched change is a frame with a new df_meta. Without df_meta.stats the rule is unchanged. The Playwright story test no longer hovers a valueless pinned cell: with the old tooltip it still passed, since AG Grid does not call the tooltip for an empty value. Co-Authored-By: Claude Sonnet 5.5 --- .../stats-pending-pinned-rows.spec.ts | 7 +- .../src/components/BuckarooWidgetInfinite.tsx | 19 +++++- .../DFViewerParts/DFViewerInfinite.tsx | 45 ++++++++++++- .../DFViewerParts/SeriesSummaryTooltip.tsx | 11 +++- .../src/components/DFViewerParts/Styler.tsx | 5 +- .../src/components/DFViewerParts/gridUtils.ts | 32 +++++++-- .../src/components/WidgetTypes.tsx | 18 +++++ packages/buckaroo-js-core/src/index.ts | 3 + .../src/server/BuckarooView.tsx | 60 ++++++++++++----- .../src/server/latestDictDecoder.ts | 34 ++++++++++ .../StatsPendingPinnedRows.stories.tsx | 6 +- packages/js/standalone.tsx | 65 ++++++++++++++++--- 12 files changed, 253 insertions(+), 52 deletions(-) create mode 100644 packages/buckaroo-js-core/src/server/latestDictDecoder.ts diff --git a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts index 88c60f4a4..91c8344e6 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts @@ -6,7 +6,7 @@ * the pinned area shows. The StatsPendingPinnedRows story flips the status and * supplies the stats on "complete": * - pending: every valueless pinned key shows a placeholder row with its own - * row id, and hovering a valueless cell does not throw + * row id * - not_computed: valueless pinned keys are omitted * - complete: the values appear and the color-mapped column restyles */ @@ -36,11 +36,6 @@ test("pinned rows follow df_meta.stats.status: placeholders, omitted, then value await expect(bodyCellA).toHaveText("1"); const backgroundBeforeBins = await backgroundOf(bodyCellA); - // Hovering a valueless pinned cell shows no tooltip and throws nothing. - await page.locator('.ag-floating-top .ag-cell[col-id="a"]').first().hover(); - await page.waitForTimeout(500); - expect(pageErrors).toEqual([]); - // not_computed: the valueless keys are omitted. await page.getByTestId("status-not_computed").click(); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index d126a87c7..07e3a7b50 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -7,7 +7,7 @@ import { DFData } from "./DFViewerParts/DFWhole"; import { StatusBar } from "./StatusBar"; import { BuckarooState } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, getStatsStatus } from "./WidgetTypes"; import { CommandConfigT } from "./CommandUtils"; import { Operation } from "./OperationUtils"; import { @@ -290,13 +290,23 @@ export function BuckarooInfiniteWidget({ // so we don't accidentally clear on the same dict the user was looking // at when they fired the change. Cleared on the *next* dict reference. const inFlightAtDictRef = useRef(null); + // The df_meta at that moment. A server that reports df_meta.stats can + // send df_data_dict on its own (a stats update for an earlier state), + // and df_meta only comes with a full frame, so for those servers a + // new df_meta is also needed before the answer to the change counts + // as arrived. Servers without df_meta.stats keep the dict-only rule. + const inFlightAtMetaRef = useRef(null); useEffect(() => { if (inFlight && inFlightAtDictRef.current !== null && df_data_dict !== inFlightAtDictRef.current) { + if (df_meta === inFlightAtMetaRef.current && df_meta.stats !== undefined) { + return; + } setInFlight(false); inFlightAtDictRef.current = null; + inFlightAtMetaRef.current = null; } - }, [df_data_dict, inFlight]); + }, [df_data_dict, df_meta, inFlight]); const wrappedOnBuckarooState = useCallback< React.Dispatch> @@ -311,10 +321,11 @@ export function BuckarooInfiniteWidget({ !== JSON.stringify(buckaroo_state.quick_command_args); if (dataflowChanged) { inFlightAtDictRef.current = df_data_dict; + inFlightAtMetaRef.current = df_meta; setInFlight(true); } on_buckaroo_state(next); - }, [buckaroo_state, df_data_dict, on_buckaroo_state]); + }, [buckaroo_state, df_data_dict, df_meta, on_buckaroo_state]); return (
{buckaroo_state.show_commands ? ( @@ -447,6 +459,7 @@ export function DFViewerInfiniteDS({ activeCol={activeCol} setActiveCol={setActiveCol} error_info={""} + stats_status={getStatsStatus(df_meta)} />
diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx index 7c8fa079e..4f4465ccb 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx @@ -7,7 +7,8 @@ import { import * as _ from "lodash-es"; import { DFData, DFDataRow, DFViewerConfig, SDFT } from "./DFWhole"; -import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT } from "./gridUtils"; +import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT, getFieldVal } from "./gridUtils"; +import type { StatsStatus } from "../WidgetTypes"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { @@ -21,6 +22,7 @@ import { CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -45,6 +47,9 @@ ModuleRegistry.registerModules([ CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + // api.refreshCells lives here. Without it the call logs AG Grid error 200 + // and does nothing. + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -150,6 +155,7 @@ export function DFViewerInfinite({ max_rows_in_configs, view_name, data_key, + stats_status, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -173,6 +179,10 @@ export function DFViewerInfinite({ // a rowId, even though their `index` values overlap (row 0 in main is a // different record than row 0 in summary). data_key?: string; + // df_meta.stats.status. While "pending" a pinned key with no value shows a + // placeholder row; when "not_computed" it is omitted. Undefined behaves + // as "complete". + stats_status?: StatsStatus; }) { /* The idea is to do some pre-setup here for @@ -233,6 +243,7 @@ export function DFViewerInfinite({ effectiveScheme={effectiveScheme} view_name={view_name} data_key={data_key} + stats_status={stats_status} />
) @@ -250,6 +261,7 @@ export function DFViewerInfiniteInner({ effectiveScheme, view_name, data_key, + stats_status, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -266,6 +278,7 @@ export function DFViewerInfiniteInner({ effectiveScheme?: 'light' | 'dark'; view_name?: string; data_key?: string; + stats_status?: StatsStatus; }) { /* const lastProps = useRef(null); @@ -344,8 +357,8 @@ export function DFViewerInfiniteInner({ // Always re-extract; upstream may mutate summary in-place without changing identity // Memoize to ensure it updates when summary_stats_data changes const topRowData = useMemo( - () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : []) as DFDataRow[], - [summary_stats_data, pinned_rows] + () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : [], stats_status) as DFDataRow[], + [summary_stats_data, pinned_rows, stats_status] ); // Pinned rows are extracted and ready @@ -442,6 +455,32 @@ export function DFViewerInfiniteInner({ // ignore until grid ready } }, [pinnedSig]); + + // color_map reads histogram_bins from the grid context when a cell is + // painted, so cells that rendered before the bins arrived keep the + // neutral style. Repaint the color-mapped columns when their bins + // change after the first render. Bins that are already there at mount + // paint correctly, so the first run only records the signature. + const colorMapCols = useMemo( + () => df_viewer_config.column_config.flatMap((cc) => + cc.color_map_config?.color_rule === "color_map" + ? [{ field: getFieldVal(cc), statsCol: cc.color_map_config.val_column }] + : []), + [df_viewer_config.column_config], + ); + const colorMapSig = useMemo(() => { + if (colorMapCols.length === 0) return ""; + const stats = extractSDFT(summary_stats_data); + return JSON.stringify(colorMapCols.map( + ({ field, statsCol }) => [field, statsCol === undefined ? undefined : stats[statsCol]?.histogram_bins])); + }, [colorMapCols, summary_stats_data]); + const colorMapSigRef = useRef(colorMapSig); + useEffect(() => { + if (colorMapSigRef.current === colorMapSig) return; + colorMapSigRef.current = colorMapSig; + if (colorMapCols.length === 0) return; + gridRef.current?.api?.refreshCells({ force: true, columns: colorMapCols.map((c) => c.field) }); + }, [colorMapSig, colorMapCols]); // Force update rowData when Raw data changes const rawDataSig = useMemo(() => { diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx index 907fef642..bfbfad3aa 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx @@ -24,11 +24,16 @@ export const getSimpleTooltip = (tooltipField:string) => { // This should be possible with the tooltipValueGetter, but that // wasn't working for some reason - if (props.data.index === "histogram") { + if (props.data?.index === "histogram") { return; } - const val = props.data[tooltipField].toString() - return
{val}
; + // A pinned row whose stats have not arrived has no value for the + // column, and a row that has not loaded has no data: show nothing. + const raw = props.data?.[tooltipField]; + if (raw === undefined || raw === null) { + return; + } + return
{raw.toString()}
; }; return simpleTooltip; } diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx index ef83da6ae..fcdd0baff 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx @@ -64,13 +64,14 @@ export function colorMap(cmr: ColorMapRules) { const summarys = params.context?.histogram_stats; const statsCol = cmr.val_column; // || col_name; + // No bins is the normal state until the summary stats arrive, so + // fall back to the neutral style quietly; the grid repaints the + // column when they do. if (statsCol === undefined || summarys === undefined){ - console.log("66 couldn't find stats_col") return baseReturn; } const summary_stats_cell = summarys[statsCol]; if (summary_stats_cell === undefined || summary_stats_cell.histogram_bins === undefined ) { - console.log("69 couldn't find summary_stats"); return baseReturn } const histogram_edges = summary_stats_cell.histogram_bins; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts index b963123e4..c9b29af66 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts @@ -38,6 +38,7 @@ import { getFormatterFromArgs, getCellRenderer, objFormatter, getFormatter } fro import { CSSProperties, Dispatch, SetStateAction } from "react"; import { CommandConfigT } from "../CommandUtils"; import { KeyAwareSmartRowCache, PayloadArgs } from "./SmartRowCache"; +import type { StatsStatus } from "../WidgetTypes"; // for now colDef stuff with less than 3 implementantions should stay in this file @@ -97,13 +98,33 @@ export function stripOptionalPinnedKey(key: string): string { return isOptionalPinnedKey(key) ? key.slice(1) : key; } -export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[]) { +// Marks a pinned row that stands in for stats that have not arrived. The row +// carries its key as `index`, so it keeps its label and gets a row id of its +// own, and the cell renderer selector leaves its value cells empty. +export const PENDING_STAT_ROW_KEY = "__stat_pending"; + +const PendingStatCell = () => null; +const pendingStatRenderer: CellRendererSelectorResult = { component: PendingStatCell }; + +// `statsStatus` is df_meta.stats.status. A required key with no value is, by +// status: +// "pending" a placeholder row, which holds the pinned area's height +// "not_computed" omitted, since no value is coming +// anything else undefined, as it was before df_meta.stats existed +export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[], statsStatus?: StatsStatus) { const result: (DFData[number] | undefined)[] = []; for (const cfg of prc) { const raw = cfg.primary_key_val; - const found = _.find(sdf, { index: stripOptionalPinnedKey(raw) }); - if (found === undefined && isOptionalPinnedKey(raw)) { - continue; + const key = stripOptionalPinnedKey(raw); + const found = _.find(sdf, { index: key }); + if (found === undefined) { + if (isOptionalPinnedKey(raw) || statsStatus === "not_computed") { + continue; + } + if (statsStatus === "pending") { + result.push({ index: key, [PENDING_STAT_ROW_KEY]: true }); + continue; + } } result.push(found); } @@ -340,6 +361,9 @@ export function getCellRendererSelector(pinned_rows: PinnedRowConfig[], column_c if (pk === undefined) { return anyRenderer; // default renderer } + if (_.get(params.node.data, PENDING_STAT_ROW_KEY) === true && params.column?.getColId() !== "index") { + return pendingStatRenderer; // a stat that has not arrived: leave the cell empty + } const maybePrc: PinnedRowConfig | undefined = _.find( pinned_rows, (cfg) => stripOptionalPinnedKey(cfg.primary_key_val) === pk, diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index e7f63015c..d15baac8e 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -1,11 +1,29 @@ +// Where the summary stats stand, as the server reports it in df_meta.stats. +// "pending": a stats_update is expected. "not_computed": none will be sent +// unless the user asks. A missing df_meta.stats means "complete", which is +// what servers that predate the field send. +export type StatsStatus = "complete" | "pending" | "not_computed" | "error"; + +export interface DFMetaStats { + status: StatsStatus; + tier?: string; + reason?: string; + gen?: number; +} + export interface DFMeta { // static, total_rows: number; columns: number; filtered_rows: number; rows_shown: number; + // Absent when the server predates the two-message protocol. + stats?: DFMetaStats; } +export const getStatsStatus = (meta: DFMeta | undefined): StatsStatus => + meta?.stats?.status ?? "complete"; + export interface BuckarooOptions { sampled: string[]; cleaning_method: string[]; diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 21e422a46..f26c4860d 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -18,6 +18,7 @@ import { BuckarooStaticTable } from './components/BuckarooStaticTable'; import { BuckarooServerView, buckarooWsUrl } from './server/BuckarooServerView'; import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; +import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -60,6 +61,7 @@ export default { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, }; // Named exports for direct imports @@ -89,6 +91,7 @@ export { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 3ede8298e..9c86f5bee 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -1,15 +1,15 @@ import * as React from "react"; import { BuckarooInfiniteWidget, DFViewerInfiniteDS, getKeySmartRowCache } from "../components/BuckarooWidgetInfinite"; -import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { DFMeta, BuckarooState, BuckarooOptions } from "../components/WidgetTypes"; import { CommandConfigT } from "../components/CommandUtils"; import { Operation } from "../components/OperationUtils"; import { OperationResult, baseOperationResults } from "../components/DependentTabs"; -import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFData } from "../components/DFViewerParts/DFWhole"; import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; +import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -161,6 +161,22 @@ export function BuckarooView({ const onMetadataRef = React.useRef(onMetadata); React.useEffect(() => { onMetadataRef.current = onMetadata; }, [onMetadata]); + // Every df_data_dict that reaches the view (the seed, change:df_data_dict, + // the metadata handler, the catch-up on subscribe) goes through one + // decoder, so a frame decodes once and the newest decode wins. The seed is + // already applied when it needs no resolution. + const dictDecoderRef = React.useRef<((raw: RawDFDataDict) => void) | null>(null); + if (dictDecoderRef.current === null) { + dictDecoderRef.current = makeLatestDictDecoder( + (d) => { + setDfDataDict(d as Record); + setDataReady(true); + }, + initialNeedsResolution ? undefined : (initialState.df_data_dict as RawDFDataDict), + ); + } + const loadDfDataDict = dictDecoderRef.current; + // Resolve any parquet-encoded payloads in df_data_dict. Pre-resolved // dicts (e.g. when BuckarooServerView already ran decodeDFDataDict) // pass through unchanged, so this is cheap in the common case. Skip @@ -168,19 +184,13 @@ export function BuckarooView({ // for the BuckarooServerView path. React.useEffect(() => { if (!initialNeedsResolution) return; - let cancelled = false; - const dict = initialState.df_data_dict as Record | undefined; + const dict = initialState.df_data_dict as RawDFDataDict; if (!dict) { setDataReady(true); return; } - decodeDFDataDict(dict).then((d) => { - if (cancelled) return; - setDfDataDict(d as Record); - setDataReady(true); - }); - return () => { cancelled = true; }; - }, [initialState, initialNeedsResolution]); + loadDfDataDict(dict); + }, [initialState, initialNeedsResolution, loadDfDataDict]); // Fire onMetadata for the initial payload, matching BuckarooServerView's // pre-split behavior. @@ -202,8 +212,7 @@ export function BuckarooView({ const onMeta = (metadata: BuckarooServerMetadata, prompt?: string) => { onMetadataRef.current?.(metadata, prompt); setDfMeta((model.get("df_meta") as DFMeta | undefined) ?? { ...DEFAULT_DF_META, total_rows: metadata?.rows ?? 0 }); - decodeDFDataDict((model.get("df_data_dict") as Record | undefined) ?? {}) - .then((d) => setDfDataDict(d as Record)); + loadDfDataDict(model.get("df_data_dict") as RawDFDataDict); setDfDisplayArgs((model.get("df_display_args") as Record | undefined) ?? {}); setBuckarooStateLocal((model.get("buckaroo_state") as BuckarooState | undefined) ?? DEFAULT_BUCKAROO_STATE); setBuckarooOptions((model.get("buckaroo_options") as BuckarooOptions | undefined) ?? DEFAULT_BUCKAROO_OPTIONS); @@ -212,9 +221,7 @@ export function BuckarooView({ setOperations((model.get("operations") as Operation[] | undefined) ?? []); }; const onDfMeta = (v: DFMeta) => setDfMeta(v); - const onDfDataDict = (v: Record) => { - decodeDFDataDict(v).then((d) => setDfDataDict(d as Record)); - }; + const onDfDataDict = (v: RawDFDataDict) => loadDfDataDict(v); const onDfDisplayArgs = (v: Record) => setDfDisplayArgs(v); const onBState = (v: BuckarooState) => setBuckarooStateLocal(v); const onBOpts = (v: BuckarooOptions) => setBuckarooOptions(v); @@ -232,6 +239,25 @@ export function BuckarooView({ model.on("change:operation_results", onOpRes); model.on("change:operations", onOps); + // A change:* emitted after the model was built and before this effect + // ran had no listener. The model holds the latest value of every key, + // so read each one now that the handlers are in place. A value that is + // the one already held is a no-op (same reference). + const catchUp: Array<[string, (v: any) => void]> = [ + ["df_meta", onDfMeta], + ["df_data_dict", onDfDataDict], + ["df_display_args", onDfDisplayArgs], + ["buckaroo_state", onBState], + ["buckaroo_options", onBOpts], + ["command_config", onCmdCfg], + ["operation_results", onOpRes], + ["operations", onOps], + ]; + for (const [key, apply] of catchUp) { + const v = model.get(key); + if (v !== undefined) apply(v); + } + return () => { model.off("metadata", onMeta); model.off("change:df_meta", onDfMeta); @@ -243,7 +269,7 @@ export function BuckarooView({ model.off("change:operation_results", onOpRes); model.off("change:operations", onOps); }; - }, [model]); + }, [model, loadDfDataDict]); const onBuckarooState = React.useCallback>>((newState) => { const resolved = typeof newState === "function" diff --git a/packages/buckaroo-js-core/src/server/latestDictDecoder.ts b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts new file mode 100644 index 000000000..16c4c3ef9 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts @@ -0,0 +1,34 @@ +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; + +export type RawDFDataDict = Record | undefined | null; + +/** + * Decode df_data_dict values arriving from a model and hand `apply` only the + * newest result. + * + * A full `initial_state` reaches a view twice, as `change:df_data_dict` and + * then as `metadata`, and both handlers read the same dict off the model. The + * returned function ignores a dict it has already seen (same reference), so + * that frame decodes once. Decoding is async, so a slow decode of an earlier + * dict can finish after a later one; each call takes a token, and a result is + * applied only if no later call has started. + * + * `seen` is a dict the caller has already applied (the view's seed), so the + * first read of that same dict off the model does not decode it again. + */ +export function makeLatestDictDecoder( + apply: (decoded: Record) => void, + seen?: RawDFDataDict, +): (raw: RawDFDataDict) => void { + let latest = 0; + let lastRaw: RawDFDataDict = seen; + return (raw) => { + if (raw === lastRaw) return; + lastRaw = raw; + const mine = ++latest; + decodeDFDataDict(raw).then((decoded) => { + if (mine === latest) apply(decoded); + }); + }; +} diff --git a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx index caa5628c1..22e3e0c44 100644 --- a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx @@ -9,9 +9,8 @@ * - "complete" stats are present; also the meaning of a missing * `df_meta.stats`, as servers without the field send * - * Column `a` is color-mapped and has a simple tooltip, so the story also shows - * that cells restyle when the histogram bins arrive and that hovering a - * valueless pinned cell does nothing. Used by stats-pending-pinned-rows.spec.ts. + * Column `a` is color-mapped, so the story also shows that its cells restyle + * when the histogram bins arrive. Used by stats-pending-pinned-rows.spec.ts. */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; @@ -45,7 +44,6 @@ const viewerConfig: DFViewerConfig = { header_name: "a", displayer_args: { displayer: "obj" }, color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, - tooltip_config: { tooltip_type: "simple", val_column: "a" }, }, { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, ], diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index 715312832..37f191c31 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -75,28 +75,57 @@ function updateFilenameDisplay(metadata: any, prompt?: string) { } } +/** What the model held for `keys` when a component first rendered. */ +function snapshotModel(model: WebSocketModel, keys: string[]): Record { + return Object.fromEntries(keys.map((k) => [k, model.get(k)])); +} + +/** A change:* emitted between the first render and the effect that registers + * the listeners had no listener. The model holds the latest value of every + * key, so apply each one that differs from what the first render read. */ +function catchUpOnModel( + model: WebSocketModel, + seed: Record, + appliers: Record void>, +) { + for (const [key, apply] of Object.entries(appliers)) { + const v = model.get(key); + if (v !== undefined && v !== seed[key]) apply(v); + } +} + function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { const [dfMeta, setDfMeta] = React.useState(model.get("df_meta") || { total_rows: 0 }); const [dfDataDict, setDfDataDict] = React.useState(model.get("df_data_dict") || {}); const [dfDisplayArgs, setDfDisplayArgs] = React.useState(patchDisplayArgsHeight(model.get("df_display_args") || {})); + const seed = React.useRef(snapshotModel(model, ["df_meta", "df_data_dict", "df_display_args"])); + // A full frame reaches the view as change:df_data_dict and again as + // metadata; this decodes each dict once and applies only the newest decode. + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: metadata.rows || 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); }; model.on("metadata", onMeta); const onDfMeta = (v: any) => setDfMeta(v); - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); model.on("change:df_meta", onDfMeta); model.on("change:df_data_dict", onDfDataDict); model.on("change:df_display_args", onDfDisplayArgs); + catchUpOnModel(model, seed.current, { + df_meta: onDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + }); // Catch up on metadata that arrived before useEffect registered listeners const existingMeta = model.get("metadata"); @@ -110,7 +139,7 @@ function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { model.off("change:df_data_dict", onDfDataDict); model.off("change:df_display_args", onDfDisplayArgs); }; - }, [model]); + }, [model, loadDict]); if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
@@ -140,12 +169,20 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { const [commandConfig, setCommandConfig] = React.useState(model.get("command_config") || {}); const [operationResults, setOperationResults] = React.useState(model.get("operation_results") || {}); const [operations, setOperations] = React.useState(model.get("operations") || []); + const seed = React.useRef(snapshotModel(model, [ + "df_meta", "df_data_dict", "df_display_args", "buckaroo_state", "buckaroo_options", + "command_config", "operation_results", "operations", + ])); + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); setBuckarooState(model.get("buckaroo_state") || {}); setBuckarooOptions(model.get("buckaroo_options") || {}); @@ -168,9 +205,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { }; // df_data_dict needs async pre-resolution of parquet_b64 values - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); model.on("change:df_data_dict", onDfDataDict); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); @@ -185,6 +220,16 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { ["operation_results", onChange("operation_results", setOperationResults)], ["operations", onChange("operations", setOperations)], ]; + catchUpOnModel(model, seed.current, { + df_meta: setDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + buckaroo_state: setBuckarooState, + buckaroo_options: setBuckarooOptions, + command_config: setCommandConfig, + operation_results: setOperationResults, + operations: setOperations, + }); return () => { model.off("metadata", onMeta); @@ -193,7 +238,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.off(`change:${key}`, handler); } }; - }, [model]); + }, [model, loadDict]); const onBuckarooState = React.useCallback((newState: any) => { // newState may be a value or a setter function From fe046acc7907af09754aa19eb7b5f91410c68459 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:22:28 -0400 Subject: [PATCH 3/9] test(client): failing tests for the stats_update channel (rows-first c2) Jest tests, driven through a WebSocketModel with a fake socket, for the client half of the stats wire: a stats_update is key-merged into all_stats and dropped when its stats_gen is not the expected one, the expected gen follows df_meta.stats.gen on every applied initial_state, a merge in flight is discarded or redone when a frame replaces the dict, and stats_aborted moves the status. Plus withStatsCapability and BuckarooServerView putting ?caps=stats_update on the WebSocket URL, and a server Playwright test that the standalone page does the same. StatsChannel.ts is a stub (identity withStatsCapability) so the tests fail on assertions rather than on a missing module. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 16 + .../server/BuckarooServerView.caps.test.tsx | 84 ++++++ .../src/server/StatsChannel.test.ts | 278 ++++++++++++++++++ .../src/server/StatsChannel.ts | 6 + 4 files changed, 384 insertions(+) create mode 100644 packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx create mode 100644 packages/buckaroo-js-core/src/server/StatsChannel.test.ts create mode 100644 packages/buckaroo-js-core/src/server/StatsChannel.ts diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index ad9634a05..3327d908a 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -485,6 +485,22 @@ test.describe('WebSocket data flow', () => { expect(await getCellText(page, COL.name, 0)).toBe('Alice'); }); + test('standalone page advertises stats_update on its WebSocket URL', async ({ page, request }) => { + const session = `ws-caps-${Date.now()}`; + await loadSession(request, session, csvPath); + + // The server reads capabilities from ?caps= at open, before the client + // sends anything, so the page has to put them on the URL it connects to. + const socketUrls: string[] = []; + page.on('websocket', (socket) => socketUrls.push(socket.url())); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + const wsUrls = socketUrls.filter((u) => u.includes(`/ws/${session}`)); + expect(wsUrls).toHaveLength(1); + expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update'); + }); + test('WebSocket receives data for scrolled rows', async ({ page, request }) => { // Create a larger dataset (100 rows) to force infinite scrolling const rows = []; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx new file mode 100644 index 000000000..21e8b8421 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx @@ -0,0 +1,84 @@ +/** + * BuckarooServerView — capability advertisement (rows-first c2). + * + * The server records a client's capabilities from `?caps=` on the WebSocket + * URL, because it sends the first message before the client says anything. A + * client that merges `stats_update` must put it there, whatever URL the host + * passes in. + */ +import { render, cleanup, waitFor } from "@testing-library/react"; +import { BuckarooServerView } from "./BuckarooServerView"; + +const capturedViewProps: any[] = []; + +jest.mock("./BuckarooView", () => ({ + BuckarooView: (props: any) => { + capturedViewProps.push(props); + return
; + }, + pickMode: (m: unknown) => (m === "buckaroo" ? "buckaroo" : "viewer"), +})); + +jest.mock("./WebSocketModel", () => ({ + WebSocketModel: class { constructor(_ws: any, _state: any) {} }, +})); + +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + binaryType = "arraybuffer"; + onopen: (() => void) | null = null; + onerror: ((e: any) => void) | null = null; + private listeners: Record void>> = {}; + constructor(public url: string) { + FakeWebSocket.instances.push(this); + setTimeout(() => { + this.onopen?.(); + setTimeout(() => { + const initial = { + type: "initial_state", + df_meta: { total_rows: 4, columns: 2, filtered_rows: 4, rows_shown: 4 }, + df_data_dict: {}, + df_display_args: {}, + mode: "viewer", + }; + this.listeners["message"]?.forEach((h) => h({ data: JSON.stringify(initial) } as any)); + }, 0); + }, 0); + } + addEventListener(ev: string, h: (e: any) => void) { + (this.listeners[ev] ??= new Set()).add(h); + } + removeEventListener(ev: string, h: (e: any) => void) { + this.listeners[ev]?.delete(h); + } + close() {} +} + +const origWebSocket = (globalThis as any).WebSocket; + +beforeAll(() => { + (globalThis as any).WebSocket = FakeWebSocket; +}); +afterAll(() => { + (globalThis as any).WebSocket = origWebSocket; +}); +afterEach(() => { + capturedViewProps.length = 0; + FakeWebSocket.instances.length = 0; + cleanup(); +}); + +describe("BuckarooServerView advertises stats_update", () => { + it("opens the socket with ?caps=stats_update", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?caps=stats_update"); + }); + + it("keeps the query string the host passed", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?token=abc&caps=stats_update"); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts new file mode 100644 index 000000000..70a8f7c8d --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -0,0 +1,278 @@ +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A capable client receives a stats-free `initial_state` (df_meta.stats.status + * "pending"), asks for the stats and merges the `stats_update` that answers, + * keyed by `stats_gen`. These tests drive a WebSocketModel with a fake socket, + * the way the server's frames reach it. + */ +import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; + +// The real decoder, except that an envelope carrying `hold` waits on `gate`, +// so a test can deliver a frame while a decode is in flight. +let gate: Promise = Promise.resolve(); +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { + ...actual, + decodeDFData: jest.fn(async (env: any, buffers?: DataView[]) => { + if (env && env.hold) await gate; + return actual.decodeDFData(env, buffers); + }), + }; +}); + +class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } +} + +// Lets every promise continuation (the payload decode) run. +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +const row = (stat: string, cells: Record) => ({ index: stat, level_0: stat, ...cells }); + +// What the schema tier ships: identity rows only. +const schemaStats = () => [ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 3, c: 3 }), +]; + +const metaFor = (stats?: Record) => ({ + total_rows: 3, columns: 3, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); + +const frame = (gen: number | undefined, allStats: any = schemaStats()) => ({ + type: "initial_state", + df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, +}); + +// The server's payload is a wide DFEnvelope; the json format decodes to the +// same row shape without a parquet fixture. +const update = (gen: number, rows: any[], extra: object = {}) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final: true, + payload: { format: "json", layout: "wide", data: rows }, + elapsed_ms: 12.5, + ...extra, +}); + +function makeModel(gen: number | undefined = 3, allStats: any = schemaStats()) { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, + }); + const events: { event: string; value: any }[] = []; + for (const key of ["df_data_dict", "df_meta"]) { + model.on(`change:${key}`, (value: any) => events.push({ event: `change:${key}`, value })); + } + return { ws, model, events }; +} + +describe("withStatsCapability", () => { + it("adds ?caps=stats_update to a bare URL", () => { + expect(withStatsCapability("ws://localhost:8700/ws/sales")).toBe("ws://localhost:8700/ws/sales?caps=stats_update"); + }); + + it("appends to an existing query string", () => { + expect(withStatsCapability("ws://h/ws/s?token=abc")).toBe("ws://h/ws/s?token=abc&caps=stats_update"); + }); + + it("extends an existing caps value", () => { + expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update"); + }); + + it("keeps the fragment last", () => { + expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update#frag"); + }); +}); + +describe("stats_update merge", () => { + it("key-merges the payload's columns into all_stats and keeps the other columns", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [ + row("length", { a: 3, b: 5 }), + row("mean", { a: 2, b: 4.5 }), + ])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 5, c: 3 }), + row("mean", { a: 2, b: 4.5 }), + ]); + }); + + it("does not let a null in the payload erase a value already merged", async () => { + const { ws, model } = makeModel(3); + // The wide pivot pads a stat a column did not carry with null. + ws.deliver(update(3, [row("dtype", { a: null, b: "float32" })])); + await settle(); + const dtype = model.get("df_data_dict").all_stats.find((r: any) => r.index === "dtype"); + expect(dtype).toEqual(row("dtype", { a: "int64", b: "float32", c: "object" })); + }); + + it("assigns a new df_data_dict and leaves the previous objects untouched", async () => { + const { ws, model, events } = makeModel(3); + const before = model.get("df_data_dict"); + const beforeStats = before.all_stats; + const snapshot = JSON.parse(JSON.stringify(beforeStats)); + ws.deliver(update(3, [row("length", { a: 99 }), row("mean", { a: 2 })])); + await settle(); + const after = model.get("df_data_dict"); + expect(after).not.toBe(before); + expect(after.all_stats).not.toBe(beforeStats); + expect(beforeStats).toEqual(snapshot); + const dictEvents = events.filter((e) => e.event === "change:df_data_dict"); + expect(dictEvents).toHaveLength(1); + expect(dictEvents[0].value).toBe(after); + }); + + it("merges onto an all_stats that arrived as an undecoded envelope", async () => { + const { ws, model } = makeModel(3); + // A later initial_state hands the model the dict as the server sent it. + ws.deliver(frame(4, { format: "json", layout: "wide", data: schemaStats() })); + ws.deliver(update(4, [row("mean", { a: 2, b: 4.5, c: null })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(Array.isArray(stats)).toBe(true); + expect(stats.map((r: any) => r.index)).toEqual(["dtype", "length", "mean"]); + }); + + it("keeps the other df_data_dict keys", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { all_stats: schemaStats(), empty: [], main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const dict = model.get("df_data_dict"); + expect(dict.empty).toEqual([]); + expect(dict.main).toEqual([{ a: 1 }]); + expect(dict.all_stats).toHaveLength(3); + }); + + it("applies updates that arrive back to back, in order", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + ws.deliver(update(3, [row("mean", { b: 4.5 }), row("max", { a: 9 })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats.find((r: any) => r.index === "mean")).toEqual(row("mean", { a: 2, b: 4.5 })); + expect(stats.find((r: any) => r.index === "max")).toEqual(row("max", { a: 9 })); + }); +}); + +describe("stats_update and df_meta.stats", () => { + it("a final update marks the stats complete at the update's tier", async () => { + const { ws, model, events } = makeModel(3); + const metaBefore = model.get("df_meta"); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const meta = model.get("df_meta"); + expect(meta).not.toBe(metaBefore); + expect(meta.stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + expect(meta.total_rows).toBe(3); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(1); + }); + + it("a non-final update merges but leaves the status pending", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + expect(model.get("df_meta").stats.status).toBe("pending"); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(0); + }); +}); + +describe("stats_gen", () => { + it("drops a stats_update whose stats_gen is not the expected one", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(2, [row("mean", { a: 2 })])); + ws.deliver(update(3, [row("max", { a: 9 })])); + await settle(); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "max"]); + expect(events.filter((e) => e.event === "change:df_data_dict")).toHaveLength(1); + }); + + it("advances the expected gen on a broadcast initial_state with no reply_seq", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(4)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + }); + + it("discards a merge when an initial_state for a newer gen arrives while it decodes", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })])); + // Still decoding: the model has not merged anything yet. + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" }), row("mean", { a: 2 })]); + }); + + it("merges onto the new dict when a same-gen initial_state replaces it while the old one decodes", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const held = { format: "json", layout: "wide", data: schemaStats(), hold: true }; + const { ws, model } = makeModel(3, held); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the update is now waiting on the held decode + expect((decodeDFData as jest.Mock).mock.calls.some(([env]) => env === held)).toBe(true); + ws.deliver(frame(3, [row("dtype", { a: "int32" })])); + release(); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int32" }), + row("mean", { a: 2 }), + ]); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + +describe("stats_aborted", () => { + it("marks the stats failed when the run for the expected gen failed", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" }); + }); + + it("marks the stats not computed when the server says they cannot be requested", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("not_computed"); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts new file mode 100644 index 000000000..2ef586390 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -0,0 +1,6 @@ +// Stub: the implementation lands in the next commit. +export const STATS_UPDATE_CAP = "stats_update"; + +export function withStatsCapability(wsUrl: string): string { + return wsUrl; +} From 3118435aa40753e293f954cd5a00fbad02d905c2 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:28:27 -0400 Subject: [PATCH 4/9] test(client): failing tests for stats_update payloads and an empty dict (rows-first c2) Three more cases for the stats_update merge, found untested after the first test commit: a wide parquet_b64 payload as the server sends it (the shared summary_stats fixture, not a json envelope), a model that holds no df_data_dict yet, and a dict with no all_stats key. Co-Authored-By: Claude Sonnet 5.5 --- .../src/server/StatsChannel.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index 70a8f7c8d..ad675f03c 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -10,6 +10,11 @@ import { WebSocketModel } from "./WebSocketModel"; import { withStatsCapability } from "./StatsChannel"; import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; +// A wide summary-stats envelope (parquet_b64, layout "wide") as the server +// sends one; the decoder tests use the same fixture. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const wideFixture = require("../components/DFViewerParts/test-fixtures/summary_stats_parquet_b64.json"); + // The real decoder, except that an envelope carrying `hold` waits on `gate`, // so a test can deliver a frame while a decode is in flight. let gate: Promise = Promise.resolve(); @@ -169,6 +174,38 @@ describe("stats_update merge", () => { expect(dict.all_stats).toHaveLength(3); }); + it("merges a wide parquet_b64 payload as the server sends it", async () => { + const { ws, model } = makeModel(3, [row("orig_col_name", { a: "first" })]); + const decoded: any[] = await decodeDFData(wideFixture); + expect(decoded.length).toBeGreaterThan(1); + ws.deliver(update(3, [], { payload: wideFixture })); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats[0]).toEqual(row("orig_col_name", { a: "first" })); + for (const decodedRow of decoded) { + expect(stats).toContainEqual(decodedRow); + } + }); + + it("builds all_stats when the model holds no dict yet", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { df_meta: metaFor(pending(3)) }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ all_stats: [row("mean", { a: 2 })] }); + }); + + it("adds all_stats to a dict that has none", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ main: [{ a: 1 }], all_stats: [row("mean", { a: 2 })] }); + }); + it("applies updates that arrive back to back, in order", async () => { const { ws, model } = makeModel(3); ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); From 1fd9cff8159741f99a8dace40142394f9ce66728 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:32:11 -0400 Subject: [PATCH 5/9] test(client): failing test for stats_update ordering (rows-first c2) Two stats_update messages for one gen are applied in arrival order even when the first payload decodes more slowly: the final update merges last and the status completes only after both merges. Co-Authored-By: Claude Sonnet 5.5 --- .../src/server/StatsChannel.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index ad675f03c..a6b61e880 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -217,6 +217,30 @@ describe("stats_update merge", () => { }); }); +describe("stats_update order", () => { + it("merges updates in arrival order when an earlier payload decodes slower", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const { ws, model, events } = makeModel(3); + const held = { format: "json", layout: "wide", data: [row("mean", { a: 1 })], hold: true }; + ws.deliver(update(3, [], { final: false, payload: held })); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the second payload has decoded; the first is held + release(); + await settle(); + // The final update goes last: its value stands and the status + // completes only after both merges. + const mean = model.get("df_data_dict").all_stats.find((r: any) => r.index === "mean"); + expect(mean.a).toBe(2); + expect(events.map((e) => e.event)).toEqual(["change:df_data_dict", "change:df_data_dict", "change:df_meta"]); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + describe("stats_update and df_meta.stats", () => { it("a final update marks the stats complete at the update's tier", async () => { const { ws, model, events } = makeModel(3); From 5935432d6926ec10409c90449829742cccc50f9e Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:38:28 -0400 Subject: [PATCH 6/9] feat(client): merge stats_update into all_stats and advertise the capability (rows-first c2) StatsChannel is the client half of the stats wire. WebSocketModel hands it every text frame first. A stats_update for the gen the model's df_meta reports is decoded and key-merged into df_data_dict.all_stats (new row objects, a new dict), updates apply one at a time in arrival order, and a final update sets df_meta.stats to complete at the update's tier. A merge is dropped when the gen moves on while it decodes and redone when a frame replaces the dict. stats_aborted for the expected gen sets error or not_computed. Other message types are ignored as before. withStatsCapability adds caps=stats_update to a WebSocket URL; both wiring copies (BuckarooServerView and the standalone page) use it so the server treats them as capable clients. The guard tests that already pass before the fix (caps already present, no df_meta.stats, ignored aborts, unknown types, infinite_resp pairing) are added here, with a makeModel(null) helper so a model without stats can be built. Co-Authored-By: Claude Sonnet 5.5 --- packages/buckaroo-js-core/src/index.ts | 3 + .../src/server/BuckarooServerView.tsx | 4 +- .../src/server/StatsChannel.test.ts | 66 +++++- .../src/server/StatsChannel.ts | 208 +++++++++++++++++- .../src/server/WebSocketModel.ts | 7 + packages/js/standalone.tsx | 3 +- 6 files changed, 285 insertions(+), 6 deletions(-) diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index f26c4860d..11d10f022 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -19,6 +19,7 @@ import { BuckarooServerView, buckarooWsUrl } from './server/BuckarooServerView'; import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; +import { withStatsCapability } from './server/StatsChannel'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -62,6 +63,7 @@ export default { buckarooWsUrl, WebSocketModel, makeLatestDictDecoder, + withStatsCapability, }; // Named exports for direct imports @@ -92,6 +94,7 @@ export { buckarooWsUrl, WebSocketModel, makeLatestDictDecoder, + withStatsCapability, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx index 49fedbe3a..1687dddaa 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx @@ -2,6 +2,7 @@ import * as React from "react"; import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; import { BuckarooView, BuckarooServerMetadata, @@ -106,7 +107,8 @@ export function BuckarooServerView({ (async () => { try { - ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + ws = new WebSocket(withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; await new Promise((resolve, reject) => { diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index a6b61e880..9d2caa839 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -78,10 +78,11 @@ const update = (gen: number, rows: any[], extra: object = {}) => ({ ...extra, }); -function makeModel(gen: number | undefined = 3, allStats: any = schemaStats()) { +// `gen` null builds a model whose df_meta carries no stats, as an old server's does. +function makeModel(gen: number | null = 3, allStats: any = schemaStats()) { const ws = new FakeSocket(); const model = new WebSocketModel(ws as unknown as WebSocket, { - df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_meta: metaFor(gen === null ? undefined : pending(gen)), df_data_dict: { all_stats: allStats }, }); const events: { event: string; value: any }[] = []; @@ -104,6 +105,11 @@ describe("withStatsCapability", () => { expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update"); }); + it("leaves a URL that already advertises the capability alone", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update")).toBe("ws://h/ws/s?caps=stats_update"); + expect(withStatsCapability("ws://h/ws/s?caps=a,stats_update")).toBe("ws://h/ws/s?caps=a,stats_update"); + }); + it("keeps the fragment last", () => { expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update#frag"); }); @@ -274,6 +280,22 @@ describe("stats_gen", () => { expect(events.filter((e) => e.event === "change:df_data_dict")).toHaveLength(1); }); + it("drops every stats_update when the server reported no stats", async () => { + const { ws, model, events } = makeModel(null); + ws.deliver(update(0, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + expect(events).toHaveLength(0); + }); + + it("clears the expectation when an initial_state carries no df_meta.stats", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(undefined)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + }); + it("advances the expected gen on a broadcast initial_state with no reply_seq", async () => { const { ws, model } = makeModel(3); ws.deliver(frame(4)); @@ -336,4 +358,44 @@ describe("stats_aborted", () => { await settle(); expect(model.get("df_meta").stats.status).toBe("not_computed"); }); + + it("changes nothing for a stale reply, an unsupported scope or a session with no data", async () => { + const { ws, events } = makeModel(3); + for (const reason of ["stale", "unsupported_scope", "no_data"]) { + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 4, scope: "raw", reason }); + } + await settle(); + expect(events).toHaveLength(0); + }); + + it("ignores a reply for a gen the client has left", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 2, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("pending"); + }); +}); + +describe("other messages", () => { + it("ignores a type it does not know and keeps applying frames", async () => { + const { ws, model, events } = makeModel(3); + expect(() => ws.deliver({ type: "something_new", stats_gen: 3 })).not.toThrow(); + ws.deliver({ type: "error", message: "boom" }); + await settle(); + expect(events).toHaveLength(0); + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + }); + + it("still pairs an infinite_resp with the binary frame that follows it", () => { + const { ws, model } = makeModel(3); + const seen: [any, DataView[]][] = []; + model.on("msg:custom", (msg: any, buffers: DataView[]) => seen.push([msg, buffers])); + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 5 }, length: 5 }); + ws.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + expect(seen).toHaveLength(1); + expect(seen[0][0].type).toBe("infinite_resp"); + expect(seen[0][1][0].byteLength).toBe(8); + }); }); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts index 2ef586390..0ead005ed 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -1,6 +1,210 @@ -// Stub: the implementation lands in the next commit. +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A client that advertises `?caps=stats_update` gets a first `initial_state` + * whose `df_meta.stats` says the stats are pending, asks for them with + * `stats_request {stats_gen, scope}`, and receives either + * + * stats_update {stats_gen, scope, tier, final, payload, elapsed_ms} + * stats_aborted {stats_gen, current_gen?, scope, reason} + * + * `payload` is an inline wide DFEnvelope holding `all_stats`. `stats_gen` is the + * server's counter for the state the stats describe; it rides on every + * `initial_state` as `df_meta.stats.gen`, and a reply for any other gen is for + * a state the client has left. Sending the request is the scheduler's job, not + * this module's. + * + * Merge semantics are WebSocket-only: `WebSocketModel` hands every frame to + * `handle()` first, and Jupyter's widget sets `df_data_dict` whole. + */ +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFMeta, DFMetaStats } from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +/** The capability this client advertises, as one value of `?caps=` on the + * WebSocket URL: it merges `stats_update` messages. The server records it per + * connection when the socket opens, since it sends the first message before + * the client can say anything. */ export const STATS_UPDATE_CAP = "stats_update"; +const decodeQueryValue = (value: string): string => { + try { + return decodeURIComponent(value); + } catch { + return value; + } +}; + +/** `wsUrl` with `caps=stats_update` added: a new query on a bare URL, a new + * parameter after an existing query, or a comma-joined value when the host + * already passes `caps`. The fragment stays last and other parameters are + * left as the host wrote them. */ export function withStatsCapability(wsUrl: string): string { - return wsUrl; + const hashAt = wsUrl.indexOf("#"); + const fragment = hashAt === -1 ? "" : wsUrl.slice(hashAt); + const beforeFragment = hashAt === -1 ? wsUrl : wsUrl.slice(0, hashAt); + const queryAt = beforeFragment.indexOf("?"); + const path = queryAt === -1 ? beforeFragment : beforeFragment.slice(0, queryAt); + const params = queryAt === -1 ? [] : beforeFragment.slice(queryAt + 1).split("&").filter((p) => p !== ""); + + const capsAt = params.findIndex((p) => p === "caps" || p.startsWith("caps=")); + if (capsAt === -1) { + params.push(`caps=${STATS_UPDATE_CAP}`); + } else { + const caps = decodeQueryValue(params[capsAt].slice("caps=".length)) + .split(",") + .map((cap) => cap.trim()) + .filter((cap) => cap !== ""); + if (!caps.includes(STATS_UPDATE_CAP)) caps.push(STATS_UPDATE_CAP); + params[capsAt] = `caps=${caps.join(",")}`; + } + return `${path}?${params.join("&")}${fragment}`; +} + +export interface StatsUpdateMessage { + type: "stats_update"; + stats_gen: number; + scope?: string; + tier?: string; + final?: boolean; + payload?: DFDataOrPayload; + elapsed_ms?: number; +} + +export interface StatsAbortedMessage { + type: "stats_aborted"; + /** The request's gen. */ + stats_gen?: number; + /** The session's gen, omitted when the session has no data. */ + current_gen?: number; + scope?: string; + reason?: "stale" | "unsupported_scope" | "not_requestable" | "error" | "no_data"; +} + +/** + * Key-merge a stats payload into `all_stats`. Both are row-per-stat tables + * (`{index: , : , ...}`). For each stat row the payload + * names, every column it carries replaces that cell, columns it does not carry + * keep theirs, and a stat the table lacks is appended. + * + * A `null` in the payload never replaces a value already merged: the wide + * pivot fills a stat a column did not carry in this message with `null`. + * + * Returns new row objects and a new array. `base` may be the decoder's cached + * array, so nothing in it is touched. + */ +export function mergeStatRows(base: DFData, update: DFData): DFData { + const merged = base.slice(); + const at = new Map(); + merged.forEach((row, i) => at.set(row.index, i)); + + for (const updateRow of update) { + const i = at.get(updateRow.index); + if (i === undefined) { + at.set(updateRow.index, merged.length); + merged.push({ ...updateRow }); + continue; + } + const row = { ...merged[i] }; + for (const [column, value] of Object.entries(updateRow)) { + if (column === "index" || column === "level_0") continue; + if (value === null && row[column] != null) continue; + row[column] = value; + } + merged[i] = row; + } + return merged; +} + +const genOf = (meta: DFMeta | undefined): number | undefined => { + const gen = meta?.stats?.gen; + return typeof gen === "number" ? gen : undefined; +}; + +/** What the channel needs of a model. */ +type StatsModel = Pick; + +export class StatsChannel { + // Updates are applied one at a time: each reads the dict the previous one + // wrote, so a later update cannot overwrite an earlier one's merge. + private applying: Promise = Promise.resolve(); + + constructor(private model: StatsModel) {} + + /** + * The stats_gen of the state the client is showing, or `undefined` when the + * server reports none; a `stats_request` carries it. It is read off the + * model's `df_meta` each time, so it starts from the frame the model was + * built from and every applied `initial_state` (a broadcast frame with no + * `reply_seq` included) moves it. A `df_meta` with no `stats.gen` leaves + * nothing expected, so a server that stops reporting stats cannot have a + * late `stats_update` merged. Reading the model, not the message, keeps this + * correct under any scheme that drops stale `initial_state` frames. + */ + get expectedGen(): number | undefined { + return genOf(this.model.get("df_meta")); + } + + /** Consume a stats message. Returns false for every other message type, + * which the model handles (or ignores) as before. */ + handle(msg: { type?: string }): boolean { + if (msg.type === "stats_update") { + this.receiveUpdate(msg as StatsUpdateMessage); + return true; + } + if (msg.type === "stats_aborted") { + this.receiveAborted(msg as StatsAbortedMessage); + return true; + } + return false; + } + + private receiveUpdate(msg: StatsUpdateMessage): void { + if (msg.stats_gen !== this.expectedGen) return; + this.applying = this.applying + .then(() => this.applyUpdate(msg)) + .catch((e) => console.error("[StatsChannel] stats_update failed:", e)); + } + + private async applyUpdate(msg: StatsUpdateMessage): Promise { + const update = await decodeDFData(msg.payload); + for (;;) { + // A frame may have moved the gen on while something decoded. + if (msg.stats_gen !== this.expectedGen) return; + const dict: Record | null | undefined = this.model.get("df_data_dict"); + // The dict is decoded when the seed built it and raw when a later + // initial_state did. + const base = await decodeDFData(dict?.all_stats); + // A frame replaced the dict (or moved the gen) while it decoded: + // start over from the new one. + if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; + this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + if (msg.final) { + this.replaceStats((stats) => ({ status: "complete", tier: msg.tier ?? stats.tier, gen: stats.gen })); + } + return; + } + } + + private receiveAborted(msg: StatsAbortedMessage): void { + // Only a reply to a request for the state on screen says anything about + // it. A `stale` reply is answered by the initial_state that carries the + // new gen; taking `current_gen` without that frame would merge stats + // for a state the client has not seen into the one it shows. + if (msg.stats_gen !== this.expectedGen) return; + if (msg.reason === "error") { + this.replaceStats((stats) => ({ ...stats, status: "error", reason: "stats_failed" })); + } else if (msg.reason === "not_requestable") { + this.replaceStats((stats) => ({ ...stats, status: "not_computed" })); + } + } + + /** Replace `df_meta.stats` in a new `df_meta`, the reference that c0a's + * `inFlight` rule and the pinned rows' placeholders key on. */ + private replaceStats(change: (stats: DFMetaStats) => DFMetaStats): void { + const meta: DFMeta | undefined = this.model.get("df_meta"); + if (!meta?.stats) return; + this.model.set("df_meta", { ...meta, stats: change(meta.stats) }); + } } diff --git a/packages/buckaroo-js-core/src/server/WebSocketModel.ts b/packages/buckaroo-js-core/src/server/WebSocketModel.ts index 1e1fd4a2d..e7e5798d2 100644 --- a/packages/buckaroo-js-core/src/server/WebSocketModel.ts +++ b/packages/buckaroo-js-core/src/server/WebSocketModel.ts @@ -13,21 +13,28 @@ * Binary protocol (matching anywidget's msg + buffers pattern): * Server sends a JSON text frame (infinite_resp), then a binary frame (Parquet). * This class pairs them and emits "msg:custom" with (msg, [DataView]). + * + * stats_update and stats_aborted frames go to `stats` (see StatsChannel). */ +import { StatsChannel } from "./StatsChannel"; + export class WebSocketModel { private ws: WebSocket; private pendingMsg: any = null; private handlers: Map> = new Map(); private state: Record; private pendingChanges: Set = new Set(); + readonly stats: StatsChannel; constructor(ws: WebSocket, initialState: Record) { this.state = { ...initialState }; this.ws = ws; + this.stats = new StatsChannel(this); this.ws.onmessage = (event: MessageEvent) => { if (typeof event.data === "string") { const msg = JSON.parse(event.data); + if (this.stats.handle(msg)) return; if (msg.type === "infinite_resp") { // Expect a following binary frame — stash this JSON diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index 37f191c31..d7ccfe48f 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -287,7 +287,8 @@ async function main() { // Show connecting state rootEl.textContent = "Connecting..."; - const ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + const ws = new WebSocket(srt.withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; // Wait for connection From dd6ebe6a2b5f700779b67761d2c869178ed641a7 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:20:51 -0400 Subject: [PATCH 7/9] test(client): failing tests for the stats scheduler and its states (rows-first c4) Jest tests for the scheduler (StateOrchestrator rewritten around an IModel: a stats_request after the first infinite_resp, one request per reply until final, nothing after a state change, nothing for a search_string-only change), for the status bar's stats column and Compute summary stats control, for the widget and BuckarooView wiring of that control, and for pinned rows in the error state. Playwright tests on Storybook cover the four states without layout shift and the control's request, and the server spec runs the real standalone page against a session whose first frame says the stats are pending. StateOrchestrator.ts, StatusBar.tsx and BuckarooWidgetInfinite.tsx carry only the signatures the tests use, so the tests fail on assertions. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 89 +++ .../pw-tests/stats-scheduler-states.spec.ts | 96 +++ .../BuckarooInfiniteWidget.flash.test.tsx | 32 + .../src/components/BuckarooWidgetInfinite.tsx | 3 + .../DFViewerParts/gridUtils.test.ts | 12 + .../src/components/StatusBar.stats.test.tsx | 150 ++++ .../src/components/StatusBar.tsx | 9 +- .../src/server/BuckarooView.stats.test.tsx | 89 +++ .../src/server/StateOrchestrator.test.ts | 643 +++++++++++++----- .../src/server/StateOrchestrator.ts | 159 +---- .../stories/StatsSchedulerStates.stories.tsx | 170 +++++ scripts/test_playwright_storybook.sh | 1 + 12 files changed, 1165 insertions(+), 288 deletions(-) create mode 100644 packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts create mode 100644 packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx create mode 100644 packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx create mode 100644 packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index 3327d908a..bf98d7d19 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -501,6 +501,95 @@ test.describe('WebSocket data flow', () => { expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update'); }); + // Rows-first c4: the standalone page's scheduler. No session the server can + // build here defers its stats (that needs a xorq expression), so these + // tests put the pending state on a real session's first frame and answer + // the page's stats_request themselves. Rows, config and the stats payload + // all come from the real server. + async function loadBuckarooSession(request: any, sessionId: string) { + const resp = await request.post(`${BASE}/load`, { + data: { session: sessionId, path: csvPath, mode: 'buckaroo' }, + }); + expect(resp.ok()).toBe(true); + } + + test('a pending session: the page asks for stats after the first rows, merges the reply and shows them', async ({ page, request }) => { + const session = `ws-sched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const order: string[] = []; + const requests: any[] = []; + let realStats: unknown; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + server.onMessage((message) => { + if (typeof message !== 'string') { + ws.send(message); + return; + } + const msg = JSON.parse(message); + if (msg.type === 'infinite_resp') order.push('infinite_resp'); + if (msg.type === 'initial_state' && realStats === undefined) { + // The first frame as a deferring server sends it: no stats yet. + realStats = msg.df_data_dict.all_stats; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } }; + ws.send(JSON.stringify(msg)); + return; + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + order.push('stats_request'); + requests.push(msg); + ws.send(JSON.stringify({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: msg.scope, tier: 'full', + final: true, payload: realStats, elapsed_ms: 1, + })); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // The stats arrive: the status bar says so and the pinned dtype row has its values. + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); + + // One whole-run request, for the gen on the first frame, after rows had arrived. + expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw' }]); + expect(order.indexOf('infinite_resp')).toBeGreaterThanOrEqual(0); + expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); + }); + + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { + const session = `ws-nosched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const sentTypes: string[] = []; + page.on('websocket', (socket) => { + socket.on('framesent', (frame) => { + if (typeof frame.payload === 'string') sentTypes.push(JSON.parse(frame.payload).type); + }); + }); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.locator('.status-bar')).toBeVisible(); + // Longer than the scheduler would wait for rows that never come. + await page.waitForTimeout(3000); + + expect(sentTypes).toContain('infinite_request'); + expect(sentTypes).not.toContain('stats_request'); + await expect(page.getByTestId('stats-status')).toHaveCount(0); + }); + test('WebSocket receives data for scrolled rows', async ({ page, request }) => { // Create a larger dataset (100 rows) to force infinite scrolling const rows = []; diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts new file mode 100644 index 000000000..4470a9324 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -0,0 +1,96 @@ +/** + * Playwright test for the client states of the stats wire (rows-first c4). + * + * The StatsSchedulerStates story flips df_meta.stats.status. Each status has + * its own text in the status bar's stats column, the pinned area follows it as + * in c0a, and switching between them moves nothing: the status bar and the + * grid keep their boxes. In "not_computed" the status bar offers a Compute + * summary stats button, which sends stats_request {force: true}. + */ +import { test, expect, Page } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--primary&globals=&args="; + +const boxes = async (page: Page) => { + const round = (b: { x: number; y: number; width: number; height: number } | null) => + b && { x: Math.round(b.x), y: Math.round(b.y), width: Math.round(b.width), height: Math.round(b.height) }; + return { + statusBar: round(await page.locator(".status-bar").boundingBox()), + grid: round(await page.locator(".df-viewer").boundingBox()), + widget: round(await page.locator(".buckaroo-widget").boundingBox()), + }; +}; + +test("each stats status shows its own text in the status bar and nothing moves between them", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const statsCell = page.getByTestId("stats-status"); + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + + // pending: the loading text, and a placeholder for each pinned key. + await expect(statsCell).toHaveAttribute("data-stats-status", "pending"); + await expect(statsCell).toContainText("Computing summary stats"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + const reference = await boxes(page); + expect(reference.statusBar).not.toBeNull(); + expect(reference.grid).not.toBeNull(); + + // not_computed: the control, and no pinned rows. + await page.getByTestId("status-not_computed").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + expect(await boxes(page)).toEqual(reference); + + // error: the reason, and no pinned rows. + await page.getByTestId("status-error").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "error"); + await expect(statsCell).toContainText("Stats error: stats_failed"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + expect(await boxes(page)).toEqual(reference); + + // complete: the values are in the pinned area where the placeholders were. + await page.getByTestId("status-complete").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "complete"); + await expect(statsCell).toContainText("Summary stats ready"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + expect(await boxes(page)).toEqual(reference); + + expect(pageErrors).toEqual([]); +}); + +test("the Compute summary stats control sends stats_request with force", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + await page.getByTestId("status-not_computed").click(); + await page.getByRole("button", { name: "Compute summary stats" }).click(); + + const sent = page.getByTestId("sent-log"); + await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", force: true }, + ]); +}); + +test("no control is offered while the stats are pending, in error or complete", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + const control = page.getByRole("button", { name: "Compute summary stats" }); + for (const status of ["pending", "error", "complete"]) { + await page.getByTestId(`status-${status}`).click(); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", status); + await expect(control).toHaveCount(0); + } +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index 51a14cb41..e74a13b95 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -759,6 +759,21 @@ describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", expect(lastPinnedRows()).toEqual(stats); }); + it("error: omits the valueless keys, since no value is coming", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("error: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + it("no df_meta.stats (an older server) behaves as complete: valueless keys stay undefined rows", () => { const stats = [{ index: "dtype", a: "int64" }]; render(); @@ -815,6 +830,23 @@ describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", }); }); + describe("compute summary stats control (rows-first c4)", () => { + it("forwards on_compute_stats to the status bar", () => { + const onComputeStats = jest.fn(); + render( + , + ); + expect(mockStatusBarProps.onComputeStats).toBe(onComputeStats); + }); + + it("passes nothing on when the host gave no callback, so the status bar shows no control", () => { + render(); + expect(mockStatusBarProps.onComputeStats).toBeUndefined(); + }); + }); + describe("in-flight indicator", () => { const inFlightAttr = () => document.querySelector('[data-testid="status-bar-stub"]')!.getAttribute("data-inflight"); const dispatchSearch = () => diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index 07e3a7b50..ecb682153 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -148,6 +148,9 @@ export function BuckarooInfiniteWidget({ df_meta: DFMeta; df_data_dict: Record; df_display_args: Record; + /** Sends a forced stats_request. Server entry points pass it; the Jupyter + * widget does not, and then the status bar offers no control. */ + on_compute_stats?: () => void; operations: Operation[]; on_operations: (ops: Operation[]) => void; operation_results: OperationResult; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts index 944ceddbc..a32f4a7db 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts @@ -200,6 +200,18 @@ describe("testing utility functions in gridUtils ", () => { ]); }); + it("omits a required key with no value when the stats status is error, and keeps one that has a value", () => { + // An error is final for the state on screen, so no value is coming for + // the keys that are missing (rows-first c4). + const data: DFData = [{ index: "row1", value: 1 }]; + const pinnedConfig: PinnedRowConfig[] = [ + { primary_key_val: "row1", displayer_args: { displayer: "obj" } }, + { primary_key_val: "missing", displayer_args: { displayer: "obj" } } + ]; + expect(extractPinnedRows(data, pinnedConfig, "error")).toStrictEqual([{ index: "row1", value: 1 }]); + expect(extractPinnedRows([], pinnedConfig, "error")).toStrictEqual([]); + }); + it("includes an optional `?`-prefixed pinned row when the unprefixed key exists in data", () => { const data: DFData = [ { index: "histogram_bins", value: 1 }, diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx new file mode 100644 index 000000000..d4fe5c99e --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -0,0 +1,150 @@ +/** + * StatusBar — summary stats status (rows-first c4). + * + * A session that reports df_meta.stats gets one extra, fixed-width column in + * the status bar showing where the stats stand: loading ("pending"), not + * computed (with a control that asks for them), error (with the reason) or + * ready. A session that does not report df_meta.stats gets the status bar it + * always had. + * + * AG Grid is stubbed to capture the props the status bar hands it; the cell + * renderer is rendered on its own. + */ +import "@testing-library/jest-dom"; +import { render, screen, fireEvent } from "@testing-library/react"; + +// The props the status bar gave AG Grid on its last render. +const mockGrid: { props: any } = { props: null }; +jest.mock("ag-grid-react", () => ({ + AgGridReact: (props: any) => { + // React also calls this stub once with no props; keep the last real ones. + if (props) mockGrid.props = props; + return
; + }, +})); +jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); + +import { StatusBar, StatsStatusCell } from "./StatusBar"; +import { BuckarooOptions, BuckarooState, DFMeta, DFMetaStats } from "./WidgetTypes"; + +const baseMeta: DFMeta = { total_rows: 378, columns: 7, filtered_rows: 297, rows_shown: 297 }; +const options: BuckarooOptions = { + sampled: [], + cleaning_method: ["", "clean1"], + post_processing: ["", "post1"], + df_display: ["main", "summary"], + show_commands: ["0", "1"], +}; +const bState: BuckarooState = { + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, +}; + +const renderBar = (dfMeta: DFMeta, onComputeStats?: () => void) => + render( + {}} + buckarooOptions={options} + onComputeStats={onComputeStats} + />, + ); + +const fields = (): string[] => mockGrid.props.columnDefs.map((c: any) => c.field); + +describe("StatusBar stats column", () => { + beforeEach(() => { + mockGrid.props = null; + }); + + it("adds no column and no row field when df_meta has no stats (every session today)", () => { + renderBar(baseMeta); + expect(fields()).not.toContain("stats"); + expect(mockGrid.props.rowData[0]).not.toHaveProperty("stats"); + }); + + it("adds a fixed-width stats column after the summary-view selector when df_meta.stats is present", () => { + const stats: DFMetaStats = { status: "pending", tier: "schema", gen: 1 }; + renderBar({ ...baseMeta, stats }); + const names = fields(); + expect(names.indexOf("stats")).toBe(names.indexOf("df_display") + 1); + + const column = mockGrid.props.columnDefs[names.indexOf("stats")]; + expect(column.cellRenderer).toBe(StatsStatusCell); + // A fixed width, so the status changing never moves the other columns. + expect(typeof column.width).toBe("number"); + expect(column.flex).toBeUndefined(); + expect(mockGrid.props.rowData[0].stats).toBe(stats); + }); + + it("keeps the column for every status", () => { + for (const status of ["pending", "not_computed", "error", "complete"] as const) { + const { unmount } = renderBar({ ...baseMeta, stats: { status, gen: 1 } }); + expect(fields()).toContain("stats"); + unmount(); + } + }); + + it("hands the compute callback to the cell renderer through the grid context", () => { + const onComputeStats = jest.fn(); + renderBar({ ...baseMeta, stats: { status: "not_computed", gen: 1 } }, onComputeStats); + expect(mockGrid.props.context.onComputeStats).toBe(onComputeStats); + }); +}); + +describe("StatsStatusCell", () => { + const cell = (value: DFMetaStats | undefined, onComputeStats?: () => void) => + render(); + + it("pending: says the stats are being computed", () => { + cell({ status: "pending", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "pending"); + expect(root).toHaveTextContent("Computing summary stats"); + expect(root).toHaveAttribute("role", "status"); + }); + + it("not_computed: offers a Compute summary stats button that calls the handler", () => { + const onComputeStats = jest.fn(); + cell({ status: "not_computed", gen: 1 }, onComputeStats); + expect(screen.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); + + fireEvent.click(screen.getByRole("button", { name: "Compute summary stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("not_computed: with no handler there is no button, only the label", () => { + cell({ status: "not_computed", gen: 1 }); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); + }); + + it("error: shows the reason", () => { + cell({ status: "error", gen: 1, reason: "stats_failed" }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "error"); + expect(root).toHaveTextContent("Stats error: stats_failed"); + }); + + it("error: without a reason still says so", () => { + cell({ status: "error", gen: 1 }); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Stats error"); + }); + + it("complete: says the stats are ready", () => { + cell({ status: "complete", tier: "full", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "complete"); + expect(root).toHaveTextContent("Summary stats ready"); + }); + + it("renders nothing without stats", () => { + const { container } = cell(undefined); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.tsx b/packages/buckaroo-js-core/src/components/StatusBar.tsx index 1a08dedd8..a7ae9c856 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.tsx @@ -4,7 +4,7 @@ import * as _ from "lodash-es"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { ColDef, GridApi, GridOptions } from "ag-grid-community"; import { basicIntFormatter } from "./DFViewerParts/Displayer"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, DFMetaStats } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; import { BuckarooState, BKeys } from "./WidgetTypes"; import { CustomCellEditorProps } from 'ag-grid-react'; @@ -307,6 +307,11 @@ export const SearchEditor = memo(({ value, onValueChange, stopEditing }: Custom ); }); +/** Renders df_meta.stats in the status bar (rows-first c4). Stub. */ +export const StatsStatusCell = function (_params: { value?: DFMetaStats; context?: { onComputeStats?: () => void } }) { + return null; +}; + export function StatusBar({ dfMeta, buckarooState, @@ -334,6 +339,8 @@ export function StatusBar({ * Python's ComponentConfig TypedDict; cell renderers read them via * params.context.componentConfig. */ componentConfig?: Record; + /** Sends a forced stats_request; shown as a control while the stats are not computed. */ + onComputeStats?: () => void; }) { if (false) { console.log("heightOverride", heightOverride); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx new file mode 100644 index 000000000..9bb495202 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx @@ -0,0 +1,89 @@ +/** + * BuckarooView — the Compute summary stats control (rows-first c4). + * + * While df_meta.stats says the stats are not computed, the status bar offers a + * control that asks the server for them. BuckarooView hands the widget the + * callback, and the callback sends `stats_request {force: true}` through + * whatever IModel the host gave it. + */ +import { render, cleanup, act } from "@testing-library/react"; +import { BuckarooView } from "./BuckarooView"; +import type { IModel } from "./IModel"; + +const mockWidgetProps: any[] = []; +jest.mock("../components/BuckarooWidgetInfinite", () => ({ + BuckarooInfiniteWidget: (props: any) => { + mockWidgetProps.push(props); + return
; + }, + DFViewerInfiniteDS: () =>
, + getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), +})); + +function makeFakeModel(state: Record): { model: IModel; sent: any[] } { + const sent: any[] = []; + const model: IModel = { + send: (msg) => { sent.push(msg); }, + get: (k) => state[k], + set: (k, v) => { state[k] = v; }, + save_changes: () => {}, + on: () => {}, + off: () => {}, + }; + return { model, sent }; +} + +const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, +}; +const metaWith = (stats?: Record) => ({ + total_rows: 3, columns: 1, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const mountBuckaroo = async (state: Record) => { + const { model, sent } = makeFakeModel(state); + await act(async () => { + render(); + }); + return { model, sent, props: () => mockWidgetProps[mockWidgetProps.length - 1] }; +}; + +afterEach(() => { + mockWidgetProps.length = 0; + cleanup(); +}); + +describe("BuckarooView on_compute_stats (rows-first c4)", () => { + it("hands the widget a callback that sends a forced stats_request for the gen on screen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + + props().on_compute_stats(); + expect(sent).toEqual([{ type: "stats_request", stats_gen: 9, scope: "raw", force: true }]); + }); + + it("sends nothing when the model's df_meta carries no stats.gen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith(), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + props().on_compute_stats(); + expect(sent).toEqual([]); + }); + + it("sends no request on its own: asking is the scheduler's job, and a session that is not pending is left alone", async () => { + const { sent } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(sent).toEqual([]); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 94c2ccef1..37f0992c8 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -1,188 +1,525 @@ -import { StateOrchestrator, WsLike } from "./StateOrchestrator"; +/** + * StateOrchestrator — the client scheduler for the stats wire (rows-first c4). + * + * A session the server defers stats for sends a stats-free first frame with + * df_meta.stats.status "pending". The scheduler asks for the stats once the + * first rows have arrived (`stats_request`), asks again for each reply that + * leaves the stats pending, and stands down when the state changes. The merge + * of the replies is StatsChannel's; the scheduler only watches the model. + * + * The scheduler tests drive a fake model the way WebSocketModel drives a real + * one (set a key, emit its change event). The last block builds a real + * WebSocketModel from a fake socket. + */ +import { StateOrchestrator, StatsModel, requestStats, touchesDataflow } from "./StateOrchestrator"; +import { WebSocketModel } from "./WebSocketModel"; -class FakeWs implements WsLike { - sent: string[] = []; - send(message: string): void { - this.sent.push(message); +class FakeModel implements StatsModel { + sent: any[] = []; + private handlers = new Map>(); + + constructor(public state: Record) {} + + get(key: string) { + return this.state[key]; + } + set(key: string, value: any) { + this.state[key] = value; + this.emit(`change:${key}`, value); + } + send(msg: any) { + this.sent.push(msg); + } + on(event: string, handler: (...args: any[]) => void) { + if (!this.handlers.has(event)) this.handlers.set(event, new Set()); + this.handlers.get(event)!.add(handler); + } + off(event: string, handler: (...args: any[]) => void) { + this.handlers.get(event)?.delete(handler); + } + emit(event: string, ...args: any[]) { + for (const h of Array.from(this.handlers.get(event) ?? [])) h(...args); } - last(): Record { - return JSON.parse(this.sent[this.sent.length - 1]); + listenerCount() { + return Array.from(this.handlers.values()).reduce((n, set) => n + set.size, 0); } - byType(type: string): Record[] { - return this.sent - .map((s) => JSON.parse(s)) - .filter((m) => m.type === type); + /** A full frame: each key lands and fires its change event in turn, as + * WebSocketModel's initial_state branch does. */ + frame(msg: Record) { + for (const [key, value] of Object.entries(msg)) this.set(key, value); } } -describe("StateOrchestrator", () => { - let ws: FakeWs; - let orch: StateOrchestrator; +const meta = (stats?: Record) => ({ + total_rows: 3, columns: 2, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); +const complete = (gen: number) => ({ status: "complete", tier: "full", gen }); +const dict = (rows: any[] = []) => ({ all_stats: rows }); +const statRow = (stat: string) => ({ index: stat, level_0: stat, a: 1 }); +// Typed loosely: the server's buckaroo_state also carries keys (search_string) +// that BuckarooState does not declare. +const bState = (over: Record = {}): any => ({ + sampled: false, cleaning_method: false, quick_command_args: {}, post_processing: false, + df_display: "main", show_commands: false, ...over, +}); + +const request = (gen: number, extra: Record = {}) => ({ + type: "stats_request", stats_gen: gen, scope: "raw", ...extra, +}); + +const makeModel = (stats?: Record) => + new FakeModel({ df_meta: meta(stats), df_data_dict: dict(), buckaroo_state: bState() }); + +/** The first rows reached the client: WebSocketModel emits msg:custom once it + * has paired an infinite_resp with its parquet frame. */ +const rowsArrived = (model: FakeModel) => + model.emit("msg:custom", { type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }, []); + +const start = (model: FakeModel, opts: Record = {}) => { + const orchestrator = new StateOrchestrator({ model, ...opts }); + orchestrator.start(); + return orchestrator; +}; + +// With the defaults, a state change waits 2 x 250 ms before it asks again. +const DEBOUNCE = 500; +const FIRST_PAINT_TIMEOUT = 1500; + +// Runs due timers and every promise continuation they leave behind. +const tick = (ms = 0) => jest.advanceTimersByTimeAsync(ms); + +beforeEach(() => { + jest.useFakeTimers(); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +describe("when nothing is pending", () => { + it("requests nothing for a session whose df_meta has no stats (every session today)", async () => { + const model = makeModel(); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["complete", "not_computed", "error"])("requests nothing when the status is %s", async (status) => { + const model = makeModel({ status, tier: "schema", gen: 3 }); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); +}); + +describe("the first request", () => { + it("goes out after the first infinite_resp, not before", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(100); + expect(model.sent).toEqual([]); - beforeEach(() => { - jest.useFakeTimers(); - ws = new FakeWs(); - orch = new StateOrchestrator({ ws, minDebounceMs: 10, maxDebounceMs: 5000 }); + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); }); - afterEach(() => { - orch.dispose(); - jest.useRealTimers(); + it("goes out anyway when no rows come (an empty frame, the summary view)", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(FIRST_PAINT_TIMEOUT - 1); + expect(model.sent).toEqual([]); + + await tick(1); + expect(model.sent).toEqual([request(3)]); }); - it("starts with token 0", () => { - expect(orch.currentToken).toBe(0); + it("carries no force flag", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + await tick(); + expect(model.sent[0]).not.toHaveProperty("force"); }); - it("onStateChange bumps token and ships state_change", () => { - orch.onStateChange({ search_string: "x" }); - expect(orch.currentToken).toBe(1); - const msg = ws.last(); - expect(msg.type).toBe("state_change"); - expect(msg.state_token).toBe(1); - expect((msg.new_state as Record).search_string).toBe("x"); + it("is sent once, however many row responses follow", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + rowsArrived(model); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); +}); + +describe("one request per reply", () => { + const afterFirstRequest = async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; - it("schedules compute_stat_group after the debounce", () => { - orch.onStateChange({ search_string: "PIZZA" }); - // Only the immediate state_change has been sent so far. - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("asks again for each reply that leaves the stats pending, and stops at the final one", async () => { + const { model } = await afterFirstRequest(); + expect(model.sent).toEqual([request(3)]); - // Default baseline is 500ms × 2× = 1000ms; clamped to maxDebounceMs=5000. - jest.advanceTimersByTime(999); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + // No reply yet: nothing more goes out, however long the server takes. + await tick(10_000); + expect(model.sent).toEqual([request(3)]); - jest.advanceTimersByTime(1); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].scope).toBe("filt"); - expect(reqs[0].group).toBe("aggregate"); - expect(reqs[0].state_token).toBe(1); + // A partial reply is a new df_data_dict under the same df_meta, which is + // what StatsChannel does for a stats_update that is not final. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + + model.set("df_data_dict", dict([statRow("mean"), statRow("std")])); + await tick(); + expect(model.sent).toHaveLength(3); + + // The final reply also sets df_meta.stats to complete. + model.set("df_data_dict", dict([statRow("mean"), statRow("std"), statRow("max")])); + model.set("df_meta", meta(complete(3))); + await tick(10_000); + expect(model.sent).toHaveLength(3); }); - it("back-to-back state_changes cancel the previous debounce timer", () => { - orch.onStateChange({ search_string: "P" }); - jest.advanceTimersByTime(500); - orch.onStateChange({ search_string: "PI" }); - // The first timer would have fired at t=1000ms; the second - // resets it so at t=999ms from the SECOND call (= 1499 overall) - // no compute_stat_group has fired yet. - jest.advanceTimersByTime(998); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("reads a reply the same way whichever of the two events comes first", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta(complete(3))); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); - // The second timer fires. - jest.advanceTimersByTime(2); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(2); // second state_change's token + it.each([ + ["df_meta then df_data_dict", ["df_meta", "df_data_dict"]], + ["df_data_dict then df_meta", ["df_data_dict", "df_meta"]], + ])("does not take a full frame for the same state as a reply (%s)", async (_label, order) => { + // A search term that changes only the highlight comes back as a full + // initial_state for the same stats_gen, with a new df_meta and a new dict. + const { model } = await afterFirstRequest(); + const full: Record = { df_meta: meta(pending(3)), df_data_dict: dict([statRow("dtype")]) }; + model.frame(Object.fromEntries(order.map((k) => [k, full[k]]))); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); - it("rapid typing produces just one aggregate request, with the latest token", () => { - // Simulate 5 keystrokes at 100ms intervals — typical typing cadence. - for (let i = 0; i < 5; i++) { - orch.onStateChange({ search_string: "P".repeat(i + 1) }); - jest.advanceTimersByTime(100); - } - // Default debounce = 1000ms after the last keystroke. No aggregate yet. - expect(ws.byType("compute_stat_group")).toHaveLength(0); - - jest.advanceTimersByTime(1000); - const reqs = ws.byType("compute_stat_group"); - // Exactly one aggregate request, with the 5th (final) token. - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(5); - }); - - it("onStatGroupResult with matching token updates the baseline", () => { - orch.onStateChange({ search_string: "x" }); - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, - scope: "filt", - group: "aggregate", - elapsed_ms: 7500, - }); - expect(applied).toBe(true); - - // Next debounce is 2× 7500 = 15000ms, clamped to maxDebounceMs=5000. - expect(orch.computeDebounce("filt")).toBe(5000); - }); - - it("onStatGroupResult with stale token is silently dropped", () => { - orch.onStateChange({ search_string: "x" }); - orch.onStateChange({ search_string: "xy" }); - // Token is now 2. - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, // stale - scope: "filt", - group: "aggregate", - elapsed_ms: 9999, - }); - expect(applied).toBe(false); - // Baseline unchanged — debounce stays at the default. - expect(orch.computeDebounce("filt")).toBe(1000); + it("stops when the server reports an error", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); - it("computeDebounce respects minDebounceMs floor", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 500, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 10, // 2× 10 = 20, well under floor - }); - expect(o.computeDebounce("filt")).toBe(500); + it("stops when the server says the stats are not computed", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "not_computed", tier: "schema", gen: 3 })); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); +}); - it("computeDebounce respects maxDebounceMs ceiling", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 200, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 6000, // 2× = 12000, hits ceiling - }); - expect(o.computeDebounce("filt")).toBe(3000); +describe("a state change", () => { + const pendingAt = async (gen: number) => { + const model = makeModel(pending(gen)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; + + // The server answers a dataflow change with a frame for the next stats_gen. + const nextFrame = (model: FakeModel, gen: number) => { + model.frame({ df_meta: meta(pending(gen)), df_data_dict: dict() }); + rowsArrived(model); + }; + + it("stops the requests for the old state, then asks once for the new one", async () => { + const { model } = await pendingAt(3); + expect(model.sent).toEqual([request(3)]); + + // The user changes a dataflow field while the request is out. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(10); + // The reply to the old request lands. It is not followed by another. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10); + nextFrame(model, 4); + + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("reads a frame whose dict comes before its df_meta as the next state, not as a reply", async () => { + const { model } = await pendingAt(3); + // The frame for the next gen carries its dict before its df_meta. + model.frame({ df_data_dict: dict(), df_meta: meta(pending(4)) }); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it.each([ + ["post_processing", { post_processing: "log_scale" }], + ["cleaning_method", { cleaning_method: "aggressive" }], + ["quick_command_args", { quick_command_args: { search: ["x"] } }], + ])("a %s change cancels a request that is waiting out its delay", async (_field, change) => { + const { model } = await pendingAt(3); + nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(300); + // 600 ms in: the request that was due at 500 ms never went out. + expect(model.sent).toEqual([request(3)]); + + nextFrame(model, 5); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(5)]); }); - it("initialAggregateMs seeds the baseline before any observed compute", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - initialAggregateMs: { filt: 250 }, - minDebounceMs: 10, - maxDebounceMs: 5000, - multiplier: 2, + it.each([ + ["search_string (the #1015 path)", { search_string: "x" }], + ["df_display", { df_display: "summary" }], + ["show_commands", { show_commands: "1" }], + ["sampled", { sampled: "sample" }], + ])("a %s-only change is skipped", async (_label, change) => { + const { model } = await pendingAt(3); + nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(200); + // The request is still due at 500 ms, as if nothing had changed. + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it("a search_string-only change does not interrupt a chain of replies", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ search_string: "x" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("a frame that repeats the same dataflow state is not a state change", async () => { + // Every full frame carries buckaroo_state back as the client sent it. + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState()); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("asks again for the same state when the server never answers the change with a frame", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(FIRST_PAINT_TIMEOUT + DEBOUNCE); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("waits 2 x the last request's time, within the limits, before the next state's request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + expect(orchestrator.computeDebounce()).toBe(500); + + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); // the reply, 400 ms after the request + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + nextFrame(model, 4); + await tick(799); + expect(model.sent).toHaveLength(2); + await tick(1); + expect(model.sent).toHaveLength(3); + expect(model.sent[2]).toEqual(request(4)); + }); + + it.each([ + [10, 100], + [400, 800], + [5000, 2000], + ])("computeDebounce after a %i ms request is %i ms (floor 100, ceiling 2000)", async (elapsed, expected) => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + rowsArrived(model); + await tick(elapsed); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(orchestrator.computeDebounce()).toBe(expected); + }); +}); + +describe("requestStats", () => { + it("sends a stats_request for the gen the model shows", () => { + const model = makeModel(pending(7)); + expect(requestStats(model)).toBe(true); + expect(model.sent).toEqual([request(7)]); + }); + + it("force sends force: true, which the Compute summary stats control uses", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + expect(requestStats(model, { force: true })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true })]); + }); + + it("sends nothing when df_meta carries no stats.gen", () => { + const model = makeModel(); + expect(requestStats(model)).toBe(false); + expect(requestStats(model, { force: true })).toBe(false); + expect(model.sent).toEqual([]); + }); +}); + +describe("touchesDataflow", () => { + it("is true for each field the server reruns the dataflow for", () => { + expect(touchesDataflow(bState(), bState({ post_processing: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ cleaning_method: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ quick_command_args: { search: ["x"] } }))).toBe(true); + }); + + it("is false for the others, and for an equal quick_command_args that is a new object", () => { + expect(touchesDataflow(bState(), bState({ search_string: "x" }))).toBe(false); + expect(touchesDataflow(bState(), bState({ df_display: "summary" }))).toBe(false); + expect(touchesDataflow(bState({ quick_command_args: { search: ["x"] } }), bState({ quick_command_args: { search: ["x"] } }))).toBe(false); + }); + + it("is false when there is no earlier state to compare with", () => { + expect(touchesDataflow(undefined, bState({ post_processing: "x" }))).toBe(false); + }); +}); + +describe("start and stop", () => { + it("stop removes every listener and cancels the pending request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + expect(model.listenerCount()).toBeGreaterThan(0); + + rowsArrived(model); + orchestrator.stop(); + expect(model.listenerCount()).toBe(0); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("start is idempotent: a second call adds no listeners", () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + const listeners = model.listenerCount(); + orchestrator.start(); + expect(model.listenerCount()).toBe(listeners); + }); + + it("can start again after a stop", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + orchestrator.stop(); + orchestrator.start(); + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); + }); +}); + +// The scheduler is started by WebSocketModel, so a session reached through +// BuckarooServerView or the standalone page gets it with no wiring of its own. +describe("wired into WebSocketModel", () => { + class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } + deliverBinary() { + this.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + } + } + + const update = (gen: number, stat: string, final: boolean) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final, + payload: { format: "json", layout: "wide", data: [{ index: stat, level_0: stat, a: 1 }] }, + elapsed_ms: 5, + }); + + const makeSocketModel = (stats?: Record) => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: meta(stats), + df_data_dict: { all_stats: [{ index: "dtype", level_0: "dtype", a: "int64" }] }, + buckaroo_state: bState(), }); - expect(o.computeDebounce("filt")).toBe(500); // 2× 250 - // Unrelated scope still uses fallback. - expect(o.computeDebounce("clean")).toBe(1000); // 2× 500 (default) + return { ws, model }; + }; + const rowsFromServer = (ws: FakeSocket) => { + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }); + ws.deliverBinary(); + }; + + it("requests the stats after the first rows, once per reply, and stops at the final one", async () => { + const { ws, model } = makeSocketModel(pending(3)); + await tick(100); + expect(ws.sent).toEqual([]); + + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver(update(3, "mean", false)); + await tick(); + expect(ws.sent).toEqual([request(3), request(3)]); + + ws.deliver(update(3, "std", true)); + await tick(10_000); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_meta").stats.status).toBe("complete"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std"]); }); - it("dispose cancels all pending aggregate timers", () => { - orch.onStateChange({ search_string: "x" }); - orch.dispose(); - jest.advanceTimersByTime(10_000); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("moves on to the next gen when a state change frame arrives", async () => { + const { ws } = makeSocketModel(pending(3)); + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + rowsFromServer(ws); + await tick(DEBOUNCE); + expect(ws.sent).toEqual([request(3), request(4)]); }); - it("scopesForAggregate parameter overrides default ['filt']", () => { - orch.onStateChange({ search_string: "x" }, { scopesForAggregate: ["filt", "clean"] }); - jest.advanceTimersByTime(10_000); - const reqs = ws.byType("compute_stat_group"); - const scopes = reqs.map((r) => r.scope).sort(); - expect(scopes).toEqual(["clean", "filt"]); + it("sends nothing for a model whose df_meta has no stats", async () => { + const { ws } = makeSocketModel(); + rowsFromServer(ws); + await tick(10_000); + expect(ws.sent).toEqual([]); }); }); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts index e096ac606..d9f1e5446 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts @@ -1,153 +1,44 @@ /** - * Client-side scheduler for the JS-driven progressive-stats protocol. + * Client-side scheduler for the stats wire (rows-first c4). * - * On a state_change, ships the cheap (scalar) request immediately so - * df_meta + scalar pinned rows update fast. Schedules the expensive - * (aggregate) compute per scope behind an adaptive debounce — by - * default 2× the last observed aggregate compute time for that scope, - * clamped to ``[minDebounceMs, maxDebounceMs]``. - * - * Token-based cancellation: every state_change bumps an internal - * token; results carrying a stale token are dropped on arrival. - * - * See plans/js-driven-stat-debounce.md for the full protocol design. - * - * This module is transport-agnostic — pass any object with a - * ``send(s: string)`` method. Existing buckaroo WS connections fit. + * Stub: the API the tests drive, with no behaviour yet. */ +import { BuckarooState } from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +export type StatsModel = Pick; + +export const DATAFLOW_STATE_FIELDS = ["post_processing", "cleaning_method", "quick_command_args"] as const; -export type ScopeName = "raw" | "clean" | "filt"; -export type CostGroup = "scalar" | "aggregate"; +export function touchesDataflow(_prev: BuckarooState | undefined, _next: BuckarooState | undefined): boolean { + return false; +} + +export interface StatsRequestOptions { + force?: boolean; +} -export interface WsLike { - send(message: string): void; +export function requestStats(_model: Pick, _opts?: StatsRequestOptions): boolean { + return false; } export interface OrchestratorOptions { - ws: WsLike; - /** Lower bound on the per-scope debounce. Default 200 ms. */ + model: StatsModel; minDebounceMs?: number; - /** Upper bound on the per-scope debounce. Default 3000 ms. */ maxDebounceMs?: number; - /** Multiplier on the last observed aggregate compute time. Default 2. */ multiplier?: number; - /** Initial aggregate baseline per scope (used until we observe a real one). */ - initialAggregateMs?: Partial>; -} - -export interface StatGroupResult { - type: "stat_group_result"; - state_token: number; - scope: ScopeName; - group: CostGroup; - elapsed_ms: number; - stats?: unknown; + initialRequestMs?: number; + firstPaintTimeoutMs?: number; } export class StateOrchestrator { - private token = 0; - private aggregateTimers = new Map>(); - private lastAggregateMs = new Map(); - private readonly ws: WsLike; - private readonly minDebounceMs: number; - private readonly maxDebounceMs: number; - private readonly multiplier: number; - /** Fallback baseline before any real aggregate compute has been observed. */ - private readonly defaultBaselineMs = 500; + constructor(_opts: OrchestratorOptions) {} - constructor(opts: OrchestratorOptions) { - this.ws = opts.ws; - this.minDebounceMs = opts.minDebounceMs ?? 200; - this.maxDebounceMs = opts.maxDebounceMs ?? 3000; - this.multiplier = opts.multiplier ?? 2; - if (opts.initialAggregateMs) { - for (const [scope, ms] of Object.entries(opts.initialAggregateMs)) { - if (ms != null) this.lastAggregateMs.set(scope as ScopeName, ms); - } - } - } - - /** - * Current state-change token. Tests inspect this; production - * code doesn't usually need it. - */ - get currentToken(): number { - return this.token; - } - - /** - * Compute the debounce delay (ms) for a given scope based on - * the last observed aggregate compute time, clamped to - * ``[minDebounceMs, maxDebounceMs]``. - */ - computeDebounce(scope: ScopeName): number { - const last = this.lastAggregateMs.get(scope) ?? this.defaultBaselineMs; - const raw = last * this.multiplier; - return Math.max(this.minDebounceMs, Math.min(this.maxDebounceMs, raw)); - } + start(): void {} - /** - * Drive a single user-initiated state change. - * - * 1. Bump the state token. - * 2. Cancel any pending aggregate timers from the previous change. - * 3. Ship the ``state_change`` message (server will reply with - * the scalar stats fast). - * 4. For each scope expected to have aggregate work, schedule a - * debounced ``compute_stat_group`` request. Timer fires the - * request only if no further state_change has bumped the - * token meanwhile. - */ - onStateChange( - newState: Record, - opts?: { scopesForAggregate?: ScopeName[] }, - ): void { - const token = ++this.token; - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); - - this.ws.send(JSON.stringify({ - type: "state_change", - state_token: token, - new_state: newState, - })); - - const scopes = opts?.scopesForAggregate ?? ["filt"]; - for (const scope of scopes) { - const delay = this.computeDebounce(scope); - const tid = setTimeout(() => { - // If a newer state_change bumped the token while we - // were waiting, drop the request — sending it would - // produce a stat_group_aborted from the server anyway. - if (token !== this.token) return; - this.ws.send(JSON.stringify({ - type: "compute_stat_group", - state_token: token, - scope, - group: "aggregate", - })); - }, delay); - this.aggregateTimers.set(scope, tid); - } - } - - /** - * Process a ``stat_group_result`` from the server. Stale results - * (mismatched token) are silently ignored. Successful results - * update the per-scope aggregate baseline used by the next - * debounce. - * - * Returns true if the result was applied, false if it was stale. - */ - onStatGroupResult(msg: StatGroupResult): boolean { - if (msg.state_token !== this.token) return false; - this.lastAggregateMs.set(msg.scope, msg.elapsed_ms); - return true; - } + stop(): void {} - /** Cancel all pending aggregate timers. Call on widget unmount. */ - dispose(): void { - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); + computeDebounce(): number { + return 0; } } diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx new file mode 100644 index 000000000..04b3922b7 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -0,0 +1,170 @@ +/** + * Story for the client states of the stats wire (rows-first c4). + * + * df_meta.stats.status says where the summary stats stand, and the status bar + * shows it in a column of its own: + * + * - "pending" "Computing summary stats", pinned keys show placeholders + * - "not_computed" a "Compute summary stats" button, pinned keys omitted + * - "error" "Stats error: ", pinned keys omitted + * - "complete" "Summary stats ready", the values in place + * + * The buttons switch the status. The button in the status bar sends + * `stats_request {force: true}` through a fake model, which logs what it was + * asked to send. Used by stats-scheduler-states.spec.ts. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import { BuckarooInfiniteWidget } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { BuckarooOptions, BuckarooState, DFMeta, StatsStatus } from "../components/WidgetTypes"; +import { CommandConfigT } from "../components/CommandUtils"; +import { Operation } from "../components/OperationUtils"; +import { baseOperationResults } from "../components/DependentTabs"; +import { requestStats } from "../server/StateOrchestrator"; + +const STATUSES: StatsStatus[] = ["pending", "not_computed", "error", "complete"]; +const GEN = 7; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 2, b: "N/A" }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const buckarooOptions: BuckarooOptions = { + sampled: [], + cleaning_method: [], + post_processing: [], + df_display: ["main"], + show_commands: [], +}; +const commandConfig: CommandConfigT = { argspecs: {}, defaultArgs: {} }; + +const StatsSchedulerStatesInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + const [sent, setSent] = useState([]); + const [buckarooState, setBuckarooState] = useState({ + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, + }); + const [operations, setOperations] = useState([]); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { + status, + tier: status === "complete" ? "full" : "schema", + gen: GEN, + ...(status === "error" ? { reason: "stats_failed" } : {}), + }, + }) as DFMeta, + [status], + ); + + // The model the control sends through: it answers get("df_meta") from the + // story's state and logs what it is asked to send. + const model = useMemo( + () => ({ + get: (key: string) => (key === "df_meta" ? df_meta : undefined), + send: (msg: unknown) => setSent((log) => [...log, msg]), + }), + [df_meta], + ); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : ([] as DFData), + empty: [] as DFData, + }), + [status], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + df_meta.stats.status = {status} +
+
+ requestStats(model, { force: true })} + /> +
+
+        {JSON.stringify(sent)}
+      
+
+ ); +}; + +const meta = { + title: "Buckaroo/StatsSchedulerStates", + component: StatsSchedulerStatesInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; diff --git a/scripts/test_playwright_storybook.sh b/scripts/test_playwright_storybook.sh index 6d835ab2a..9445be842 100755 --- a/scripts/test_playwright_storybook.sh +++ b/scripts/test_playwright_storybook.sh @@ -96,6 +96,7 @@ STORYBOOK_TESTS=( "pw-tests/outside-params.spec.ts" "pw-tests/fitcontent-height.spec.ts" "pw-tests/stats-pending-pinned-rows.spec.ts" + "pw-tests/stats-scheduler-states.spec.ts" # "pw-tests/example.spec.ts" # Has pre-existing failures, excluded for now ) From d6afbf9835c2c9801591a4e9ebe15204bc7c3e6b Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:34:54 -0400 Subject: [PATCH 8/9] test(client): failing tests for the first state change, the stats status layout and the control wiring (rows-first c4) Cases found untested or mis-specified after the first push, written against the same stubs: a model that starts complete waits out the delay for its first state change, as does a change made after the earlier stats completed; a request's time is measured once; the Compute summary stats button calls its handler with no arguments; the standalone page offers that button for a session whose stats are not computed and asks for nothing until it is clicked. The layout spec now says what holds across statuses (nothing above or beside the grid moves, and the grid's height follows the pinned area), and the story loads the widget's stylesheet as the real page does. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 47 +++++++++++++ .../pw-tests/stats-scheduler-states.spec.ts | 27 ++++++-- .../src/components/StatusBar.stats.test.tsx | 2 + .../src/server/StateOrchestrator.test.ts | 69 ++++++++++++++++--- .../stories/StatsSchedulerStates.stories.tsx | 1 + 5 files changed, 131 insertions(+), 15 deletions(-) diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index bf98d7d19..af04c3f6e 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -569,6 +569,53 @@ test.describe('WebSocket data flow', () => { expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); }); + test('a session whose stats are not computed: the page asks for nothing until the Compute summary stats button is clicked', async ({ page, request }) => { + const session = `ws-notcomputed-${Date.now()}`; + await loadBuckarooSession(request, session); + + const requests: any[] = []; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + let firstFrame = true; + server.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'initial_state' && firstFrame) { + firstFrame = false; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'not_computed', tier: 'schema', gen: 4, reason: 'host' } }; + ws.send(JSON.stringify(msg)); + return; + } + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + requests.push(msg); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed'); + // The pinned rows are omitted, not left as placeholders. + await expect(page.locator('.ag-floating-top .ag-row')).toHaveCount(0); + + // Nothing is asked for on its own, however long the page waits. + await page.waitForTimeout(2500); + expect(requests).toEqual([]); + + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + await expect.poll(() => requests).toEqual([{ type: 'stats_request', stats_gen: 4, scope: 'raw', force: true }]); + }); + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { const session = `ws-nosched-${Date.now()}`; await loadBuckarooSession(request, session); diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts index 4470a9324..64788ad1d 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -2,10 +2,13 @@ * Playwright test for the client states of the stats wire (rows-first c4). * * The StatsSchedulerStates story flips df_meta.stats.status. Each status has - * its own text in the status bar's stats column, the pinned area follows it as - * in c0a, and switching between them moves nothing: the status bar and the - * grid keep their boxes. In "not_computed" the status bar offers a Compute - * summary stats button, which sends stats_request {force: true}. + * its own text in the status bar's stats column, and the pinned area follows it + * as in c0a. Switching between statuses moves nothing above or beside the grid: + * the status bar keeps its box, and the grid keeps its position and width. The + * grid's height changes only with the pinned area: placeholders hold the height + * of the values that replace them, and omitted rows take theirs away. In + * "not_computed" the status bar offers a Compute summary stats button, which + * sends stats_request {force: true}. */ import { test, expect, Page } from "@playwright/test"; import { waitForCells } from "./ag-pw-utils"; @@ -44,22 +47,32 @@ test("each stats status shows its own text in the status bar and nothing moves b const reference = await boxes(page); expect(reference.statusBar).not.toBeNull(); expect(reference.grid).not.toBeNull(); + // Nothing above or beside the grid moves, whatever the status. + const staysPut = async () => { + const now = await boxes(page); + expect(now.statusBar).toEqual(reference.statusBar); + expect(now.grid).toMatchObject({ x: reference.grid!.x, y: reference.grid!.y, width: reference.grid!.width }); + return now; + }; // not_computed: the control, and no pinned rows. await page.getByTestId("status-not_computed").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed"); await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); - expect(await boxes(page)).toEqual(reference); + const withoutPinned = await staysPut(); + // The omitted rows take their height with them. + expect(withoutPinned.grid!.height).toBeLessThan(reference.grid!.height); // error: the reason, and no pinned rows. await page.getByTestId("status-error").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "error"); await expect(statsCell).toContainText("Stats error: stats_failed"); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); - expect(await boxes(page)).toEqual(reference); + expect((await staysPut()).grid).toEqual(withoutPinned.grid); - // complete: the values are in the pinned area where the placeholders were. + // complete: the values are in the pinned area where the placeholders were, so + // the grid is as tall as it was while pending. await page.getByTestId("status-complete").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "complete"); await expect(statsCell).toContainText("Summary stats ready"); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx index d4fe5c99e..7db44f0e8 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -115,7 +115,9 @@ describe("StatsStatusCell", () => { expect(screen.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); fireEvent.click(screen.getByRole("button", { name: "Compute summary stats" })); + // Called with no arguments, not with the click event. expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); }); it("not_computed: with no handler there is no button, only the label", () => { diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 37f0992c8..2e7f92eda 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -134,7 +134,7 @@ describe("the first request", () => { await tick(FIRST_PAINT_TIMEOUT - 1); expect(model.sent).toEqual([]); - await tick(1); + await tick(2); expect(model.sent).toEqual([request(3)]); }); @@ -240,9 +240,12 @@ describe("a state change", () => { return { model, orchestrator }; }; - // The server answers a dataflow change with a frame for the next stats_gen. - const nextFrame = (model: FakeModel, gen: number) => { + // The server answers a dataflow change with a frame for the next stats_gen, + // then the grid's refetch brings rows. They are separate messages, so the + // scheduler has read the frame by the time the rows arrive. + const nextFrame = async (model: FakeModel, gen: number) => { model.frame({ df_meta: meta(pending(gen)), df_data_dict: dict() }); + await tick(); rowsArrived(model); }; @@ -256,7 +259,7 @@ describe("a state change", () => { // The reply to the old request lands. It is not followed by another. model.set("df_data_dict", dict([statRow("mean")])); await tick(10); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(DEBOUNCE - 1); expect(model.sent).toEqual([request(3)]); @@ -270,6 +273,7 @@ describe("a state change", () => { const { model } = await pendingAt(3); // The frame for the next gen carries its dict before its df_meta. model.frame({ df_data_dict: dict(), df_meta: meta(pending(4)) }); + await tick(); rowsArrived(model); await tick(10_000); expect(model.sent).toEqual([request(3), request(4)]); @@ -281,7 +285,7 @@ describe("a state change", () => { ["quick_command_args", { quick_command_args: { search: ["x"] } }], ])("a %s change cancels a request that is waiting out its delay", async (_field, change) => { const { model } = await pendingAt(3); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(300); model.set("buckaroo_state", bState(change)); @@ -289,7 +293,7 @@ describe("a state change", () => { // 600 ms in: the request that was due at 500 ms never went out. expect(model.sent).toEqual([request(3)]); - nextFrame(model, 5); + await nextFrame(model, 5); await tick(DEBOUNCE - 1); expect(model.sent).toEqual([request(3)]); await tick(1); @@ -303,7 +307,7 @@ describe("a state change", () => { ["sampled", { sampled: "sample" }], ])("a %s-only change is skipped", async (_label, change) => { const { model } = await pendingAt(3); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(300); model.set("buckaroo_state", bState(change)); @@ -329,6 +333,37 @@ describe("a state change", () => { expect(model.sent).toEqual([request(3), request(3)]); }); + it("waits out the delay for a state change made after the earlier stats completed", async () => { + const { model } = await pendingAt(3); + await tick(250); // a request that took as long as the default assumes, so the delay is DEBOUNCE + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); + + // The first state asked at once; this one is a change to it. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it("waits out the delay for the first state change of a model that started with its stats complete", async () => { + const model = makeModel(complete(3)); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + + model.set("buckaroo_state", bState({ quick_command_args: { search: ["a"] } })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([]); + await tick(1); + expect(model.sent).toEqual([request(4)]); + }); + it("asks again for the same state when the server never answers the change with a frame", async () => { const { model } = await pendingAt(3); model.set("buckaroo_state", bState({ post_processing: "log_scale" })); @@ -348,7 +383,7 @@ describe("a state change", () => { expect(orchestrator.computeDebounce()).toBe(800); model.set("buckaroo_state", bState({ post_processing: "log_scale" })); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(799); expect(model.sent).toHaveLength(2); await tick(1); @@ -356,6 +391,23 @@ describe("a state change", () => { expect(model.sent[2]).toEqual(request(4)); }); + it("measures a request once: later events do not stretch the delay", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 20_000 }); + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + // A full frame for the finished state, long afterwards. + await tick(10_000); + model.frame({ df_meta: meta(complete(3)), df_data_dict: dict() }); + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + }); + it.each([ [10, 100], [400, 800], @@ -511,6 +563,7 @@ describe("wired into WebSocketModel", () => { expect(ws.sent).toEqual([request(3)]); ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); rowsFromServer(ws); await tick(DEBOUNCE); expect(ws.sent).toEqual([request(3), request(4)]); diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx index 04b3922b7..2644d4dc7 100644 --- a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -15,6 +15,7 @@ */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; +import "../style/dcf-npm.css"; import { BuckarooInfiniteWidget } from "../components/BuckarooWidgetInfinite"; import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; From 95dcceecbf164814f3a7e572d145618c2d6bae30 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:55:23 -0400 Subject: [PATCH 9/9] feat(client): scheduler requests stats after first paint, with pending, not computed and error states (rows-first c4) StateOrchestrator becomes the client scheduler for the stats wire. It takes an IModel, so it sends stats_request {stats_gen, scope: "raw"} through model.send and watches change:df_meta, change:df_data_dict, change:buckaroo_state and msg:custom. It asks once the first infinite_resp has arrived (or after 1.5 s without one), asks again for each reply that leaves df_meta.stats pending, and stops when the status changes. A change to post_processing, cleaning_method or quick_command_args waits out a delay of twice the last request's time (200 to 3000 ms) before asking for the next gen; a search_string-only change, or any other field, leaves the schedule alone. Nothing is requested unless df_meta.stats says pending. WebSocketModel starts the scheduler, so BuckarooServerView and the standalone page get it with no wiring of their own. The status bar gets a fixed-width stats column, for sessions that report df_meta.stats, showing loading, a Compute summary stats button that sends stats_request {force: true}, the error reason, or ready. Valueless pinned rows are omitted in the error state, as in not_computed. requestStats and StateOrchestrator are exported for hosts with their own IModel. Co-Authored-By: Claude Sonnet 5.5 --- .../src/components/BuckarooWidgetInfinite.tsx | 2 + .../src/components/DFViewerParts/gridUtils.ts | 8 +- .../src/components/StatusBar.tsx | 64 ++++- packages/buckaroo-js-core/src/index.ts | 5 + .../src/server/BuckarooView.tsx | 8 + .../src/server/StateOrchestrator.ts | 239 +++++++++++++++++- .../src/server/WebSocketModel.ts | 9 +- .../buckaroo-js-core/src/style/dcf-npm.css | 18 ++ packages/js/standalone.tsx | 7 + 9 files changed, 340 insertions(+), 20 deletions(-) diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index ecb682153..e83cb3250 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -134,6 +134,7 @@ export function BuckarooInfiniteWidget({ df_data_dict, df_display_args, df_meta, + on_compute_stats, operations, on_operations, operation_results, @@ -348,6 +349,7 @@ export function BuckarooInfiniteWidget({ themeConfig={cDisp.df_viewer_config?.component_config?.theme} inFlight={inFlight} componentConfig={effectiveDisplayArgs['main']?.df_viewer_config?.component_config as Record | undefined} + onComputeStats={on_compute_stats} /> void } }) { - return null; +/** + * Where the summary stats stand, as the server reports it in df_meta.stats: + * loading while they are pending, a control to ask for them while they are not + * computed, the reason when they failed. The cell always renders one line in a + * fixed-width column, so changing status moves nothing. + */ +export const StatsStatusCell = function (params: { value?: DFMetaStats; context?: { onComputeStats?: () => void } }) { + const stats = params.value; + if (stats === undefined) return null; + const onComputeStats = params.context?.onComputeStats; + const cell = (content: React.ReactNode, extra: React.HTMLAttributes = {}) => ( + + {content} + + ); + switch (stats.status) { + case "pending": + return cell( + <> +
diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 11d10f022..4a56e1810 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -20,6 +20,7 @@ import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { withStatsCapability } from './server/StatsChannel'; +import { StateOrchestrator, requestStats } from './server/StateOrchestrator'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -64,6 +65,8 @@ export default { WebSocketModel, makeLatestDictDecoder, withStatsCapability, + StateOrchestrator, + requestStats, }; // Named exports for direct imports @@ -95,6 +98,8 @@ export { WebSocketModel, makeLatestDictDecoder, withStatsCapability, + StateOrchestrator, + requestStats, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 9c86f5bee..d69eab528 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -10,6 +10,7 @@ import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; +import { requestStats } from "./StateOrchestrator"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -284,6 +285,12 @@ export function BuckarooView({ model.save_changes(); }, [model]); + // The status bar's Compute summary stats button, shown while the server + // reports the stats as not computed. + const onComputeStats = React.useCallback(() => { + requestStats(model, { force: true }); + }, [model]); + // gridUtils honors component_config.layoutType. Stamp it per entry so the // prop wins when provided. autoHeight=undefined → server value left intact. const effectiveDisplayArgs = React.useMemo( @@ -322,6 +329,7 @@ export function BuckarooView({ on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} /> ) : ( ; +/** The fields of buckaroo_state the server reruns the dataflow for, and so + * bumps stats_gen on. Mirrors _DATAFLOW_FIELDS in + * buckaroo/server/websocket_handler.py. */ export const DATAFLOW_STATE_FIELDS = ["post_processing", "cleaning_method", "quick_command_args"] as const; -export function touchesDataflow(_prev: BuckarooState | undefined, _next: BuckarooState | undefined): boolean { - return false; +/** Whether `next` differs from `prev` in a dataflow field. False when there is + * no earlier state to compare with. */ +export function touchesDataflow(prev: BuckarooState | undefined, next: BuckarooState | undefined): boolean { + if (prev === undefined || next === undefined) return false; + return DATAFLOW_STATE_FIELDS.some((field) => JSON.stringify(prev[field]) !== JSON.stringify(next[field])); } export interface StatsRequestOptions { + /** Ask for stats the server did not plan to compute. The "Compute summary + * stats" control sends this. */ force?: boolean; } -export function requestStats(_model: Pick, _opts?: StatsRequestOptions): boolean { - return false; +/** + * Send a `stats_request` for the stats_gen of the state the model shows. + * Returns false, and sends nothing, when its df_meta carries no stats.gen. + */ +export function requestStats(model: Pick, opts: StatsRequestOptions = {}): boolean { + const gen = (model.get("df_meta") as DFMeta | undefined)?.stats?.gen; + if (typeof gen !== "number") return false; + model.send({ type: "stats_request", stats_gen: gen, scope: "raw", ...(opts.force ? { force: true } : {}) }); + return true; } export interface OrchestratorOptions { model: StatsModel; + /** Lower bound on the delay before asking for a new state's stats. Default 200 ms. */ minDebounceMs?: number; + /** Upper bound on that delay. Default 3000 ms. */ maxDebounceMs?: number; + /** Multiplier on the last observed request time. Default 2. */ multiplier?: number; + /** Request time assumed until one has been observed. Default 250 ms. */ initialRequestMs?: number; + /** How long to wait for the first rows before asking anyway (an empty + * frame, the summary view, and a grid that never fetches send none). + * Default 1500 ms. */ firstPaintTimeoutMs?: number; } +type Timer = ReturnType; + export class StateOrchestrator { - constructor(_opts: OrchestratorOptions) {} + private readonly model: StatsModel; + private readonly minDebounceMs: number; + private readonly maxDebounceMs: number; + private readonly multiplier: number; + private readonly initialRequestMs: number; + private readonly firstPaintTimeoutMs: number; - start(): void {} + private started = false; + // The stats_gen being driven; undefined while nothing is pending. + private gen: number | undefined; + // A request is out and its reply has not been seen. + private inFlight = false; + // Delay before the next request: 0 for the first state and for each reply + // in a chain, the debounce after a state change. + private delayMs = 0; + // The state the model held at start has been read, so a pending state after + // it is a state change, whatever the first one was. + private began = false; + private sentAt = 0; + private lastRequestMs: number | undefined; + private requestTimer: Timer | undefined; + private paintTimer: Timer | undefined; + private syncQueued = false; + private seenMeta: unknown; + private seenDict: unknown; + private seenState: BuckarooState | undefined; - stop(): void {} + constructor(opts: OrchestratorOptions) { + this.model = opts.model; + this.minDebounceMs = opts.minDebounceMs ?? 200; + this.maxDebounceMs = opts.maxDebounceMs ?? 3000; + this.multiplier = opts.multiplier ?? 2; + this.initialRequestMs = opts.initialRequestMs ?? 250; + this.firstPaintTimeoutMs = opts.firstPaintTimeoutMs ?? 1500; + } + /** Start watching the model, and adopt the state it already holds. */ + start(): void { + if (this.started) return; + this.started = true; + this.seenMeta = this.model.get("df_meta"); + this.seenDict = this.model.get("df_data_dict"); + this.seenState = this.model.get("buckaroo_state"); + this.model.on("change:df_meta", this.onModelChange); + this.model.on("change:df_data_dict", this.onModelChange); + this.model.on("change:buckaroo_state", this.onState); + this.model.on("msg:custom", this.onMessage); + this.sync(); + this.began = true; + } + + /** Stop watching and cancel anything scheduled. Call on unmount. */ + stop(): void { + if (!this.started) return; + this.started = false; + this.model.off("change:df_meta", this.onModelChange); + this.model.off("change:df_data_dict", this.onModelChange); + this.model.off("change:buckaroo_state", this.onState); + this.model.off("msg:custom", this.onMessage); + this.standDown(); + this.began = false; + } + + /** + * The delay (ms) before asking for a new state's stats: the last observed + * request time times the multiplier, clamped to + * `[minDebounceMs, maxDebounceMs]`. A request that took longer means the + * server was busy, so the next one waits longer. + */ computeDebounce(): number { - return 0; + const raw = (this.lastRequestMs ?? this.initialRequestMs) * this.multiplier; + return Math.max(this.minDebounceMs, Math.min(this.maxDebounceMs, raw)); + } + + // A frame fires one change event per key, in the order the server wrote + // them (df_data_dict before df_meta), so the model is read once they have + // all landed, in a microtask, not at the first event. + private readonly onModelChange = (): void => { + if (this.syncQueued) return; + this.syncQueued = true; + void Promise.resolve().then(() => { + this.syncQueued = false; + if (this.started) this.sync(); + }); + }; + + private readonly onState = (next?: BuckarooState): void => { + const prev = this.seenState; + this.seenState = next ?? this.model.get("buckaroo_state"); + if (this.gen === undefined || !touchesDataflow(prev, this.seenState)) return; + this.begin(this.gen); + }; + + private readonly onMessage = (msg?: { type?: string }): void => { + if (msg?.type === "infinite_resp") this.markPainted(); + }; + + private sync(): void { + const meta = this.model.get("df_meta") as DFMeta | undefined; + const dict = this.model.get("df_data_dict"); + const metaChanged = meta !== this.seenMeta; + const dictChanged = dict !== this.seenDict; + this.seenMeta = meta; + this.seenDict = dict; + + const stats = meta?.stats; + if (stats?.status !== "pending" || typeof stats.gen !== "number") { + // Complete, not computed, an error, or a server that reports no stats. + this.standDown(); + } else if (stats.gen !== this.gen) { + this.begin(stats.gen); + } else if (this.inFlight && dictChanged && !metaChanged) { + // A new df_data_dict under the same df_meta is a stats_update that + // is not final (a full frame for this state replaces both). One + // request per reply: ask again. + this.inFlight = false; + this.noteRequestTime(); + this.arm(); + } + } + + // Wait for rows, then ask, for `gen`. The state the model starts with asks as + // soon as rows are up; every later one is a state change and waits out the + // debounce. + private begin(gen: number): void { + this.clearTimers(); + this.gen = gen; + this.inFlight = false; + this.delayMs = this.began ? this.computeDebounce() : 0; + this.paintTimer = setTimeout(() => { + this.paintTimer = undefined; + this.markPainted(); + }, this.firstPaintTimeoutMs); + } + + private standDown(): void { + if (this.inFlight) this.noteRequestTime(); + this.clearTimers(); + this.gen = undefined; + this.inFlight = false; + } + + // The first rows are up (or the wait for them timed out): ask. + private markPainted(): void { + if (this.paintTimer !== undefined) { + clearTimeout(this.paintTimer); + this.paintTimer = undefined; + } + this.arm(); + } + + private arm(): void { + if (this.inFlight || this.gen === undefined || this.requestTimer !== undefined) return; + this.requestTimer = setTimeout(() => { + this.requestTimer = undefined; + this.fire(); + }, this.delayMs); + } + + private fire(): void { + if (requestStats(this.model)) { + this.inFlight = true; + this.sentAt = Date.now(); + this.delayMs = 0; + } + } + + private noteRequestTime(): void { + this.lastRequestMs = Date.now() - this.sentAt; + } + + private clearTimers(): void { + if (this.requestTimer !== undefined) clearTimeout(this.requestTimer); + if (this.paintTimer !== undefined) clearTimeout(this.paintTimer); + this.requestTimer = undefined; + this.paintTimer = undefined; } } diff --git a/packages/buckaroo-js-core/src/server/WebSocketModel.ts b/packages/buckaroo-js-core/src/server/WebSocketModel.ts index e7e5798d2..0d249081d 100644 --- a/packages/buckaroo-js-core/src/server/WebSocketModel.ts +++ b/packages/buckaroo-js-core/src/server/WebSocketModel.ts @@ -14,9 +14,11 @@ * Server sends a JSON text frame (infinite_resp), then a binary frame (Parquet). * This class pairs them and emits "msg:custom" with (msg, [DataView]). * - * stats_update and stats_aborted frames go to `stats` (see StatsChannel). + * stats_update and stats_aborted frames go to `stats` (see StatsChannel), and + * `scheduler` asks for the stats a session defers (see StateOrchestrator). */ import { StatsChannel } from "./StatsChannel"; +import { StateOrchestrator } from "./StateOrchestrator"; export class WebSocketModel { private ws: WebSocket; @@ -25,11 +27,13 @@ export class WebSocketModel { private state: Record; private pendingChanges: Set = new Set(); readonly stats: StatsChannel; + readonly scheduler: StateOrchestrator; constructor(ws: WebSocket, initialState: Record) { this.state = { ...initialState }; this.ws = ws; this.stats = new StatsChannel(this); + this.scheduler = new StateOrchestrator({ model: this }); this.ws.onmessage = (event: MessageEvent) => { if (typeof event.data === "string") { @@ -67,6 +71,9 @@ export class WebSocketModel { } } }; + + // Idle unless df_meta.stats says pending, which no default session does. + this.scheduler.start(); } send(msg: any): void { diff --git a/packages/buckaroo-js-core/src/style/dcf-npm.css b/packages/buckaroo-js-core/src/style/dcf-npm.css index d64c928f5..f7f689283 100644 --- a/packages/buckaroo-js-core/src/style/dcf-npm.css +++ b/packages/buckaroo-js-core/src/style/dcf-npm.css @@ -435,6 +435,24 @@ div.dependent-tabs ul.tabs li.active { @media (prefers-reduced-motion: reduce) { .bk-status-inflight-dot { animation: none; opacity: 1; } } +/* Summary stats status in the status bar (rows-first c4). One line in a + * fixed-width column, whatever the status, so a status change moves nothing. */ +.bk-stats-status { + display: inline-flex; + align-items: center; + gap: 6px; + max-width: 100%; + font-size: 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.bk-stats-status button { + font-size: 11px; + line-height: 16px; + padding: 0 6px; + cursor: pointer; +} .status-bar .ag-center-cols-viewport { min-height: unset !important; } diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index d7ccfe48f..a9487912f 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -252,6 +252,12 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.save_changes(); }, [model]); + // The status bar's Compute summary stats button, shown while the server + // reports the stats as not computed. + const onComputeStats = React.useCallback(() => { + srt.requestStats(model, { force: true }); + }, [model]); + if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
Waiting for data... @@ -272,6 +278,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} />
);