From f2ac8d509686cc1e3752f241a350d0547331a453 Mon Sep 17 00:00:00 2001 From: Amrit Krishnan Date: Wed, 18 Nov 2026 09:39:36 -0500 Subject: [PATCH 1/2] Agent demo: "Ask your data", a brief for whoever asked; hosted serves insurance The demo page was built for whoever runs the agent: an agent picker, the raw prompt, bash commands and JSON. It is now built for whoever asked the question (a business lead, a clinician): the question in plain English with its two files, then the answer as a short brief, one section per step, each a plain question ("Is the data usable?"), a one-sentence answer written from the run's numbers, and one visual. Paper and ink, a serif for what is asked and answered, color only for data. Every command and tool call is still there behind "Show the work". - ui/cleaning.py sets the raw export beside the agent's clean table (joined on the id): rows removed, spellings merged, the messiest rows. The page shows those rows as exported, then cleaned. /api/runs/{agent}/cleaning serves it; make_showcase.py saves it as cleaning.json, since showcase runs don't keep the data files. - Each scenario's LABELS gain the question, the audience and the column names as people say them; insurance gains RAW_COLUMNS and CLEAN_HISTORY. - The hosted image serves the insurance scenario. Claude Code's sandbox (bubblewrap) can't run in a container, so CLAUDE_SANDBOX=off gives it Bash as the run's own Unix user, which can't read the answers or other runs, as Codex already does there. Co-Authored-By: Claude Opus 5.5 --- examples/agent_demo/Dockerfile | 10 +- examples/agent_demo/README.md | 57 ++- examples/agent_demo/make_showcase.py | 9 +- examples/agent_demo/run.sh | 9 +- .../agent_demo/scenarios/insurance/task.py | 41 +++ .../agent_demo/scenarios/readmission/task.py | 29 ++ .../showcase/insurance/claude/cleaning.json | 296 ++++++++++++++++ examples/agent_demo/ui/app.py | 28 +- examples/agent_demo/ui/cleaning.py | 153 +++++++++ examples/agent_demo/ui/runner.py | 9 +- examples/agent_demo/ui/static/answer.css | 209 +++++++++++ examples/agent_demo/ui/static/answer.js | 325 ++++++++++++++++++ examples/agent_demo/ui/static/app.js | 99 ++++-- examples/agent_demo/ui/static/index.html | 198 +++++------ examples/agent_demo/ui/static/style.css | 178 ++++------ tests/unit/test_agent_demo_cleaning.py | 128 +++++++ tests/unit/test_agent_demo_scenarios.py | 15 +- 17 files changed, 1504 insertions(+), 289 deletions(-) create mode 100644 examples/agent_demo/showcase/insurance/claude/cleaning.json create mode 100644 examples/agent_demo/ui/cleaning.py create mode 100644 examples/agent_demo/ui/static/answer.css create mode 100644 examples/agent_demo/ui/static/answer.js create mode 100644 tests/unit/test_agent_demo_cleaning.py diff --git a/examples/agent_demo/Dockerfile b/examples/agent_demo/Dockerfile index eaa5b11..7291a3a 100644 --- a/examples/agent_demo/Dockerfile +++ b/examples/agent_demo/Dockerfile @@ -37,10 +37,11 @@ RUN pip install --no-cache-dir ".[mcp]" "fastapi>=0.115" "uvicorn>=0.30" COPY examples/agent_demo ./examples/agent_demo # The scenario the image serves (scenarios//), and its data, # prepared at build time. The answers stay in the image for scoring; agents -# only ever get the workspace copy. A scenario with a "clean" step gives -# Claude Code a sandboxed shell, which on Linux needs bubblewrap and socat -# (not installed here): the hosted demo serves readmission. -ARG DEMO_SCENARIO=readmission +# only ever get the workspace copy. The insurance scenario has the agent +# clean raw exports with a script, so each agent gets a shell; Claude Code's +# sandbox (bubblewrap) can't run in a container, so, like Codex, it runs +# in the per-run user below (CLAUDE_SANDBOX=off). +ARG DEMO_SCENARIO=insurance ENV DEMO_SCENARIO=$DEMO_SCENARIO RUN python examples/agent_demo/prepare_data.py \ && rm -rf examples/agent_demo/runs \ @@ -57,6 +58,7 @@ ENV DEMO_HOSTED=1 \ DEMO_RUNS=/tmp/runs \ DEMO_RUN_UIDS=20000-20063 \ CODEX_SANDBOX=danger-full-access \ + CLAUDE_SANDBOX=off \ TABCTX_BASE_URL=https://proxy.vectorinstitute.ai \ TABCTX_MCP_BIN=/usr/local/bin/tabctx-mcp \ DISABLE_AUTOUPDATER=1 \ diff --git a/examples/agent_demo/README.md b/examples/agent_demo/README.md index 0778280..b23f7fa 100644 --- a/examples/agent_demo/README.md +++ b/examples/agent_demo/README.md @@ -2,9 +2,9 @@ Two scenarios, each a question about a table, in [`scenarios/`](scenarios/): -- **readmission** (the default, and the hosted demo): which hospital +- **readmission** (the default): which hospital patients will be back within 30 days? Public data, below. -- **insurance**: which policyholders will cancel at renewal, and what +- **insurance** (the hosted demo): which policyholders will cancel at renewal, and what might keep them? The agent first cleans two messy raw exports with a script it writes, then does the same steps. Synthetic data, see [Insurance scenario](#insurance-scenario). @@ -97,20 +97,33 @@ uv run --extra mcp --with fastapi --with uvicorn python examples/agent_demo/ui/a # open http://127.0.0.1:8765 ``` -Pick an agent, edit the task if you like, and click **Run agent**. Each -tabctx call appears as a card as soon as the agent makes it: - -- **Inspect:** a map of the table's columns. -- **Evaluate:** both models against gradient boosting on held-out rows. -- **Predict:** the predicted split, and a risk table. -- **Explain:** which columns matter. -- **What-if:** the probability with and without a change, for one patient. - -When the agent finishes, the page shows its score against the real -outcomes and its report. **Replay last run** plays a finished run back -without running anything. Hover over any mark for its numbers. The page -has light and dark themes, and listens on 127.0.0.1 only, since it -starts agents on this machine. +The page is built for whoever asked the question, not for whoever runs +the agent. It shows the question in plain English with its two files +(`LABELS["question"]` in the scenario's `task.py`). **Ask** starts the +agent; **watch a recorded run** plays a finished one back without +running anything. The answer comes back as a short brief: one section +per step of the task, each a plain question, a one-sentence answer +written from the run's own numbers, and one visual, in the scenario's +own column names (`LABELS["column_names"]`): + +- **Is the data usable?** (a task with a "clean" step): five of the + messiest raw rows, underlined like misspellings, then shown as the + agent cleaned them, and the spellings merged into one value. The page + sets the raw file beside the clean one (`ui/cleaning.py`, using the + task's `RAW_COLUMNS` and `CLEAN_HISTORY`). +- **Can it be predicted?** The tabular model against gradient boosting, + as "N times in 100". +- **Who is most at risk?** The expected count, and the top ten. +- **What drives it?** What the predictions lean on. +- **What could change it?** Each what-if as a move from today's risk. +- **The memo,** its summary up front, and **was it right?**: its top ten + against the real outcomes. + +**Show the work** (top right) opens every command and tabctx call beside +the answer, with the agent's full instructions (editable locally). The +agent and, when hosted, the gateway key are under the agent button. +Hover over any mark for its numbers. The page has light and dark themes, +and listens on 127.0.0.1 only, since it starts agents on this machine. The UI reads the same `runs///transcript.jsonl` files that `run.sh` writes. `ui/transcripts.py` turns each agent's format into one @@ -124,8 +137,11 @@ The image holds the page, Claude Code, Codex and pi, and `tabctx-mcp`, and nothing secret. - **Anyone with the link** can replay the showcase runs in - [`showcase/readmission/`](showcase/readmission/). These are three real runs, copied by - `make_showcase.py` with session details and local paths removed. + [`showcase/insurance/`](showcase/insurance/) (Claude Code's run). Showcase runs are + real runs, copied by + `make_showcase.py` with session details and local paths removed. For a + task with a cleaning step it also saves the page's view of what was + cleaned (`cleaning.json`), since the raw and clean files aren't copied. - **To run an agent**, a visitor signs in with their own gateway key. The app checks the key's project models (`/admin/me/models`) and gives each agent the best model that project can use. Agents the project can't run @@ -164,7 +180,10 @@ When the task has a "clean" step, the agent must run code: - **Claude Code** also gets Bash, only inside its sandbox (`allowUnsandboxedCommands: false`). It can't write outside its run - folder or read `answers/`. + folder or read `answers/`. The hosted demo can't run that sandbox + (bubblewrap needs privileges a container lacks), so there + `CLAUDE_SANDBOX=off` gives it Bash as the run's own Unix user, which + can't read `answers/` or other runs, as Codex does. - **Gemini CLI** and **pi** get their shells with no sandbox, as you, so they could read `answers/`. Treat their scores on such a task as unchecked, or run them as another user. diff --git a/examples/agent_demo/make_showcase.py b/examples/agent_demo/make_showcase.py index d532c32..e9c8a6f 100644 --- a/examples/agent_demo/make_showcase.py +++ b/examples/agent_demo/make_showcase.py @@ -1,7 +1,8 @@ """Copy finished runs into showcase/, the runs anyone can replay in the hosted demo: the transcript (without the session details an agent CLI logs at start, with local paths made relative and the login name masked), the report, and the -score against the real outcomes (so the predictions file isn't needed). +score against the real outcomes (so the predictions file isn't needed), +and, for a task with a cleaning step, the page's view of what it cleaned. uv run --extra mcp python examples/agent_demo/make_showcase.py claude codex pi @@ -20,6 +21,9 @@ import scenario as scenarios from score import score +sys.path.insert(0, str(Path(__file__).resolve().parent / "ui")) +from cleaning import SAVED, for_run # noqa: E402 + HERE = Path(__file__).resolve().parent # Events that record the machine, not the run (the session's tools, cwd, # ids and account details), and pi's token-by-token streaming, which the @@ -67,6 +71,9 @@ def main(agents: list[str]) -> None: (out / "score.json").write_text( json.dumps(score(run_dir, scenario), indent=2) + "\n" ) + view = for_run(run_dir, scenario) + if view: # the page's messy -> tidy view, without the raw files + (out / SAVED).write_text(json.dumps(view, indent=1) + "\n") print(agent, len(kept), "events ->", out) diff --git a/examples/agent_demo/run.sh b/examples/agent_demo/run.sh index e8db249..0b01696 100755 --- a/examples/agent_demo/run.sh +++ b/examples/agent_demo/run.sh @@ -27,7 +27,7 @@ # The hosted demo (ui/, Dockerfile) sets: TABCTX_MCP_BIN (the tabctx-mcp # command), RUN_DIR (this run's folder), # ISOLATE_HOME=1 (the agent gets HOME inside RUN_DIR, so no state is shared -# between visitors) and CODEX_SANDBOX. +# between visitors), CODEX_SANDBOX and CLAUDE_SANDBOX=off. set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) @@ -86,6 +86,13 @@ claude) # (which would also allow an unsandboxed retry). tools+=,Bash settings=(--settings "$(printf '{"sandbox": {"enabled": true, "autoAllowBashIfSandboxed": true, "allowUnsandboxedCommands": false}, "permissions": {"deny": ["Read(/%s/**)"]}}' "$SCENARIO_ANSWERS")") + if [[ ${CLAUDE_SANDBOX:-on} == off ]]; then + # The hosted demo: its sandbox (bubblewrap) can't run in a container, + # so Bash runs as this run's own Unix user instead, which can't read + # the answers or other runs (as Codex does there). + allowed+=,Bash + settings=(--settings "$(printf '{"permissions": {"deny": ["Read(/%s/**)"]}}' "$SCENARIO_ANSWERS")") + fi fi # ${a[@]+...}: an empty array is "unbound" to macOS's bash 3.2 under set -u. claude -p "$PROMPT" ${model[@]+"${model[@]}"} ${settings[@]+"${settings[@]}"} --mcp-config "$config" --strict-mcp-config \ diff --git a/examples/agent_demo/scenarios/insurance/task.py b/examples/agent_demo/scenarios/insurance/task.py index 644cb38..2df100e 100644 --- a/examples/agent_demo/scenarios/insurance/task.py +++ b/examples/agent_demo/scenarios/insurance/task.py @@ -23,6 +23,18 @@ POSITIVE: ("Cancelled", "LAPSED", "cxl"), NEGATIVE: ("Renewed", "RENEWED", "Active"), } +# The page's "messy -> tidy" view: which raw column became which clean one, +# and the clean history the agent writes (PROMPT.md names it). +RAW_COLUMNS = { + "Prov": "province", + "Product Type": "product_type", + "Annual Premium": "annual_premium", + "Pay Method": "pay_method", + "Advisor?": "has_advisor", + "Age": "age", + "Status at Renewal": "cancelled", +} +CLEAN_HISTORY = "renewals_2025.csv" POLICY_PREFIX = "MM-" # real policies; the export also has test rows and a total @@ -37,6 +49,13 @@ def positive_rate(history) -> float: # For the page: what a row is, what the positive outcome is called, and # PROMPT.md's steps as icons (a tool's icon, "clean", or "report"). LABELS = { + "org": "Maple Mutual", + "question": ( + "Clean up these two exports, find who is likely to cancel at renewal " + "and why, and test what could keep them. Write it up for the head of " + "retention." + ), + "audience": "the head of retention", "entity": "policyholders", "outcome": "cancelled", "window": "at renewal", @@ -50,4 +69,26 @@ def positive_rate(history) -> float: ["what_if", "What-if", "Test retention offers on the likeliest to cancel"], ["report", "Report", "Write report.md"], ], + # Column names as a retention team says them, and how a value reads + # ("{}" is the value). + "column_names": { + "policy_id": ["Policy"], + "years_as_customer": ["Years as a customer"], + "age": ["Age"], + "province": ["Province"], + "product_type": ["Product"], + "annual_premium": ["Annual premium", "${}"], + "premium_change_pct": ["Premium increase", "+{}%"], + "pay_method": ["Pays by"], + "pay_freq": ["Payment frequency"], + "missed_payments_12m": ["Missed payments"], + "has_advisor": ["Has an advisor"], + "online_account": ["Online account"], + "smoker": ["Smoker"], + "advisor_contacts_12m": ["Advisor contacts"], + "other_products": ["Other products held"], + "claims_12m": ["Recent claims"], + "dependents": ["Dependents"], + "cancelled": ["Cancelled?"], + }, } diff --git a/examples/agent_demo/scenarios/readmission/task.py b/examples/agent_demo/scenarios/readmission/task.py index 9a9b225..a54fd25 100644 --- a/examples/agent_demo/scenarios/readmission/task.py +++ b/examples/agent_demo/scenarios/readmission/task.py @@ -17,6 +17,13 @@ # For the page: what a row is, what the positive outcome is called, and # PROMPT.md's steps as icons (a tool's icon, or "report"). LABELS = { + "org": "Hospital readmissions", + "question": ( + "Which of the patients we're discharging now are most likely to be " + "back within 30 days, why, and what could lower that? Write it up for " + "the care team." + ), + "audience": "the care team", "entity": "patients", "outcome": "readmitted", "window": "within 30 days", @@ -29,4 +36,26 @@ ["what_if", "What-if", "Test care changes on the highest-risk patients"], ["report", "Report", "Write report.md"], ], + # Column names as a care team says them. + "column_names": { + "patient_id": ["Patient"], + "time_in_hospital": ["Days in hospital"], + "num_lab_procedures": ["Lab tests"], + "num_procedures": ["Procedures"], + "num_medications": ["Medications"], + "number_outpatient": ["Outpatient visits (past year)"], + "number_emergency": ["Emergency visits (past year)"], + "number_inpatient": ["Hospital stays (past year)"], + "number_diagnoses": ["Diagnoses"], + "discharge_disposition": ["Discharged to"], + "admission_type": ["Admission type"], + "admission_source": ["Admitted from"], + "medical_specialty": ["Specialty"], + "diag_1": ["Main diagnosis"], + "A1Cresult": ["A1C result"], + "max_glu_serum": ["Glucose test"], + "diabetesMed": ["On diabetes medication"], + "change": ["Medication changed"], + "readmitted_30d": ["Back within 30 days?"], + }, } diff --git a/examples/agent_demo/showcase/insurance/claude/cleaning.json b/examples/agent_demo/showcase/insurance/claude/cleaning.json new file mode 100644 index 0000000..aa550ad --- /dev/null +++ b/examples/agent_demo/showcase/insurance/claude/cleaning.json @@ -0,0 +1,296 @@ +{ + "file": "renewals_2025_export.csv", + "rows_before": 6127, + "rows_after": 6000, + "removed": [ + { + "n": 120, + "what": "duplicate rows" + }, + { + "n": 7, + "what": "rows that aren't records (tests, totals)" + } + ], + "columns": [ + { + "raw": "Prov", + "clean": "province", + "kind": "category", + "before": 33, + "after": 9, + "example": { + "value": "ON", + "spellings": [ + "Ontario", + "ON", + " on", + "ont", + "ONT." + ] + } + }, + { + "raw": "Product Type", + "clean": "product_type", + "kind": "category", + "before": 14, + "after": 4, + "example": { + "value": "Health & Dental", + "spellings": [ + "H&D", + "Health & Dental", + "Health and dental", + "HEALTH/DENTAL" + ] + } + }, + { + "raw": "Annual Premium", + "clean": "annual_premium", + "kind": "numeric", + "before": 3899, + "after": 0 + }, + { + "raw": "Pay Method", + "clean": "pay_method", + "kind": "category", + "before": 12, + "after": 3, + "example": { + "value": "Cheque/invoice", + "spellings": [ + "cheque/invoice", + "Cheque", + "Check", + "INVOICE" + ] + } + }, + { + "raw": "Advisor?", + "clean": "has_advisor", + "kind": "category", + "before": 10, + "after": 2, + "example": { + "value": "No", + "spellings": [ + "No", + "FALSE", + "N", + "no", + "0" + ] + } + }, + { + "raw": "Age", + "clean": "age", + "kind": "numeric", + "before": 0, + "after": 36 + }, + { + "raw": "Status at Renewal", + "clean": "cancelled", + "kind": "category", + "before": 6, + "after": 2, + "example": { + "value": "No", + "spellings": [ + "Renewed", + "RENEWED", + "Active" + ] + } + } + ], + "sample": { + "columns": [ + [ + "Prov", + "province" + ], + [ + "Product Type", + "product_type" + ], + [ + "Annual Premium", + "annual_premium" + ], + [ + "Pay Method", + "pay_method" + ], + [ + "Advisor?", + "has_advisor" + ], + [ + "Age", + "age" + ], + [ + "Status at Renewal", + "cancelled" + ] + ], + "rows": [ + [ + [ + "ab", + "AB" + ], + [ + "Term-Life", + "Term Life" + ], + [ + "CAD 710", + "710" + ], + [ + "PAD", + "Pre-authorized debit" + ], + [ + "Y", + "Yes" + ], + [ + "N/A", + "" + ], + [ + "Renewed", + "No" + ] + ], + [ + [ + "Ontario", + "ON" + ], + [ + "term life 20", + "Term Life" + ], + [ + "$549.82", + "549.82" + ], + [ + "PAD", + "Pre-authorized debit" + ], + [ + "N", + "No" + ], + [ + "-1", + "" + ], + [ + "RENEWED", + "No" + ] + ], + [ + [ + "ont", + "ON" + ], + [ + "HEALTH/DENTAL", + "Health & Dental" + ], + [ + "$2,798.18", + "2798.18" + ], + [ + "PAD", + "Pre-authorized debit" + ], + [ + "FALSE", + "No" + ], + [ + "n/a", + "" + ], + [ + "Renewed", + "No" + ] + ], + [ + [ + "Saskatchewan", + "SK" + ], + [ + "crit. illness", + "Critical Illness" + ], + [ + "$1,354.18", + "1354.18" + ], + [ + "credit", + "Credit card" + ], + [ + "TRUE", + "Yes" + ], + [ + "0", + "" + ], + [ + "Renewed", + "No" + ] + ], + [ + [ + "ont", + "ON" + ], + [ + "H&D", + "Health & Dental" + ], + [ + "$2,107.27", + "2107.27" + ], + [ + "Auto debit", + "Pre-authorized debit" + ], + [ + "N", + "No" + ], + [ + "n/a", + "" + ], + [ + "Renewed", + "No" + ] + ] + ] + } +} diff --git a/examples/agent_demo/ui/app.py b/examples/agent_demo/ui/app.py index d183fd8..a966ae1 100644 --- a/examples/agent_demo/ui/app.py +++ b/examples/agent_demo/ui/app.py @@ -38,6 +38,7 @@ str(Path(__file__).resolve().parent), str(Path(__file__).resolve().parents[1]), ] +from cleaning import for_run as cleaning_of # noqa: E402 from gateway import ( # noqa: E402 TABULAR, RefusedKey, @@ -207,6 +208,7 @@ def dataset() -> dict | None: return { **task.LABELS, "history_file": task.HISTORY, + "current_file": task.CURRENT, "history_rows": len(history), "current_rows": len(current), "features": len(current.columns) - 1, # without the id @@ -315,12 +317,9 @@ def start(body: StartRun, request: Request) -> dict: return {"agent": run.agent, "started": run.started} -@app.get("/api/runs/{agent}/events") -async def events( - agent: str, request: Request, source: str = "auto" -) -> StreamingResponse: - """A run's events: the viewer's own run of agent, or (source=showcase, - or when they have none) the showcase run.""" +def _run_for(agent: str, request: Request, source: str) -> Run: + """The viewer's own run of agent, or (source=showcase, or when they + have none) the showcase run.""" if agent not in AGENTS: raise HTTPException(404, f"unknown agent {agent!r}") mine = my_runs(viewer(request)) @@ -329,9 +328,26 @@ async def events( run = Run(agent, SHOWCASE / agent) if HOSTED else None if not run or not run.exists: raise HTTPException(404, f"no run of {agent} yet") + return run + + +@app.get("/api/runs/{agent}/events") +async def events( + agent: str, request: Request, source: str = "auto" +) -> StreamingResponse: + run = _run_for(agent, request, source) return StreamingResponse(_stream(run), media_type="text/event-stream") +@app.get("/api/runs/{agent}/cleaning") +def cleaning(agent: str, request: Request, source: str = "auto") -> dict: + """The run's raw export beside the clean table its agent wrote.""" + view = cleaning_of(_run_for(agent, request, source).dir, SCENARIO) + if view is None: + raise HTTPException(404, "no cleaned table in this run") + return view + + async def _stream(run: Run): """Server-sent events: the transcript's events as they are written (the whole file at once for a finished run), then the outcome.""" diff --git a/examples/agent_demo/ui/cleaning.py b/examples/agent_demo/ui/cleaning.py new file mode 100644 index 0000000..11ac40a --- /dev/null +++ b/examples/agent_demo/ui/cleaning.py @@ -0,0 +1,153 @@ +"""What the agent's cleaning did, for the page's "messy -> tidy" view: the +raw export the agent got, set beside the clean table it wrote, joined on +the id. + + {"file", "rows_before", "rows_after", + "removed": [{"n", "what"}, ...], + "columns": [{"raw", "clean", "kind", "before", "after", "example"}], + "sample": {"columns": [[raw, clean], ...], "rows": [[[raw, clean], ...]]}} + +A categorical column reports how many spellings became how many values +(and the value with the most spellings); a numeric one, how many values +were text (converted) or impossible (left empty). The sample is the rows +with the most fixed cells. Which raw column became which clean one is the +scenario's call (task.RAW_COLUMNS): headers alone don't say that "Prov" +is `province`. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +import pandas as pd + +SAMPLE_ROWS = 5 +CLEAN = "clean:" +SPELLINGS_SHOWN = 5 + + +def _numeric(clean: pd.Series) -> bool: + return pd.api.types.is_numeric_dtype(clean) + + +def _show(value) -> str: + """A cell as the page shows it: blanks as empty, 2384.0 as 2384.""" + if value is None or (isinstance(value, float) and math.isnan(value)): + return "" + if isinstance(value, float): + return f"{value:.2f}".rstrip("0").rstrip(".") + return str(value) + + +def _changed(raw: str, clean) -> bool: + shown = _show(clean) + if raw == shown: + return False + try: # "2384" and "2384.0", or "55" and "55.0", are the same number + return float(raw) != float(shown) + except ValueError: + return True + + +def summarize( + raw_file: Path, + clean_file: Path, + raw_id: str, + clean_id: str, + columns: dict[str, str], +) -> dict: + raw = pd.read_csv(raw_file, dtype=str, keep_default_na=False) + clean = pd.read_csv(clean_file) + clean_ids = clean[clean_id].astype(str) + duplicates = int(raw.duplicated(raw_id).sum()) + extra = int((~raw[raw_id].drop_duplicates().isin(clean_ids)).sum()) + removed = [ + {"n": duplicates, "what": "duplicate rows"}, + {"n": extra, "what": "rows that aren't records (tests, totals)"}, + ] + pairs = [(r, c) for r, c in columns.items() if r in raw and c in clean] + # Clean columns prefixed, so a raw header can't collide with one. + joined = raw.drop_duplicates(raw_id).merge( + clean.astype({clean_id: str}).add_prefix(CLEAN), + left_on=raw_id, + right_on=CLEAN + clean_id, + ) + out = [] + for r, c in pairs: + before, after = joined[r], joined[CLEAN + c] + if _numeric(clean[c]): + as_number = pd.to_numeric(before.str.strip(), errors="coerce") + converted = int((as_number.isna() & after.notna()).sum()) + emptied = int((as_number.notna() & after.isna()).sum()) + out.append( + { + "raw": r, + "clean": c, + "kind": "numeric", + "before": converted, + "after": emptied, + } + ) + continue + spellings = before.groupby(after.astype(str)).unique() + top = max(spellings.index, key=lambda v: len(spellings[v])) + out.append( + { + "raw": r, + "clean": c, + "kind": "category", + "before": int(before.nunique()), + "after": int(after.nunique()), + "example": { + "value": top, + "spellings": [str(s) for s in spellings[top][:SPELLINGS_SHOWN]], + }, + } + ) + return { + "file": raw_file.name, + "rows_before": len(raw), + "rows_after": len(clean), + "removed": [x for x in removed if x["n"]], + "columns": out, + "sample": _sample(joined, pairs), + } + + +def _sample(joined: pd.DataFrame, pairs: list[tuple[str, str]]) -> dict: + """The rows with the most fixed cells, raw beside clean.""" + rows = [ + [(raw, clean) for raw, clean in zip(cells[::2], cells[1::2], strict=True)] + for cells in joined[[x for r, c in pairs for x in (r, CLEAN + c)]].itertuples( + index=False + ) + ] + fixed = [sum(_changed(raw, clean) for raw, clean in row) for row in rows] + best = sorted(range(len(rows)), key=lambda i: -fixed[i])[:SAMPLE_ROWS] + return { + "columns": [[r, c] for r, c in pairs], + "rows": [[[raw, _show(clean)] for raw, clean in rows[i]] for i in best], + } + + +SAVED = "cleaning.json" # a showcase run's copy (make_showcase.py) + + +def for_run(run_dir: Path, scenario) -> dict | None: + """A run's cleaning view: its saved copy, else from its own files, else + None (no cleaning step, or the agent hasn't written the clean table).""" + saved = run_dir / SAVED + if saved.is_file(): + return json.loads(saved.read_text()) + task = scenario.task + columns = getattr(task, "RAW_COLUMNS", None) + clean = run_dir / getattr(task, "CLEAN_HISTORY", "") + raw = run_dir / task.HISTORY + if not columns or not clean.is_file() or not raw.is_file(): + return None + try: + return summarize(raw, clean, scenario.raw_id, task.ID, columns) + except (KeyError, ValueError, pd.errors.ParserError): + return None # the agent's table isn't what the prompt asked for diff --git a/examples/agent_demo/ui/runner.py b/examples/agent_demo/ui/runner.py index ac5ddcd..4da1e36 100644 --- a/examples/agent_demo/ui/runner.py +++ b/examples/agent_demo/ui/runner.py @@ -32,13 +32,20 @@ # What run.sh gives an agent besides the tabctx tools: a shell only when # the scenario's task has it run code. _SHELL = SCENARIO.needs_shell +# The hosted demo runs Claude Code's shell as the run's own user, without +# its sandbox (run.sh, CLAUDE_SANDBOX=off). +_CLAUDE_SHELL = ( + "MCP · shell" + if os.environ.get("CLAUDE_SANDBOX") == "off" + else "MCP · sandboxed shell" +) AGENTS = { "claude": { "label": "Claude Code", "binary": "claude", "llm": "Claude (your login)", - "tools": "MCP · sandboxed shell" if _SHELL else "MCP · no shell", + "tools": _CLAUDE_SHELL if _SHELL else "MCP · no shell", }, "codex": { "label": "Codex", diff --git a/examples/agent_demo/ui/static/answer.css b/examples/agent_demo/ui/static/answer.css new file mode 100644 index 0000000..7c59898 --- /dev/null +++ b/examples/agent_demo/ui/static/answer.css @@ -0,0 +1,209 @@ +/* The brief (answer.js) and the page around it. Paper and ink: a serif + for what is asked and answered, hairlines instead of boxes, and color + only where the data needs it (risk, fixed, better). Tokens: style.css. */ + +/* ---- page ---------------------------------------------------------------- */ +body { background: var(--bg); } +.layout { display: grid; grid-template-columns: minmax(0, 820px); justify-content: center; gap: 56px; padding: 0 24px 120px; } +.work { display: none; } +body.show-work .layout { grid-template-columns: minmax(0, 760px) minmax(0, 560px); } +body.show-work .work { display: block; position: sticky; top: 72px; align-self: start; max-height: calc(100vh - 88px); overflow: auto; margin-top: 48px; padding-left: 28px; border-left: 1px solid var(--border); } +.work .steps { grid-template-columns: repeat(auto-fit, minmax(56px, 1fr)); } +.work .prompt-box { margin: 0 0 16px; } +.work .prompt-box summary { font-size: 12.5px; color: var(--text-3); cursor: pointer; } +.work .prompt-box textarea { margin-top: 8px; } +.work .prompt-box .link { margin-top: 6px; } +.work .run-title { font: 500 20px/1.2 var(--serif); } + +.topbar { height: 56px; padding: 0 28px; background: color-mix(in srgb, var(--bg) 88%, transparent); border-bottom: 1px solid transparent; } +.brand { gap: 12px; } +.brand-name { font: 600 17px var(--serif); letter-spacing: -0.005em; } +.brand-sub { font-size: 13px; color: var(--text-3); } +.top-right { gap: 18px; } +.text-btn { display: inline-flex; align-items: center; gap: 6px; background: none; border: 0; padding: 4px 0; font-size: 13px; color: var(--text-2); cursor: pointer; } +.text-btn:hover { color: var(--text); } +.text-btn:focus-visible, .link-btn:focus-visible { outline: 2px solid var(--accent-line); outline-offset: 3px; border-radius: 4px; } + +.switch { display: inline-flex; align-items: center; gap: 8px; font-size: 13px; color: var(--text-2); cursor: pointer; user-select: none; } +.switch input { position: absolute; opacity: 0; pointer-events: none; } +.switch .track { width: 26px; height: 15px; border-radius: 999px; background: var(--track); position: relative; transition: background 0.2s; } +.switch .knob { position: absolute; top: 2px; left: 2px; width: 11px; height: 11px; border-radius: 50%; background: var(--surface); box-shadow: 0 0 0 1px var(--border-strong); transition: transform 0.2s; } +.switch input:checked + .track { background: var(--text); } +.switch input:checked + .track .knob { transform: translateX(11px); box-shadow: none; } +.switch input:focus-visible + .track { outline: 2px solid var(--accent-line); outline-offset: 2px; } + +.popover { position: fixed; top: 52px; right: 28px; z-index: 30; width: 360px; display: flex; flex-direction: column; gap: 18px; padding: 18px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; box-shadow: 0 18px 50px rgba(20, 18, 12, 0.12); } +.popover h2 { margin: 0 0 8px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.08em; color: var(--text-3); } +.popover .agent { background: transparent; } +.popover .agent[aria-checked="true"] { border-color: var(--text); background: var(--accent-soft); box-shadow: none; } +.gateway { display: inline-flex; align-items: center; gap: 8px; font-size: 12.5px; color: var(--text-2); } +.panel-head { display: flex; align-items: baseline; justify-content: space-between; } +.link { background: none; border: 0; padding: 0; color: var(--text-2); text-decoration: underline; text-underline-offset: 3px; cursor: pointer; font-size: 12.5px; } +.link-btn { background: none; border: 0; padding: 0; font-size: 14px; color: var(--text-2); cursor: pointer; text-decoration: underline; text-decoration-color: var(--border-strong); text-underline-offset: 4px; } +.link-btn:hover:not(:disabled) { color: var(--text); text-decoration-color: currentColor; } +.link-btn:disabled { opacity: 0.4; cursor: default; } + +/* ---- the question --------------------------------------------------------- */ +.question { font: 400 34px/1.28 var(--serif); letter-spacing: -0.012em; margin: 0; text-wrap: pretty; } +.hero { padding-top: 16vh; } +.hero .question { font-size: 40px; margin: 22px 0 34px; } +.files { display: flex; flex-wrap: wrap; gap: 8px 28px; } +.file { display: inline-flex; align-items: baseline; gap: 10px; font-size: 14px; } +.file-name { font-weight: 550; } +.file-meta { color: var(--text-3); font-size: 13px; } +/* a small drawn table: a header row and a first column */ +.glyph { align-self: center; width: 16px; height: 13px; border: 1.3px solid var(--text-2); border-radius: 2px; background: linear-gradient(var(--text-2), var(--text-2)) 0 3px / 100% 1.3px no-repeat, linear-gradient(var(--text-2), var(--text-2)) 5px 0 / 1.3px 100% no-repeat; } +.hero-actions { display: flex; align-items: center; gap: 22px; } +.hero-actions .btn { height: 44px; padding: 0 26px; border-radius: 999px; font-size: 15px; } +.hero .notice { margin-top: 14px; } + +/* ---- the brief ------------------------------------------------------------- */ +.thread { padding-top: 64px; } +.brief .question { max-width: 640px; } +.files-line { margin: 14px 0 0; font-size: 13.5px; color: var(--text-3); } +.again { display: flex; align-items: center; gap: 12px; margin-top: 48px; padding-top: 24px; border-top: 1px solid var(--border); } +.again .sep { color: var(--text-3); } + +.progress { list-style: none; display: grid; grid-auto-flow: column; grid-auto-columns: 1fr; gap: 6px; margin: 40px 0 0; padding: 0; } +.progress li { font-size: 12px; color: var(--text-3); padding-top: 10px; border-top: 2px solid var(--track); transition: color 0.3s, border-color 0.3s; } +.progress li[data-state="working"] { color: var(--text); border-top-color: var(--text); animation: breathe 1.6s ease-in-out infinite; } +.progress li[data-state="done"] { color: var(--text); border-top-color: var(--text); } +@keyframes breathe { 50% { border-top-color: var(--border-strong); } } +.status-line { display: flex; align-items: center; gap: 10px; margin: 16px 0 0; font-size: 13.5px; color: var(--text-2); } +.pulse-dot { width: 6px; height: 6px; border-radius: 50%; background: var(--text); animation: pulse 1.2s ease-in-out infinite; } + +.sec { display: grid; grid-template-columns: 56px minmax(0, 1fr); padding: 40px 0 8px; margin-top: 40px; border-top: 1px solid var(--border); animation: arrive 0.5s ease-out; } +.sec-num { font: 500 12px var(--mono); color: var(--text-3); padding-top: 9px; } +.sec[data-state="working"] .sec-num { color: var(--text); } +.sec h2 { margin: 0; font: 500 28px/1.2 var(--serif); letter-spacing: -0.01em; } +.lede { margin: 12px 0 0; font-size: 17px; line-height: 1.6; color: var(--text-2); max-width: 660px; } +.lede b { color: var(--text); font-weight: 600; } +.lede.doing { color: var(--text-3); font-style: italic; } +.viz:not(:empty) { margin-top: 26px; } +.viz-foot { display: flex; justify-content: space-between; align-items: baseline; margin-top: 10px; font-size: 12.5px; color: var(--text-3); } +.viz-note { margin: 14px 0 0; font-size: 13px; color: var(--text-3); } +@keyframes arrive { from { opacity: 0; transform: translateY(8px); } to { opacity: 1; transform: none; } } + +/* 01 · tidy: the raw cells underlined like a misspelling, then cleaned */ +.tidy { overflow-x: auto; } +.tidy-table { width: 100%; border-collapse: collapse; font-size: 14px; } +.tidy-table th, .tidy-table td { padding: 9px 14px 9px 0; text-align: left; white-space: nowrap; border-bottom: 1px solid var(--border); background: none; } +.tidy-table th { font-size: 11.5px; font-weight: 500; color: var(--text-3); text-transform: none; letter-spacing: 0; border-bottom-color: var(--border-strong); } +.tidy-table tr:last-child td { border-bottom: 0; } +.tidy-table .cell { display: inline-grid; } +.tidy-table .cell > span { grid-area: 1 / 1; transition: opacity 0.5s ease, filter 0.5s ease; } +.tidy-table .clean { opacity: 0; filter: blur(2px); } +.tidy-table .raw { font-family: var(--mono); font-size: 12px; } +.tidy-table th .raw { font-size: 11px; } +.tidy-table td.fixed .raw { color: var(--text); text-decoration: underline wavy var(--bad); text-decoration-thickness: 1px; text-underline-offset: 4px; } +.tidy-table.tidy .raw { opacity: 0; filter: blur(2px); } +.tidy-table.tidy .clean { opacity: 1; filter: none; } +.tidy-table td.fixed .clean { font-weight: 550; } +/* each fixed cell flashes once as it is cleaned, then settles to ink */ +.tidy-table.tidy td.fixed { animation: settled 1.8s ease-out; } +@keyframes settled { from { background: var(--good-soft); } to { background: transparent; } } +.tidy-table .blank { color: var(--text-3); font-style: normal; } +.merges { margin-top: 26px; display: flex; flex-direction: column; } +.merge { display: grid; grid-template-columns: 120px minmax(0, 1fr) 20px auto; gap: 14px; align-items: baseline; padding: 10px 0; border-top: 1px solid var(--border); font-size: 13.5px; } +.merge-name { color: var(--text-3); font-size: 12.5px; } +.spellings { display: flex; flex-wrap: wrap; gap: 4px 12px; font-family: var(--mono); font-size: 12.5px; color: var(--text-2); } +.spellings s { text-decoration: line-through; text-decoration-color: color-mix(in srgb, var(--bad) 70%, transparent); white-space: pre; } +.merge .arrow { color: var(--text-3); } +.merged { font-weight: 600; } + +/* 02 · accuracy */ +.acc { display: grid; grid-template-columns: 270px minmax(0, 1fr) 56px; gap: 18px; align-items: center; padding: 8px 0; } +.acc-name { font-size: 14px; font-weight: 550; } +.acc-name small { display: block; font-size: 12px; font-weight: 400; color: var(--text-3); margin-top: 1px; } +.acc-track { height: 6px; background: var(--track); border-radius: 3px; } +.acc-fill { height: 100%; background: var(--s1); border-radius: 3px; transform-origin: left; animation: grow 0.9s cubic-bezier(.2,.7,.2,1); } +.acc.base .acc-fill { background: var(--neutral); } +.acc-n { font: 400 34px/1 var(--serif); text-align: right; font-variant-numeric: tabular-nums; } +.acc.base .acc-n { color: var(--text-3); } +.acc-axis { display: flex; justify-content: space-between; margin: 4px 74px 0 288px; font-size: 11.5px; color: var(--text-3); } + +/* 03 · who */ +.ranked { width: 100%; border-collapse: collapse; font-size: 14px; } +.ranked td { padding: 9px 12px 9px 0; border-bottom: 1px solid var(--border); white-space: nowrap; background: none; animation: arrive 0.4s ease-out both; animation-delay: calc(var(--i) * 50ms); } +.ranked tr:last-child td { border-bottom: 0; } +.ranked .rank { width: 24px; color: var(--text-3); font: 12px var(--mono); } +.ranked .id { font: 500 12.5px var(--mono); width: 100px; } +.risk { width: 170px; } +.risk-track { display: inline-block; vertical-align: middle; width: 110px; height: 6px; border-radius: 3px; background: var(--track); overflow: hidden; margin-right: 10px; } +.risk-fill { display: block; height: 100%; background: var(--s2); border-radius: 3px; } +.risk b { font-weight: 600; font-variant-numeric: tabular-nums; } +.ranked .why { color: var(--text-3); font-size: 12.5px; white-space: normal; } + +/* 04 · drivers */ +.drv { display: grid; grid-template-columns: 200px minmax(0, 1fr); gap: 18px; align-items: center; padding: 6px 0; font-size: 14px; color: var(--text-3); } +.drv.main { color: var(--text); } +.drv-track { height: 6px; } +.drv-fill { height: 100%; border-radius: 3px; background: var(--neutral); transform-origin: left; animation: grow 0.9s cubic-bezier(.2,.7,.2,1); } +.drv.main .drv-fill { background: var(--s1); } + +/* 05 · what-ifs: today's risk, and where one change moves it */ +.person { padding: 16px 0 10px; border-top: 1px solid var(--border); animation: arrive 0.4s ease-out; } +.person:first-child { border-top: 0; padding-top: 0; } +.person-head { display: flex; align-items: baseline; gap: 14px; margin-bottom: 8px; } +.person-head .id { font: 500 12.5px var(--mono); } +.person-head .facts { font-size: 12.5px; color: var(--text-3); } +.person-head .today { margin-left: auto; font-size: 12.5px; color: var(--text-3); white-space: nowrap; } +.person-head .today b { color: var(--s2); font-weight: 600; font-variant-numeric: tabular-nums; } +.try { display: grid; grid-template-columns: 250px minmax(0, 1fr) 44px; gap: 18px; align-items: center; padding: 5px 0; font-size: 13.5px; } +.try-what { color: var(--text-2); line-height: 1.35; } +.try-track { position: relative; height: 14px; } +.try-track::before { content: ""; position: absolute; left: 0; right: 0; top: 50%; height: 1px; background: var(--border); } +.try-span { position: absolute; top: 50%; height: 2px; transform: translateY(-50%); background: var(--good); opacity: 0.55; } +.try-span.up { background: var(--bad); } +.try-track i { position: absolute; top: 50%; width: 9px; height: 9px; border-radius: 50%; transform: translate(-50%, -50%); } +.try-track i.now { background: var(--s2); } +.try-track i.down { background: var(--good); } +.try-track i.up { background: var(--bad); } +.try b { text-align: right; font-weight: 600; font-variant-numeric: tabular-nums; } +.try b.down { color: var(--good); } +.try b.up { color: var(--bad); } + +/* the memo: a letter, not a card */ +.memo { font: 400 17px/1.65 var(--serif); color: var(--text); max-width: 620px; } +.memo h2 { font: 600 12px var(--sans); text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-3); margin: 22px 0 6px; } +.memo p { margin: 10px 0; } +.memo ul, .memo ol { padding-left: 20px; } +.memo li { margin: 8px 0; } +.memo strong { font-weight: 600; } +.memo table { display: block; overflow-x: auto; font: 13px var(--sans); border-collapse: collapse; margin: 12px 0; } +.memo th, .memo td { border-bottom: 1px solid var(--border); padding: 6px 12px 6px 0; text-align: left; white-space: normal; background: none; } +.memo th { font-size: 12px; text-transform: none; letter-spacing: 0; } +.memo code { font-size: 0.8em; background: var(--surface-2); border-radius: 4px; padding: 0 4px; } +.memo-more summary { font: 14px var(--sans); color: var(--text-2); cursor: pointer; margin-top: 14px; text-decoration: underline; text-decoration-color: var(--border-strong); text-underline-offset: 4px; list-style: none; } +.memo-more summary::-webkit-details-marker { display: none; } +.memo-more[open] summary { margin-bottom: 8px; } + +/* the check */ +.check .sec-num { color: var(--good); font-size: 14px; } +.tally { display: grid; grid-template-columns: 120px auto 1fr; gap: 18px; align-items: center; padding: 8px 0; font-size: 13.5px; } +.tally-name { color: var(--text-2); } +.tally .dots { display: grid; grid-template-columns: repeat(10, 14px); gap: 6px; } +.tally .dots i { width: 14px; height: 14px; border-radius: 50%; border: 1.3px solid var(--border-strong); } +.tally .dots i.hit { background: var(--s2); border-color: var(--s2); } +.tally.base .dots i.hit { background: var(--neutral); border-color: var(--neutral); } +.tally b { font: 400 26px/1 var(--serif); } +.tally.base b { color: var(--text-3); } + +@media (max-width: 1240px) { + body.show-work .layout { grid-template-columns: minmax(0, 820px); } + body.show-work .work { position: static; max-height: none; margin-top: 0; padding: 24px 0 0; border-left: 0; border-top: 1px solid var(--border); } +} +@media (max-width: 720px) { + .topbar { padding: 0 16px; } + .brand-sub, .switch-label { display: none; } + .layout { padding: 0 16px 80px; } + .hero { padding-top: 8vh; } + .hero .question { font-size: 28px; } + .question { font-size: 25px; } + .sec { grid-template-columns: 1fr; } + .sec-num { padding: 0 0 6px; } + .acc, .drv, .try, .merge, .tally { grid-template-columns: 1fr; gap: 6px; } + .acc-axis { margin: 4px 0 0; } + .ranked .why { display: none; } + .popover { left: 16px; right: 16px; width: auto; } +} diff --git a/examples/agent_demo/ui/static/answer.js b/examples/agent_demo/ui/static/answer.js new file mode 100644 index 0000000..4b3c47c --- /dev/null +++ b/examples/agent_demo/ui/static/answer.js @@ -0,0 +1,325 @@ +// The answer as a brief for whoever asked: one section per step of the +// task, each a plain question ("Is the data usable?"), a one-sentence +// answer written from the run's own numbers, and one quiet visual. Then +// the agent's memo, and a check against what really happened. Every +// command and tool call stays in the work panel (app.js), which reads the +// same events. Helpers (esc, fmt, pct, tipAttr, base) are app.js's. + +const Answer = (() => { + let S = null; // this run + + // Each step: its question, and its short name on the progress line. + const STEPS = { + clean: ["Is the data usable?", "Tidy"], + evaluate: ["Can it be predicted?", "Check"], + predict: ["Who is most at risk?", "Score"], + explain: ["What drives it?", "Explain"], + what_if: ["What could change it?", "Test"], + report: ["The memo", "Write"], + }; + const STEP_OF = { inspect_table: "evaluate", list_models: "evaluate", evaluate: "evaluate", fit_model: "predict", predict: "predict", explain: "explain", what_if: "what_if" }; + const DOING = { + inspect_table: "Reading the clean table", + list_models: "Seeing which models it can use", + evaluate: "Testing on last year's rows it hasn't seen", + fit_model: "Learning from the past rows", + predict: "Scoring every row", + explain: "Scrambling one column at a time", + what_if: "Changing one thing at a time", + }; + + // ---- words ------------------------------------------------------------------ + const cols = () => S.d.column_names || {}; + const label = (c) => cols()[c]?.[0] || String(c).replace(/_/g, " ").replace(/^\w/, (x) => x.toUpperCase()); + const lower = (t) => t.charAt(0).toLowerCase() + t.slice(1); + const num = (v) => { + const n = Number(v); + if (v === "" || v == null || Number.isNaN(n)) return String(v ?? ""); + return Math.abs(n) >= 100 ? fmt(n) : fmt(n, Number.isInteger(n) ? 0 : 1); + }; + const value = (c, v) => (v === "" || v == null ? "unknown" : (cols()[c]?.[1] || "{}").replace("{}", num(v))); + const said = (c, v) => `${label(c)} ${value(c, v)}`; + const changed = (c, v) => `${label(c)} → ${value(c, v)}`; + const list = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} and ${xs.at(-1)}`); + const words = (n) => ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"][n] ?? fmt(n); + + // ---- the run ---------------------------------------------------------------- + function reset(setup, agent, replaying) { + const d = setup.dataset || {}; + S = { d, agent, replaying, steps: new Map(), calls: new Map(), top: new Map(), people: 0, predicted: false, cleaned: false }; + const keys = (d.steps || []).map(([k]) => k).filter((k) => STEPS[k]); + $("#answer").innerHTML = ` +
    ${keys.map((k) => `
  1. ${STEPS[k][1]}
  2. `).join("")}
+

Working on it

+
`; + keys.forEach((key, i) => { + const el = document.createElement("section"); + el.className = "sec"; + el.hidden = true; + el.innerHTML = `
${String(i + 1).padStart(2, "0")}

${STEPS[key][0]}

`; + $("#sections").appendChild(el); + S.steps.set(key, { el, state: "idle" }); + }); + } + + function setStep(key, state, doing) { + const step = S.steps.get(key); + if (!step) return; + if (step.state === "done" && state === "working") state = "done"; // more of a finished step + step.state = state; + step.el.hidden = state === "idle"; + step.el.dataset.state = state; + $(`#progress [data-step="${key}"]`)?.setAttribute("data-state", state); + const lede = step.el.querySelector(".lede"); + if (state === "working" && doing && lede && !lede.dataset.final) { + lede.textContent = `${doing}…`; + lede.classList.add("doing"); + } + } + function write(key, sentence, viz) { + const step = S.steps.get(key); + if (!step) return; + const lede = step.el.querySelector(".lede"); + lede.innerHTML = sentence; + lede.dataset.final = "1"; + lede.classList.remove("doing"); + if (viz != null) step.el.querySelector(".viz").innerHTML = viz; + setStep(key, "done"); + } + const has = (key) => S.steps.has(key); + + function doingOf(tool, a = {}) { + if (/^(write|edit)/i.test(tool)) return `Writing ${base(a.file_path || a.path || "a script")}`; + if (/clean\.py/.test(a.command || "")) return "Running its cleaning script"; + return "Looking through the raw files"; + } + + function event(ev) { + if (!S) return; + if (ev.type === "tool_call") { + S.calls.set(ev.id, ev); + if (!ev.tabctx) { + const path = ev.args?.file_path || ev.args?.path || ""; + if (/report\.md$/.test(path)) { + for (const [k, s] of S.steps) if (k !== "report" && s.state === "working") setStep(k, "done"); + return setStep("report", "working", "Writing the memo"); + } + if (has("clean") && !S.cleaned) setStep("clean", "working", doingOf(ev.tool, ev.args)); + return; + } + if (has("clean") && !S.cleaned) loadCleaning(); + // Later predicts (a program tried on many rows) belong to the what-ifs. + const step = ev.tool === "predict" && S.predicted ? "what_if" : STEP_OF[ev.tool]; + if (step) setStep(step, "working", DOING[ev.tool]); + return; + } + if (ev.type !== "tool_result") return; + const call = S.calls.get(ev.id); + if (!call) return; + if (!call.tabctx) { + if (has("clean") && !S.cleaned && !ev.error && /clean\.py/.test(call.args?.command || "")) loadCleaning(); + return; + } + const r = ev.result; + if (ev.error || !r) return; + if (call.tool === "evaluate") evaluate(r); + else if (call.tool === "predict" && !S.predicted) predict(r, call.args); + else if (call.tool === "explain") explain(r); + else if (call.tool === "what_if") whatIf(r, call.args); + } + + // ---- 01 · is the data usable? ------------------------------------------------- + async function loadCleaning() { + S.cleaned = true; + setStep("clean", "working", "Setting the raw rows beside the clean ones"); + let v = null; + try { + const res = await fetch(`/api/runs/${S.agent}/cleaning`); + if (res.ok) v = await res.json(); + } catch {} + if (!v) return write("clean", "The agent cleaned both exports with a script of its own.", ""); + const removed = v.removed.map((x) => `${fmt(x.n)} ${x.what.replace(/ \(.*\)$/, "")}`); + const top = v.columns.filter((c) => c.kind === "category" && c.before > c.after).sort((a, b) => b.before - a.before)[0]; + const sentence = + `Not as exported. The file had ${fmt(v.rows_before)} rows; ${fmt(v.rows_after)} are distinct ${esc(S.d.entity || "records")}. ` + + `The agent wrote a script that removed ${esc(list(removed))}` + + (top ? `, and merged ${fmt(top.before)} spellings of ${esc(lower(label(top.clean)))} into ${fmt(top.after)}.` : "."); + write("clean", sentence, cleanViz(v)); + const table = S.steps.get("clean").el.querySelector(".tidy-table"); + setTimeout(() => flip(table, true), 2200); // the mess first, then the fix + } + + function flip(table, tidy) { + if (!table) return; + table.classList.toggle("tidy", tidy); + table.closest(".viz").querySelector(".flip").textContent = tidy ? "Show as exported" : "Show cleaned"; + } + + function cleanViz(v) { + const cell = (raw, clean) => { + const fixed = raw !== clean && !(raw !== "" && clean !== "" && Number(raw) === Number(clean)); + const shownRaw = esc(raw).replace(/^ | $/g, "␣") || " "; + return `${shownRaw}${esc(clean) || '—'}`; + }; + const head = v.sample.columns.map(([raw, clean]) => `${esc(raw)}${esc(label(clean))}`).join(""); + const rows = v.sample.rows.map((row) => `${row.map(([raw, clean]) => cell(raw, clean)).join("")}`).join(""); + const merges = v.columns + .filter((c) => c.kind === "category" && c.before > c.after) + .sort((a, b) => b.before - a.before) + .slice(0, 3) + .map( + (c) => `
${esc(label(c.clean))} + ${c.example.spellings.map((s) => `${esc(s).replace(/^ | $/g, "␣")}`).join("")} + →${esc(c.example.value)}
`, + ) + .join(""); + return `
${head}${rows}
+
Five of the messiest rows
+
${merges}
`; + } + + // ---- 02 · can it be predicted? ------------------------------------------------- + function evaluate(r) { + const m = r.best_by; + const ok = r.results.filter((x) => !x.error && x.metrics[m] != null); + const gbm = ok.find((x) => x.model === "gradient-boosting"); + const tfm = ok.filter((x) => x.model !== "gradient-boosting").sort((a, b) => b.metrics[m] - a.metrics[m])[0]; + if (!tfm) return write("evaluate", "The models couldn't be compared on this table.", ""); + if (m !== "roc_auc") { + return write("evaluate", `On last year's rows it hadn't seen, the tabular model scores ${fmt(tfm.metrics[m], 3)} (${esc(metricName(m))})${gbm ? `, against ${fmt(gbm.metrics[m], 3)} for the standard approach` : ""}.`, ""); + } + const n = (x) => Math.round(x.metrics[m] * 100); + const verdict = n(tfm) >= 75 ? "Yes." : n(tfm) >= 65 ? "Fairly well." : "Only somewhat."; + const sentence = + `${verdict} Shown one who ${esc(S.d.outcome)} and one who didn't, it picks right ${n(tfm)} times in 100, on ${fmt(r.test_rows)} of last year's rows it hadn't seen.` + + (gbm ? ` A model built for this table the standard way gets ${n(gbm)}.` : ""); + const x = (v) => `${Math.max(0, (v - 0.5) / 0.5) * 100}%`; + const row = (res, name, sub, cls) => + `
${name}${sub}
${n(res)}
`; + const viz = + row(tfm, "Tabular foundation model", `${esc(tfm.model)} · nothing trained on your data`, "main") + + (gbm ? row(gbm, "Standard approach", "gradient boosting, built for this table", "base") : "") + + `
50 · a coin flip100 · always right
`; + write("evaluate", sentence, viz); + } + + // ---- 03 · who is most at risk? ------------------------------------------------- + function predict(r, args = {}) { + S.predicted = true; + const key = r.ranked_by || ""; + const pos = S.d.positive || "Yes"; + const mean = r.summary?.mean_probability?.[pos]; + const id = Object.keys(r.top_rows[0] || {})[0]; // the id comes first + const show = (args.show || []).filter((c) => c !== id).slice(0, 2); + S.show = show; + const k = r.top_rows.length; + const sentence = + (mean != null ? `About ${fmt(Math.round(mean * r.rows))} of the ${fmt(r.rows)} ${esc(S.d.current)} are likely ${esc(S.d.outcome)}. ` : "") + + `These ${words(k)} are the most likely.`; + const rows = r.top_rows + .map((row, i) => { + const p = Number(row[key]); + S.top.set(String(row[id]), row); + return `${i + 1}${esc(row[id])} + ${pct(p)} + ${esc(show.filter((c) => c in row).map((c) => said(c, row[c])).join(" · "))}`; + }) + .join(""); + write("predict", sentence, `${rows}
`); + } + + // ---- 04 · what drives it? ------------------------------------------------------- + function explain(r) { + const top = r.importance.filter((i) => i.score_drop > 0).slice(0, 6); + if (!top.length) return write("explain", "No single column stands out.", ""); + const max = top[0].score_drop; + const names = top.map((i) => lower(label(i.column))); + const sentence = `Mostly ${esc(names[0])}${names.length > 2 ? `, then ${esc(list(names.slice(1, 3)))}` : ""}. That is what the predictions lean on, not proof of what causes it.`; + const viz = top + .map( + (i, k) => `
${esc(label(i.column))} +
${esc(label(i.column))} makes the predictions worse by ${fmt(i.score_drop, 3)}`)}>
`, + ) + .join(""); + write("explain", sentence, viz); + } + + // ---- 05 · what could change it? -------------------------------------------------- + function whatIf(r, args = {}) { + const pos = S.d.positive || "Yes"; + const res = r.results || []; + if (!res.length || !res[0].probabilities) return; + S.people++; + const today = res[0].probabilities[pos]; + const id = args.base_where ? Object.values(args.base_where).join(", ") : ""; + const known = S.top.get(String(id)); + const facts = known ? (S.show || []).filter((c) => c in known).map((c) => said(c, known[c])).join(" · ") : ""; + const at = (p) => `${Math.max(0, Math.min(1, p)) * 100}%`; + const tries = res + .slice(1) + .map((x, i) => { + const p = x.probabilities[pos]; + const what = Object.entries(x.changes ?? args.rows?.[i + 1] ?? {}).map(([c, v]) => changed(c, v)).join(" + ") || "As it is"; + const dir = p < today ? "down" : "up"; + const lo = Math.min(p, today), hi = Math.max(p, today); + return `
${esc(what)} + + ${pct(p)}
`; + }) + .join(""); + const person = `
${esc(id)}${esc(facts)}today ${pct(today)}
${tries}
`; + const viz = S.steps.get("what_if").el.querySelector(".viz").innerHTML + person; + write("what_if", `For the ${words(S.people)} most at risk, the agent changed one thing at a time and asked the model again. These are patterns the model learned, worth a pilot, not proven effects.`, viz); + } + + // ---- the memo and the check ------------------------------------------------------- + function outcome(o, stats) { + if (!S) return; + for (const [k, s] of S.steps) if (s.state === "working") setStep(k, "done"); + if (o.report && has("report")) { + const step = S.steps.get("report"); + step.el.querySelector(".lede")?.remove(); + step.el.querySelector(".viz").innerHTML = memo(o.report); + setStep("report", "done"); + } + const secs = stats?.seconds; + $("#agent-status").innerHTML = o.report + ? `Answered${secs ? ` in ${Math.floor(secs / 60)} min ${Math.round(secs % 60)} s` : ""}, from one question and two raw files.` + : "The agent stopped without an answer."; + $("#final").innerHTML = check(o.score || {}); + } + + function memo(md) { + // The memo's summary up front; the rest folded. + const lines = md.split("\n").filter((l) => !l.startsWith("# ")); + const parts = lines.join("\n").split(/\n(?=## )/); + const lead = parts.findIndex((p) => p.startsWith("## ")); + const head = lead >= 0 ? parts.slice(0, lead + 1).join("\n") : parts[0]; + const more = lead >= 0 ? parts.slice(lead + 1).join("\n") : ""; + const html = (m) => DOMPurify.sanitize(marked.parse(m)); + return `
${html(head)} + ${more.trim() ? `
Read the whole memo${html(more)}
` : ""}
`; + } + + function check(s) { + if (s.error || s.positive_in_top_10 == null) return ""; + const what = S.d.outcome || "positive"; + const hits = Math.round(s.positive_in_top_10 * 10); + const random = Math.max(1, Math.round(s.base_rate * 10)); + const dots = (n) => Array.from({ length: 10 }, (_, i) => ``).join(""); + return `
✓

Was it right?

+

The real outcomes were hidden from the agent. Of the ten it ranked highest, ${words(hits)} really ${esc(what)}; ten picked at random would hold about ${words(random)}.

+
Its top ten${dots(hits)}${hits}
+
Ten at random${dots(random)}${random}
+

Of its top hundred, ${pct(s.positive_in_top_100)} ${esc(what)}, against ${pct(s.base_rate)} of everyone.

`; + } + + document.addEventListener("click", (e) => { + const btn = e.target.closest(".flip"); + if (!btn) return; + const t = btn.closest(".viz").querySelector(".tidy-table"); + flip(t, !t.classList.contains("tidy")); + }); + + return { reset, event, outcome }; +})(); diff --git a/examples/agent_demo/ui/static/app.js b/examples/agent_demo/ui/static/app.js index 4e70022..f55548f 100644 --- a/examples/agent_demo/ui/static/app.js +++ b/examples/agent_demo/ui/static/app.js @@ -1,6 +1,7 @@ // tabctx agent demo: one page, no framework. The server streams each -// agent's transcript as normalized events (../transcripts.py); each tabctx -// tool gets a card built around one visual, with details on hover. +// agent's transcript as normalized events (../transcripts.py). Two views of +// the same events: the answer a business reader follows (answer.js), and, +// with "Show the work", every command and tabctx call as a card here. const $ = (sel) => document.querySelector(sel); const esc = (v) => @@ -113,17 +114,19 @@ function initTooltips() { async function init() { initTheme(); initTooltips(); + initPanels(); const setup = await loadSetup(); - renderDataset(setup.dataset); - renderTask(setup.dataset); + renderComposer(setup.dataset); $("#prompt").value = setup.prompt; $("#prompt").readOnly = !setup.prompt_editable; $("#reset-prompt").hidden = !setup.prompt_editable; state.agent = setup.running || setup.runs.at(-1)?.agent || setup.agents.find((a) => a.available)?.id; renderAgents(); $("#reset-prompt").onclick = () => ($("#prompt").value = setup.prompt); - $("#run").onclick = start; + $("#run").onclick = () => (state.setup.hosted && !state.setup.signed_in ? openSettings(true) : start()); $("#replay").onclick = () => watch(state.agent, true); + $("#again-replay").onclick = () => watch(state.agent, true); + $("#again-new").onclick = newQuestion; $("#login").onsubmit = signIn; $("#logout").onclick = signOut; if (setup.running) watch(setup.running, false); @@ -186,37 +189,54 @@ function renderGateway(setup) { : "No TABCTX_API_KEY: runs will fail"; } -function renderDataset(d) { +// The question as it would be asked, with the two files it is about. +function renderComposer(d) { if (!d) { - $("#dataset").innerHTML = `

Run prepare_data.py first.

`; + $("#question").innerHTML = `Run prepare_data.py first.`; return; } - const yes = d.positive_rate; - const classes = [d.negative, d.positive]; - $("#dataset").innerHTML = ` -
-
${fmt(d.history_rows)}
${esc(d.history)}
-
${segBar([[d.negative, 1 - yes], [d.positive, yes]], classes, 8)}
-
${pct(yes, 1)} ${esc(d.outcome)}
-
-
-
${fmt(d.current_rows)}
${esc(d.current)}
-
${svg('', 12)} outcome hidden
-
${d.features} columns
-
`; -} - -// The task (PROMPT.md's steps, from task.py) as icons; the prompt is folded away. -const REPORT_ICON = ''; -function renderTask(d) { - const steps = d?.steps || []; - $("#task-flow").innerHTML = steps - .map( - ([tool, label, tip]) => - `
${svg(ICONS[tool] || REPORT_ICON)}${esc(label)}
`, - ) - .join(""); - $("#prompt-box").open = Boolean(state.setup.prompt_editable); + $("#org").textContent = d.org || ""; + const file = (name, rows, what, note) => + `
${esc(what)}${fmt(rows)} rows${note ? ` · ${note}` : ""}
`; + $("#attachments").innerHTML = file(d.history_file, d.history_rows, cap(d.history)) + file(d.current_file, d.current_rows, cap(d.current), "outcome hidden"); + $("#you-files").textContent = `From ${d.history} (${fmt(d.history_rows)} rows) and ${d.current} (${fmt(d.current_rows)} rows, outcome hidden)`; + $("#question").textContent = d.question || ""; + $("#you-text").textContent = d.question || ""; +} +const cap = (t) => String(t || "").replace(/^\w/, (c) => c.toUpperCase()); + +// The settings popover and the "Show the work" switch. +function initPanels() { + $("#settings-btn").onclick = (e) => { + e.stopPropagation(); + openSettings($("#settings").hidden); + }; + document.addEventListener("click", (e) => { + if (!$("#settings").hidden && !e.target.closest("#settings")) openSettings(false); + }); + document.addEventListener("keydown", (e) => e.key === "Escape" && openSettings(false)); + let work = false; + try { + work = localStorage.getItem("work") === "1"; + } catch {} + const toggle = $("#work-toggle"); + toggle.checked = work; + document.body.classList.toggle("show-work", work); + toggle.onchange = () => { + document.body.classList.toggle("show-work", toggle.checked); + try { + localStorage.setItem("work", toggle.checked ? "1" : "0"); + } catch {} + }; +} +function openSettings(open) { + $("#settings").hidden = !open; + $("#settings-btn").setAttribute("aria-expanded", String(open)); +} +function newQuestion() { + state.source?.close(); + $("#thread").hidden = true; + $("#empty").hidden = false; } function renderAgents() { @@ -248,8 +268,9 @@ function renderAgents() { $("#replay").disabled = !(state.agent in scores) || isBusy(); const agent = state.setup.agents.find((a) => a.id === state.agent); const signedOut = hosted && !state.setup.signed_in; - $("#run").disabled = isBusy() || signedOut || !agent?.allowed; - $("#run-label").textContent = signedOut ? "Sign in to run" : "Run agent"; + $("#run").disabled = isBusy() || (!signedOut && !agent?.allowed); + $("#run-label").textContent = signedOut ? "Sign in to ask" : "Ask"; + $("#agent-name").textContent = agent ? agent.label : "Agent"; } function selectAgent(id) { @@ -288,7 +309,10 @@ function watch(agent, replay) { Object.assign(state, { calls: new Map(), tabctxCalls: 0, used: new Set(), queue: [], replaying: replay, stats: null }); const info = state.setup.agents.find((a) => a.id === agent); $("#empty").hidden = true; - $("#run-view").hidden = false; + $("#thread").hidden = false; + $("#again").hidden = true; + openSettings(false); + Answer.reset(state.setup, agent, replay); $("#timeline").innerHTML = ""; $("#outcome").hidden = true; $("#report-card").open = false; @@ -340,6 +364,7 @@ function inlineMd(text) { } function handle(ev, info) { + Answer.event(ev); const tl = $("#timeline"); if (ev.type === "message") { const el = document.createElement("div"); @@ -406,6 +431,8 @@ function finish(outcome, info) { ); } renderOutcome(outcome); + Answer.outcome(outcome, state.stats); + $("#again").hidden = false; refreshRuns(); // re-enables Run as the viewer's access allows } diff --git a/examples/agent_demo/ui/static/index.html b/examples/agent_demo/ui/static/index.html index b3fe0d7..974efa6 100644 --- a/examples/agent_demo/ui/static/index.html +++ b/examples/agent_demo/ui/static/index.html @@ -3,140 +3,124 @@ - tabctx agent demo + Ask your data · tabctx - + +
- - tabctx - - Agent demo + Ask your data +
- - + +
-
- -
-
-

A coding agent that can predict from a table.

-
-
-
-
Two CSVs
-
past · current
-
- -
-
-
Coding agent
-
plans · calls tools
-
- -
-
-
tabctx on a GPU
-
TabPFNTabICL
-
- -
-
-
Scored answer
-
vs real outcomes
-
+ +
-
+ diff --git a/examples/agent_demo/ui/static/style.css b/examples/agent_demo/ui/static/style.css index 419458e..364f0f2 100644 --- a/examples/agent_demo/ui/static/style.css +++ b/examples/agent_demo/ui/static/style.css @@ -1,79 +1,80 @@ /* tabctx agent demo. Tokens by role; both themes are chosen, not flipped. - Data colors (validated for CVD and contrast in both modes, dataviz skill): - class "No" = categorical slot 1 (blue), class "Yes" = slot 2 (orange); + Paper and ink, with color only for data: a blue and vermilion pair that + stays apart for color-blind readers. class "No" = categorical slot 1 (blue), class "Yes" = slot 2 (vermilion); every model bar = slot 1; the gradient-boosting baseline = neutral. */ :root { color-scheme: light; - --bg: #f4f4f1; - --surface: #fcfcfb; - --surface-2: #f6f6f3; - --border: #e5e4df; - --border-strong: #d4d2cb; - --text: #0b0b0b; - --text-2: #52514e; - --text-3: #8a8881; - --accent: #4f46e5; - --accent-soft: rgba(79, 70, 229, 0.08); - --accent-line: rgba(79, 70, 229, 0.35); - --s1: #2a78d6; /* slot 1, blue: class No, model bars */ - --s2: #eb6834; /* slot 2, orange: the positive class (risk) */ - --neutral: #a9a69e; /* baseline, de-emphasized */ - --track: #ecebe6; - --good: #0f7b3e; - --good-soft: rgba(15, 123, 62, 0.1); - --bad: #b42318; - --bad-soft: rgba(180, 35, 24, 0.08); - --shadow: 0 1px 2px rgba(20, 18, 12, 0.05); - --radius: 14px; + --bg: #f7f6f2; + --surface: #fdfcfa; + --surface-2: #f0eee8; + --border: #e4e1d9; + --border-strong: #cfcbc1; + --text: #191817; + --text-2: #57544e; + --text-3: #8f8b83; + --accent: #191817; + --accent-soft: rgba(25, 24, 23, 0.05); + --accent-line: rgba(25, 24, 23, 0.22); + --s1: #2a6cb0; /* slot 1, blue: class No, model bars */ + --s2: #c9532a; /* slot 2, vermilion: the positive class (risk) */ + --neutral: #b3afa6; /* baseline, de-emphasized */ + --track: #e9e6df; + --good: #2f7a4b; + --good-soft: rgba(47, 122, 75, 0.09); + --bad: #b3361f; + --bad-soft: rgba(179, 54, 31, 0.07); + --shadow: none; + --radius: 10px; --mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace; --sans: Inter, ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif; + --serif: Newsreader, "Iowan Old Style", Georgia, serif; } @media (prefers-color-scheme: dark) { :root:where(:not([data-theme="light"])) { color-scheme: dark; - --bg: #121211; - --surface: #1a1a19; - --surface-2: #20201e; - --border: #2b2a27; - --border-strong: #3a3935; - --text: #f0efec; - --text-2: #c3c2b7; - --text-3: #8c8a82; - --accent: #8b85ff; - --accent-soft: rgba(139, 133, 255, 0.12); - --accent-line: rgba(139, 133, 255, 0.4); - --s1: #3987e5; - --s2: #d95926; - --neutral: #6b6963; - --track: #262523; - --good: #4ade80; - --good-soft: rgba(74, 222, 128, 0.12); - --bad: #f87171; - --bad-soft: rgba(248, 113, 113, 0.1); + --bg: #131312; + --surface: #191918; + --surface-2: #201f1d; + --border: #2c2b28; + --border-strong: #3b3a36; + --text: #ecebe6; + --text-2: #b9b6ad; + --text-3: #807d76; + --accent: #ecebe6; + --accent-soft: rgba(236, 235, 230, 0.06); + --accent-line: rgba(236, 235, 230, 0.25); + --s1: #4a8fdc; + --s2: #e0683a; + --neutral: #5f5d57; + --track: #2a2926; + --good: #5fbf85; + --good-soft: rgba(95, 191, 133, 0.1); + --bad: #ef7b62; + --bad-soft: rgba(239, 123, 98, 0.09); --shadow: none; } } :root[data-theme="dark"] { color-scheme: dark; - --bg: #121211; - --surface: #1a1a19; - --surface-2: #20201e; - --border: #2b2a27; - --border-strong: #3a3935; - --text: #f0efec; - --text-2: #c3c2b7; - --text-3: #8c8a82; - --accent: #8b85ff; - --accent-soft: rgba(139, 133, 255, 0.12); - --accent-line: rgba(139, 133, 255, 0.4); - --s1: #3987e5; - --s2: #d95926; - --neutral: #6b6963; - --track: #262523; - --good: #4ade80; - --good-soft: rgba(74, 222, 128, 0.12); - --bad: #f87171; - --bad-soft: rgba(248, 113, 113, 0.1); + --bg: #131312; + --surface: #191918; + --surface-2: #201f1d; + --border: #2c2b28; + --border-strong: #3b3a36; + --text: #ecebe6; + --text-2: #b9b6ad; + --text-3: #807d76; + --accent: #ecebe6; + --accent-soft: rgba(236, 235, 230, 0.06); + --accent-line: rgba(236, 235, 230, 0.25); + --s1: #4a8fdc; + --s2: #e0683a; + --neutral: #5f5d57; + --track: #2a2926; + --good: #5fbf85; + --good-soft: rgba(95, 191, 133, 0.1); + --bad: #ef7b62; + --bad-soft: rgba(239, 123, 98, 0.09); --shadow: none; } @@ -94,28 +95,12 @@ code, .mono { font-family: var(--mono); font-size: 0.9em; } border-bottom: 1px solid var(--border); } .brand { display: flex; align-items: center; gap: 10px; } -.logo { display: grid; place-items: center; width: 28px; height: 28px; border-radius: 8px; background: var(--accent); color: #fff; } .brand-name { font-weight: 650; letter-spacing: -0.01em; } -.brand-sep { width: 1px; height: 16px; background: var(--border-strong); } .brand-sub { color: var(--text-2); } .top-right { display: flex; align-items: center; gap: 8px; } -.pill { - display: inline-flex; align-items: center; gap: 8px; height: 30px; padding: 0 11px; - font-size: 12.5px; color: var(--text-2); border: 1px solid var(--border); border-radius: 999px; background: var(--surface); -} -.icon-btn { width: 30px; padding: 0; justify-content: center; cursor: pointer; } -.icon-btn:hover { color: var(--text); border-color: var(--border-strong); } .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--good); } .dot.off { background: var(--bad); } -/* ---- layout ------------------------------------------------------------ */ -.layout { display: grid; grid-template-columns: 340px minmax(0, 1fr); gap: 24px; max-width: 1320px; margin: 0 auto; padding: 24px; } -.sidebar { display: flex; flex-direction: column; gap: 12px; position: sticky; top: 80px; align-self: start; max-height: calc(100vh - 96px); overflow: auto; } -.panel { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 14px 16px; box-shadow: var(--shadow); } -.panel h2 { margin: 0 0 10px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-3); } -.panel-head { display: flex; align-items: baseline; justify-content: space-between; } -.link { background: none; border: 0; padding: 0; color: var(--accent); cursor: pointer; font-size: 12.5px; } - /* account */ .account form { display: flex; gap: 8px; } .account input { @@ -130,15 +115,6 @@ code, .mono { font-family: var(--mono); font-size: 0.9em; } .signed .link { margin-left: auto; } textarea[readonly] { color: var(--text-2); cursor: default; } -/* data */ -.dataset { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; } -.ds { padding: 10px 12px; border-radius: 10px; background: var(--surface-2); border: 1px solid var(--border); } -.ds .v { font-size: 20px; font-weight: 650; letter-spacing: -0.02em; font-variant-numeric: tabular-nums; } -.ds .k { font-size: 12px; color: var(--text-2); } -.ds .f { font: 11px var(--mono); color: var(--text-3); margin-top: 6px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -.ds .split { margin-top: 8px; } -.locked { display: inline-flex; align-items: center; gap: 4px; font-size: 11px; color: var(--text-2); margin-top: 8px; } - /* agents */ .agents { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; } .agent { @@ -156,10 +132,6 @@ textarea[readonly] { color: var(--text-2); cursor: default; } .agent-tools { display: flex; gap: 4px; margin-top: 8px; } .chip-sm { display: inline-flex; align-items: center; height: 18px; padding: 0 6px; border-radius: 5px; font: 500 10.5px var(--mono); color: var(--text-2); background: var(--surface); border: 1px solid var(--border); } -/* task: the prompt's steps as icons; the prompt itself folded */ -.task { display: grid; grid-auto-flow: column; grid-auto-columns: minmax(0, 1fr); gap: 4px; } -.task-step { display: flex; flex-direction: column; align-items: center; gap: 5px; font-size: 11px; color: var(--text-2); cursor: default; } -.task-step .tool-icon { width: 34px; height: 34px; border-radius: 10px; } .prompt-box { margin-top: 12px; } .prompt-box summary { font-size: 12px; color: var(--text-3); cursor: pointer; list-style: none; } .prompt-box summary::-webkit-details-marker { display: none; } @@ -174,7 +146,6 @@ textarea { } textarea:focus { border-color: var(--accent-line); box-shadow: 0 0 0 3px var(--accent-soft); } -.actions { display: grid; gap: 8px; } .btn { display: inline-flex; align-items: center; justify-content: center; gap: 8px; height: 40px; padding: 0 16px; border-radius: 10px; cursor: pointer; border: 1px solid var(--border-strong); background: var(--surface); font-weight: 550; @@ -183,27 +154,12 @@ textarea:focus { border-color: var(--accent-line); box-shadow: 0 0 0 3px var(--a .btn:hover:not(:disabled) { background: var(--surface-2); } .btn:active:not(:disabled) { transform: translateY(1px); } .btn:disabled { opacity: 0.45; cursor: not-allowed; } -.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; } -.btn.primary:hover:not(:disabled) { background: color-mix(in srgb, var(--accent) 88%, #000); } -.btn:focus-visible, .agent:focus-visible, .link:focus-visible, .icon-btn:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.btn.primary { background: var(--accent); border-color: var(--accent); color: var(--bg); } +.btn.primary:hover:not(:disabled) { background: color-mix(in srgb, var(--accent) 85%, var(--bg)); } +.btn:focus-visible, .agent:focus-visible, .link:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } .notice { margin: 0; font-size: 12.5px; color: var(--bad); } -/* ---- empty state: the pipeline --------------------------------------- */ -.empty { background: var(--surface); border: 1px solid var(--border); border-radius: 18px; padding: 52px 48px 44px; box-shadow: var(--shadow); } -.empty h1 { margin: 0 0 36px; font-size: 30px; line-height: 1.15; letter-spacing: -0.025em; font-weight: 680; } -.flow { display: grid; grid-template-columns: 1fr auto 1fr auto 1.25fr auto 1fr; align-items: center; gap: 10px; } -.node { padding: 18px 14px; border-radius: 14px; border: 1px solid var(--border); background: var(--surface-2); text-align: center; } -.node.hub { border-color: var(--accent-line); background: var(--accent-soft); } -.node-icon { width: 40px; height: 40px; margin: 0 auto 10px; border-radius: 11px; display: grid; place-items: center; background: var(--surface); border: 1px solid var(--border); color: var(--text-2); } -.node.hub .node-icon { background: var(--accent); border-color: var(--accent); color: #fff; } -.node-title { font-weight: 600; font-size: 13.5px; } -.node-sub { font-size: 12px; color: var(--text-3); margin-top: 2px; } -.arrow { color: var(--text-3); } -.models { display: flex; justify-content: center; gap: 4px; margin-top: 8px; } -.models span { font: 500 10.5px var(--mono); padding: 1px 6px; border-radius: 5px; background: var(--surface); border: 1px solid var(--border); color: var(--text-2); } - /* ---- run ----------------------------------------------------------------- */ -.stage { min-width: 0; } .run-head { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 12px; } .run-title { font-size: 20px; font-weight: 680; letter-spacing: -0.015em; } .run-meta { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 6px; } @@ -359,10 +315,6 @@ pre.json { margin: 8px 0 0; padding: 10px; max-height: 280px; overflow: auto; fo .report code { background: var(--surface-2); border: 1px solid var(--border); border-radius: 5px; padding: 0 4px; } @media (max-width: 1040px) { - .layout { grid-template-columns: 1fr; } - .sidebar { position: static; max-height: none; } - .flow { grid-template-columns: 1fr; } - .arrow { transform: rotate(90deg); justify-self: center; } .scores { grid-template-columns: 1fr; } .bar-row { grid-template-columns: 110px minmax(0, 1fr) 50px; } .axis-note { margin-left: 122px; padding-right: 62px; } diff --git a/tests/unit/test_agent_demo_cleaning.py b/tests/unit/test_agent_demo_cleaning.py new file mode 100644 index 0000000..8c2bc6a --- /dev/null +++ b/tests/unit/test_agent_demo_cleaning.py @@ -0,0 +1,128 @@ +"""The agent demo page's "messy -> tidy" view (examples/agent_demo/ui/ +cleaning.py): a raw export set beside the clean table an agent wrote, +and the words the page uses for each scenario. No agent, no model.""" + +import json +import sys +from pathlib import Path + +import pytest + +pd = pytest.importorskip("pandas") + +DEMO = Path(__file__).resolve().parents[2] / "examples" / "agent_demo" +sys.path[:0] = [str(DEMO), str(DEMO / "ui")] +import cleaning # noqa: E402 +import scenario as scenarios # noqa: E402 + +RAW = """Policy ID,Prov,Premium,Age,Status +A-1,ON,"$1,200.50",40,Renewed +A-2,ont,1300,999,RENEWED +A-2,ont,1300,999,RENEWED +A-3,Ontario,CAD 900,n/a,cxl +A-4, on,800,52,Cancelled +A-5,QC,700,33,Renewed +TEST-1,ON,1,1,Renewed +TOTAL,,5001,, +""" +CLEAN = """policy_id,province,premium,age,cancelled +A-1,ON,1200.5,40,No +A-2,ON,1300,,No +A-3,ON,900,,Yes +A-4,ON,800,52,Yes +A-5,QC,700,33,No +""" +COLUMNS = { + "Prov": "province", + "Premium": "premium", + "Age": "age", + "Status": "cancelled", + "Missing": "not_there", +} + + +@pytest.fixture +def run(tmp_path): + (tmp_path / "raw.csv").write_text(RAW) + (tmp_path / "clean.csv").write_text(CLEAN) + return tmp_path + + +def view(run): + return cleaning.summarize( + run / "raw.csv", run / "clean.csv", "Policy ID", "policy_id", COLUMNS + ) + + +def test_rows_removed_are_split_into_duplicates_and_non_records(run): + v = view(run) + assert (v["rows_before"], v["rows_after"]) == (8, 5) + assert v["removed"] == [ + {"n": 1, "what": "duplicate rows"}, + {"n": 2, "what": "rows that aren't records (tests, totals)"}, + ] + + +def test_categories_report_spellings_and_the_most_spelled_value(run): + by = {c["clean"]: c for c in view(run)["columns"]} + assert "not_there" not in by # a mapped column the files lack is skipped + prov = by["province"] + assert (prov["kind"], prov["before"], prov["after"]) == ("category", 5, 2) + assert prov["example"]["value"] == "ON" + assert set(prov["example"]["spellings"]) == {"ON", "ont", "Ontario", " on"} + status = by["cancelled"] + assert (status["before"], status["after"]) == (4, 2) + + +def test_numbers_report_text_converted_and_impossible_values_emptied(run): + by = {c["clean"]: c for c in view(run)["columns"]} + assert by["premium"] == { + "raw": "Premium", + "clean": "premium", + "kind": "numeric", + "before": 2, # "$1,200.50" and "CAD 900" + "after": 0, + } + assert (by["age"]["before"], by["age"]["after"]) == (0, 1) # 999 emptied + + +def test_the_sample_puts_the_most_fixed_rows_first_and_shows_cells_plainly(run): + sample = view(run)["sample"] + assert sample["columns"][0] == ["Prov", "province"] + first = sample["rows"][0] + # A-3: province, premium, age (n/a) and status all fixed. + assert first == [["Ontario", "ON"], ["CAD 900", "900"], ["n/a", ""], ["cxl", "Yes"]] + # 1200.5 shows as written, not as 1200.50 or 1200.5000. + a1 = next( + r for r in sample["rows"] if r[0] == ["ON", "ON"] and r[1][0].startswith("$") + ) + assert a1[1] == ["$1,200.50", "1200.5"] + assert len(sample["rows"]) == cleaning.SAMPLE_ROWS + + +def test_a_run_prefers_its_saved_view_then_its_files(run): + insurance = scenarios.load("insurance") + assert cleaning.for_run(run, insurance) is None # not the scenario's files + (run / cleaning.SAVED).write_text(json.dumps({"saved": True})) + assert cleaning.for_run(run, insurance) == {"saved": True} + + +def test_a_run_whose_agent_wrote_a_different_table_has_no_view(tmp_path): + insurance = scenarios.load("insurance") + task = insurance.task + (tmp_path / task.HISTORY).write_text(RAW.replace("Policy ID", insurance.raw_id)) + (tmp_path / task.CLEAN_HISTORY).write_text("something,else\n1,2\n") + assert cleaning.for_run(tmp_path, insurance) is None + + +@pytest.mark.parametrize("name", scenarios.names()) +def test_every_scenario_tells_the_page_its_question_and_column_names(name): + labels = scenarios.load(name).task.LABELS + assert labels["question"].strip() and labels["audience"] and labels["org"] + names = labels["column_names"] + assert all(1 <= len(v) <= 2 for v in names.values()) + assert all("{}" in v[1] for v in names.values() if len(v) == 2) + task = scenarios.load(name).task + assert task.ID in names and task.TARGET in names + for clean in getattr(task, "RAW_COLUMNS", {}).values(): + assert clean in names # the tidy view names every column it shows diff --git a/tests/unit/test_agent_demo_scenarios.py b/tests/unit/test_agent_demo_scenarios.py index f40f07d..9f35e52 100644 --- a/tests/unit/test_agent_demo_scenarios.py +++ b/tests/unit/test_agent_demo_scenarios.py @@ -144,7 +144,7 @@ def fake_agents(tmp_path): return bin_dir, server -def _run(name, agent, tmp_path, fake_agents): +def _run(name, agent, tmp_path, fake_agents, **extra): bin_dir, server = fake_agents task = scenarios.load(name).task workspace = tmp_path / "workspace" @@ -159,6 +159,7 @@ def _run(name, agent, tmp_path, fake_agents): RUN_DIR=str(run_dir), TABCTX_MCP_BIN=str(server), TABCTX_API_KEY="vp_test", + **extra, ) subprocess.run( [str(DEMO / "run.sh"), agent], env=env, check=True, capture_output=True @@ -192,6 +193,18 @@ def test_claude_gets_bash_only_inside_its_sandbox(tmp_path, fake_agents): assert "Bash" not in _arg(args, "--tools") and "--settings" not in args +def test_hosted_claude_runs_bash_as_its_run_user_without_the_sandbox( + tmp_path, fake_agents +): + run_dir = _run("insurance", "claude", tmp_path, fake_agents, CLAUDE_SANDBOX="off") + args = (run_dir / "args.txt").read_text().splitlines() + allowed = args[args.index("--allowedTools") + 1 : args.index("--output-format")] + assert "Bash" in allowed + settings = json.loads(_arg(args, "--settings")) + assert "sandbox" not in settings + assert settings["permissions"]["deny"] == [f"Read(/{INSURANCE.answers}/**)"] + + def test_gemini_keeps_its_shell_only_when_the_task_runs_code(tmp_path, fake_agents): run_dir = _run("insurance", "gemini", tmp_path, fake_agents) settings = json.loads((run_dir / ".gemini" / "settings.json").read_text()) From a86f1d49c5339fa5af4b7ef3b7de53ce170d299a Mon Sep 17 00:00:00 2001 From: Amrit Krishnan Date: Wed, 18 Nov 2026 10:10:05 -0500 Subject: [PATCH 2/2] Agent demo: fixes from review of the brief and the hosted setup Hosting: - The insurance data's generator is public and its seed was fixed, so an agent with a shell could rebuild the withheld outcomes. The hosted image now builds its data from a seed drawn at build time (DEMO_SEED) and kept nowhere. - A finished run's folder goes back to root, so the next run given its uid (another visitor's) can't read it. - Codex's label says "shell" where it runs without its sandbox, as Claude's now does. Cleaning view (the agent's clean table is untrusted): - A saved cleaning.json is read only for showcase runs, and bad JSON there gives no view instead of a 500. - Only plain files inside the run's folder under 50 MB are read, never a link, and each version is summarized once. - True/False and 1/0 columns are categories; blanks are not a merged value ("nan"); columns left empty are skipped instead of hiding the whole view; duplicate ids in the clean table don't repeat sample rows. The page: - One event that fails to draw no longer stops a replay (and the work panel still draws it); tool results of unexpected shape are guarded. - A lost connection says so and offers the way back; "Watch again" replays what was shown. - "Sign in to ask" no longer closes the popover it opens; picking an agent keeps it open; it takes focus and Escape returns it. - Steps the agent never answered say so instead of looking done. - Regression tasks: the best model for lower-is-better metrics, predicted values instead of risk bars. - A cleaning fetch from an earlier run can't land in a new one. - Muted text passes WCAG AA in both themes; the switch keeps its name on phones; duplicated rules are in one stylesheet. Co-Authored-By: Claude Opus 5.5 --- examples/agent_demo/Dockerfile | 5 +- examples/agent_demo/README.md | 5 + .../scenarios/insurance/prepare_data.py | 10 +- examples/agent_demo/ui/app.py | 4 +- examples/agent_demo/ui/cleaning.py | 84 ++++++++++++--- examples/agent_demo/ui/runner.py | 12 ++- examples/agent_demo/ui/static/answer.css | 18 ++-- examples/agent_demo/ui/static/answer.js | 86 ++++++++++----- examples/agent_demo/ui/static/app.js | 55 +++++++--- examples/agent_demo/ui/static/index.html | 2 +- examples/agent_demo/ui/static/style.css | 26 +++-- tests/unit/test_agent_demo_cleaning.py | 101 +++++++++++++++++- tests/unit/test_agent_demo_scenarios.py | 31 ++++++ 13 files changed, 347 insertions(+), 92 deletions(-) diff --git a/examples/agent_demo/Dockerfile b/examples/agent_demo/Dockerfile index 7291a3a..f341ecb 100644 --- a/examples/agent_demo/Dockerfile +++ b/examples/agent_demo/Dockerfile @@ -43,7 +43,10 @@ COPY examples/agent_demo ./examples/agent_demo # in the per-run user below (CLAUDE_SANDBOX=off). ARG DEMO_SCENARIO=insurance ENV DEMO_SCENARIO=$DEMO_SCENARIO -RUN python examples/agent_demo/prepare_data.py \ +# The synthetic data's generator is public, so its seed is drawn here and +# kept nowhere (not in the image, its layers or its history): with a known +# seed an agent with a shell could rebuild the outcomes. +RUN DEMO_SEED=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ') python examples/agent_demo/prepare_data.py \ && rm -rf examples/agent_demo/runs \ && chmod -R go-rwx examples/agent_demo/answers diff --git a/examples/agent_demo/README.md b/examples/agent_demo/README.md index b23f7fa..1b8d579 100644 --- a/examples/agent_demo/README.md +++ b/examples/agent_demo/README.md @@ -77,6 +77,11 @@ steps gives each agent its shell: Claude Code's Bash runs in its sandbox, Codex in its own, and Gemini CLI and pi without one (see the tools by agent, below). +The data comes from a fixed seed, so runs are repeatable. The generator +is public, so with that seed an agent with a shell could rebuild the +withheld outcomes: the hosted image draws a seed at build time +(`DEMO_SEED`) and keeps it nowhere. + ```bash export DEMO_SCENARIO=insurance uv run --extra mcp python examples/agent_demo/prepare_data.py diff --git a/examples/agent_demo/scenarios/insurance/prepare_data.py b/examples/agent_demo/scenarios/insurance/prepare_data.py index 8aa3cce..6c6245c 100644 --- a/examples/agent_demo/scenarios/insurance/prepare_data.py +++ b/examples/agent_demo/scenarios/insurance/prepare_data.py @@ -28,6 +28,7 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING import numpy as np @@ -317,8 +318,13 @@ def build( def main(scenario: Scenario, **sizes: int) -> None: - """Write the scenario's data; sizes are build()'s row counts.""" - history, current, answers = build(**sizes) + """Write the scenario's data; sizes are build()'s row counts. + + DEMO_SEED replaces the fixed seed. The hosted image draws one at build + time and keeps it nowhere: this generator is public, so with the fixed + seed an agent with a shell could rebuild the withheld outcomes.""" + seed = int(os.environ.get("DEMO_SEED") or SEED) + history, current, answers = build(seed, **sizes) scenario.workspace.mkdir(parents=True, exist_ok=True) scenario.answers.mkdir(parents=True, exist_ok=True) history.to_csv(scenario.workspace / task.HISTORY, index=False) diff --git a/examples/agent_demo/ui/app.py b/examples/agent_demo/ui/app.py index a966ae1..4c29dda 100644 --- a/examples/agent_demo/ui/app.py +++ b/examples/agent_demo/ui/app.py @@ -342,7 +342,9 @@ async def events( @app.get("/api/runs/{agent}/cleaning") def cleaning(agent: str, request: Request, source: str = "auto") -> dict: """The run's raw export beside the clean table its agent wrote.""" - view = cleaning_of(_run_for(agent, request, source).dir, SCENARIO) + run = _run_for(agent, request, source) + showcase = HOSTED and run.dir.resolve().is_relative_to(SHOWCASE.resolve()) + view = cleaning_of(run.dir, SCENARIO, saved=showcase) if view is None: raise HTTPException(404, "no cleaned table in this run") return view diff --git a/examples/agent_demo/ui/cleaning.py b/examples/agent_demo/ui/cleaning.py index 11ac40a..2d0bc1a 100644 --- a/examples/agent_demo/ui/cleaning.py +++ b/examples/agent_demo/ui/cleaning.py @@ -13,12 +13,18 @@ with the most fixed cells. Which raw column became which clean one is the scenario's call (task.RAW_COLUMNS): headers alone don't say that "Prov" is `province`. + +The clean table is the agent's, so untrusted: for_run reads it only as a +plain file inside the run's folder and under a size cap, never a saved +view the agent could have written, and a table that isn't what the +prompt asked for gives no view rather than an error. """ from __future__ import annotations import json import math +from functools import lru_cache from pathlib import Path import pandas as pd @@ -26,10 +32,15 @@ SAMPLE_ROWS = 5 CLEAN = "clean:" SPELLINGS_SHOWN = 5 +MAX_BYTES = 50_000_000 # a larger file isn't a table the page sets side by side def _numeric(clean: pd.Series) -> bool: - return pd.api.types.is_numeric_dtype(clean) + """A number column; yes/no written as True/False or 1/0 is a category.""" + if pd.api.types.is_bool_dtype(clean): + return False + numbers = clean.dropna() + return pd.api.types.is_numeric_dtype(clean) and not numbers.isin([0, 1]).all() def _show(value) -> str: @@ -67,10 +78,16 @@ def summarize( {"n": duplicates, "what": "duplicate rows"}, {"n": extra, "what": "rows that aren't records (tests, totals)"}, ] - pairs = [(r, c) for r, c in columns.items() if r in raw and c in clean] - # Clean columns prefixed, so a raw header can't collide with one. + # Columns the agent left empty say nothing about what it fixed. + pairs = [ + (r, c) + for r, c in columns.items() + if r in raw and c in clean and clean[c].notna().any() + ] + # Clean columns prefixed, so a raw header can't collide with one; one + # row per id, should the agent have left duplicates. joined = raw.drop_duplicates(raw_id).merge( - clean.astype({clean_id: str}).add_prefix(CLEAN), + clean.drop_duplicates(clean_id).astype({clean_id: str}).add_prefix(CLEAN), left_on=raw_id, right_on=CLEAN + clean_id, ) @@ -91,7 +108,12 @@ def summarize( } ) continue - spellings = before.groupby(after.astype(str)).unique() + # Spellings of a value, not of a blank ("N/A", "-" became empty). + known = after.notna() + if not known.any(): + continue + before, after = before[known], after[known].astype(str) + spellings = before.groupby(after).unique() top = max(spellings.index, key=lambda v: len(spellings[v])) out.append( { @@ -135,19 +157,49 @@ def _sample(joined: pd.DataFrame, pairs: list[tuple[str, str]]) -> dict: SAVED = "cleaning.json" # a showcase run's copy (make_showcase.py) -def for_run(run_dir: Path, scenario) -> dict | None: - """A run's cleaning view: its saved copy, else from its own files, else - None (no cleaning step, or the agent hasn't written the clean table).""" - saved = run_dir / SAVED - if saved.is_file(): - return json.loads(saved.read_text()) +def for_run(run_dir: Path, scenario, saved: bool = False) -> dict | None: + """A run's cleaning view, else None (no cleaning step, or no clean + table the page can read). saved: the run is a showcase run, whose saved + copy is read first; a visitor's agent could write that file itself.""" + if saved and (run_dir / SAVED).is_file(): + try: + view = json.loads((run_dir / SAVED).read_text()) + except ValueError: + return None + return view if isinstance(view, dict) else None task = scenario.task columns = getattr(task, "RAW_COLUMNS", None) - clean = run_dir / getattr(task, "CLEAN_HISTORY", "") - raw = run_dir / task.HISTORY - if not columns or not clean.is_file() or not raw.is_file(): + raw = _readable(run_dir, task.HISTORY) + clean = _readable(run_dir, getattr(task, "CLEAN_HISTORY", None)) + if not columns or raw is None or clean is None: + return None + return _cached(raw, clean, scenario.raw_id, task.ID, tuple(columns.items())) + + +def _readable(run_dir: Path, name: str | None) -> tuple[str, int, int] | None: + """(path, size, mtime) of a plain file in run_dir under the size cap. The + server may read as root: an agent's link to another file isn't followed.""" + if not name: + return None + path = run_dir / name + try: + if path.is_symlink() or not path.is_file(): + return None + if not path.resolve().is_relative_to(run_dir.resolve()): + return None + stat = path.stat() + except OSError: return None + if stat.st_size > MAX_BYTES: + return None + return str(path), stat.st_size, stat.st_mtime_ns + + +@lru_cache(maxsize=32) +def _cached(raw, clean, raw_id, clean_id, columns) -> dict | None: + """summarize(), once per version of the two files (their size and time + are part of the key).""" try: - return summarize(raw, clean, scenario.raw_id, task.ID, columns) - except (KeyError, ValueError, pd.errors.ParserError): + return summarize(Path(raw[0]), Path(clean[0]), raw_id, clean_id, dict(columns)) + except (KeyError, ValueError, TypeError, pd.errors.ParserError): return None # the agent's table isn't what the prompt asked for diff --git a/examples/agent_demo/ui/runner.py b/examples/agent_demo/ui/runner.py index 4da1e36..28eb5c7 100644 --- a/examples/agent_demo/ui/runner.py +++ b/examples/agent_demo/ui/runner.py @@ -51,7 +51,11 @@ "label": "Codex", "binary": "codex", "llm": "gpt-5 via Vector gateway", - "tools": "MCP · sandboxed shell", + "tools": ( + "MCP · shell" + if os.environ.get("CODEX_SANDBOX") == "danger-full-access" + else "MCP · sandboxed shell" + ), }, "gemini": { "label": "Gemini CLI", @@ -212,6 +216,10 @@ def _watch(self, run: Run) -> None: del run.log[:-50] run.process.wait() timer.cancel() + if run.uid is not None: + # Back to root: the next run given this uid (another visitor's) + # can't read this one's folder. + _own(run.dir.parent, 0) def _stop(self, run: Run, reason: str) -> None: if run.running and run.process: @@ -225,7 +233,7 @@ def _stop(self, run: Run, reason: str) -> None: def _own(folder: Path, uid: int) -> None: """Give folder and everything in it to uid, readable by it alone (the - server, as root, still reads it).""" + server, as root, still reads it). uid 0 takes it back once a run ends.""" for root, dirs, files in os.walk(folder): for name in (*dirs, *files): os.lchown(os.path.join(root, name), uid, uid) diff --git a/examples/agent_demo/ui/static/answer.css b/examples/agent_demo/ui/static/answer.css index 7c59898..1d70339 100644 --- a/examples/agent_demo/ui/static/answer.css +++ b/examples/agent_demo/ui/static/answer.css @@ -9,17 +9,8 @@ body { background: var(--bg); } body.show-work .layout { grid-template-columns: minmax(0, 760px) minmax(0, 560px); } body.show-work .work { display: block; position: sticky; top: 72px; align-self: start; max-height: calc(100vh - 88px); overflow: auto; margin-top: 48px; padding-left: 28px; border-left: 1px solid var(--border); } .work .steps { grid-template-columns: repeat(auto-fit, minmax(56px, 1fr)); } -.work .prompt-box { margin: 0 0 16px; } -.work .prompt-box summary { font-size: 12.5px; color: var(--text-3); cursor: pointer; } -.work .prompt-box textarea { margin-top: 8px; } -.work .prompt-box .link { margin-top: 6px; } .work .run-title { font: 500 20px/1.2 var(--serif); } -.topbar { height: 56px; padding: 0 28px; background: color-mix(in srgb, var(--bg) 88%, transparent); border-bottom: 1px solid transparent; } -.brand { gap: 12px; } -.brand-name { font: 600 17px var(--serif); letter-spacing: -0.005em; } -.brand-sub { font-size: 13px; color: var(--text-3); } -.top-right { gap: 18px; } .text-btn { display: inline-flex; align-items: center; gap: 6px; background: none; border: 0; padding: 4px 0; font-size: 13px; color: var(--text-2); cursor: pointer; } .text-btn:hover { color: var(--text); } .text-btn:focus-visible, .link-btn:focus-visible { outline: 2px solid var(--accent-line); outline-offset: 3px; border-radius: 4px; } @@ -68,6 +59,8 @@ body.show-work .work { display: block; position: sticky; top: 72px; align-self: .progress li { font-size: 12px; color: var(--text-3); padding-top: 10px; border-top: 2px solid var(--track); transition: color 0.3s, border-color 0.3s; } .progress li[data-state="working"] { color: var(--text); border-top-color: var(--text); animation: breathe 1.6s ease-in-out infinite; } .progress li[data-state="done"] { color: var(--text); border-top-color: var(--text); } +.progress li[data-state="missing"] { border-top-style: dashed; border-top-color: var(--border-strong); } +.sec[data-state="missing"] h2 { color: var(--text-3); } @keyframes breathe { 50% { border-top-color: var(--border-strong); } } .status-line { display: flex; align-items: center; gap: 10px; margin: 16px 0 0; font-size: 13.5px; color: var(--text-2); } .pulse-dot { width: 6px; height: 6px; border-radius: 50%; background: var(--text); animation: pulse 1.2s ease-in-out infinite; } @@ -79,13 +72,13 @@ body.show-work .work { display: block; position: sticky; top: 72px; align-self: .lede { margin: 12px 0 0; font-size: 17px; line-height: 1.6; color: var(--text-2); max-width: 660px; } .lede b { color: var(--text); font-weight: 600; } .lede.doing { color: var(--text-3); font-style: italic; } -.viz:not(:empty) { margin-top: 26px; } +.sec .viz:not(:empty) { margin-top: 26px; } .viz-foot { display: flex; justify-content: space-between; align-items: baseline; margin-top: 10px; font-size: 12.5px; color: var(--text-3); } .viz-note { margin: 14px 0 0; font-size: 13px; color: var(--text-3); } @keyframes arrive { from { opacity: 0; transform: translateY(8px); } to { opacity: 1; transform: none; } } /* 01 · tidy: the raw cells underlined like a misspelling, then cleaned */ -.tidy { overflow-x: auto; } +.tidy-scroll { overflow-x: auto; } .tidy-table { width: 100%; border-collapse: collapse; font-size: 14px; } .tidy-table th, .tidy-table td { padding: 9px 14px 9px 0; text-align: left; white-space: nowrap; border-bottom: 1px solid var(--border); background: none; } .tidy-table th { font-size: 11.5px; font-weight: 500; color: var(--text-3); text-transform: none; letter-spacing: 0; border-bottom-color: var(--border-strong); } @@ -195,7 +188,8 @@ body.show-work .work { display: block; position: sticky; top: 72px; align-self: } @media (max-width: 720px) { .topbar { padding: 0 16px; } - .brand-sub, .switch-label { display: none; } + .brand-sub { display: none; } + .switch-label { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; } .layout { padding: 0 16px 80px; } .hero { padding-top: 8vh; } .hero .question { font-size: 28px; } diff --git a/examples/agent_demo/ui/static/answer.js b/examples/agent_demo/ui/static/answer.js index 4b3c47c..2f992f5 100644 --- a/examples/agent_demo/ui/static/answer.js +++ b/examples/agent_demo/ui/static/answer.js @@ -37,10 +37,11 @@ const Answer = (() => { if (v === "" || v == null || Number.isNaN(n)) return String(v ?? ""); return Math.abs(n) >= 100 ? fmt(n) : fmt(n, Number.isInteger(n) ? 0 : 1); }; - const value = (c, v) => (v === "" || v == null ? "unknown" : (cols()[c]?.[1] || "{}").replace("{}", num(v))); + const value = (c, v) => (v === "" || v == null ? "unknown" : (cols()[c]?.[1] || "{}").replace("{}", () => num(v))); const said = (c, v) => `${label(c)} ${value(c, v)}`; const changed = (c, v) => `${label(c)} → ${value(c, v)}`; const list = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} and ${xs.at(-1)}`); + const LOWER_IS_BETTER = new Set(["rmse", "mae", "log_loss"]); const words = (n) => ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"][n] ?? fmt(n); // ---- the run ---------------------------------------------------------------- @@ -76,14 +77,17 @@ const Answer = (() => { lede.classList.add("doing"); } } - function write(key, sentence, viz) { + // A step's answer: its sentence, and its visual (replaced, or with + // append: added below what is there, which keeps its animations still). + function write(key, sentence, viz, append = false) { const step = S.steps.get(key); if (!step) return; const lede = step.el.querySelector(".lede"); lede.innerHTML = sentence; lede.dataset.final = "1"; lede.classList.remove("doing"); - if (viz != null) step.el.querySelector(".viz").innerHTML = viz; + const box = step.el.querySelector(".viz"); + if (viz != null) append ? box.insertAdjacentHTML("beforeend", viz) : (box.innerHTML = viz); setStep(key, "done"); } const has = (key) => S.steps.has(key); @@ -130,6 +134,7 @@ const Answer = (() => { // ---- 01 · is the data usable? ------------------------------------------------- async function loadCleaning() { + const run = S; S.cleaned = true; setStep("clean", "working", "Setting the raw rows beside the clean ones"); let v = null; @@ -137,16 +142,17 @@ const Answer = (() => { const res = await fetch(`/api/runs/${S.agent}/cleaning`); if (res.ok) v = await res.json(); } catch {} + if (S !== run) return; // another run started meanwhile if (!v) return write("clean", "The agent cleaned both exports with a script of its own.", ""); - const removed = v.removed.map((x) => `${fmt(x.n)} ${x.what.replace(/ \(.*\)$/, "")}`); - const top = v.columns.filter((c) => c.kind === "category" && c.before > c.after).sort((a, b) => b.before - a.before)[0]; + const removed = (v.removed || []).map((x) => `${fmt(x.n)} ${x.what.replace(/ \(.*\)$/, "")}`); + const top = (v.columns || []).filter((c) => c.kind === "category" && c.before > c.after).sort((a, b) => b.before - a.before)[0]; const sentence = `Not as exported. The file had ${fmt(v.rows_before)} rows; ${fmt(v.rows_after)} are distinct ${esc(S.d.entity || "records")}. ` + `The agent wrote a script that removed ${esc(list(removed))}` + (top ? `, and merged ${fmt(top.before)} spellings of ${esc(lower(label(top.clean)))} into ${fmt(top.after)}.` : "."); write("clean", sentence, cleanViz(v)); const table = S.steps.get("clean").el.querySelector(".tidy-table"); - setTimeout(() => flip(table, true), 2200); // the mess first, then the fix + setTimeout(() => S === run && flip(table, true), 2200); // the mess first, then the fix } function flip(table, tidy) { @@ -173,7 +179,7 @@ const Answer = (() => { →${esc(c.example.value)}`, ) .join(""); - return `
${head}${rows}
+ return `
${head}${rows}
Five of the messiest rows
${merges}
`; } @@ -181,9 +187,10 @@ const Answer = (() => { // ---- 02 · can it be predicted? ------------------------------------------------- function evaluate(r) { const m = r.best_by; - const ok = r.results.filter((x) => !x.error && x.metrics[m] != null); + const ok = (r.results || []).filter((x) => !x.error && x.metrics?.[m] != null); const gbm = ok.find((x) => x.model === "gradient-boosting"); - const tfm = ok.filter((x) => x.model !== "gradient-boosting").sort((a, b) => b.metrics[m] - a.metrics[m])[0]; + const better = LOWER_IS_BETTER.has(m) ? (a, b) => a.metrics[m] - b.metrics[m] : (a, b) => b.metrics[m] - a.metrics[m]; + const tfm = ok.filter((x) => x.model !== "gradient-boosting").sort(better)[0]; if (!tfm) return write("evaluate", "The models couldn't be compared on this table.", ""); if (m !== "roc_auc") { return write("evaluate", `On last year's rows it hadn't seen, the tabular model scores ${fmt(tfm.metrics[m], 3)} (${esc(metricName(m))})${gbm ? `, against ${fmt(gbm.metrics[m], 3)} for the standard approach` : ""}.`, ""); @@ -207,34 +214,38 @@ const Answer = (() => { function predict(r, args = {}) { S.predicted = true; const key = r.ranked_by || ""; + const proba = key.startsWith("p_"); // a chance; else a predicted value const pos = S.d.positive || "Yes"; const mean = r.summary?.mean_probability?.[pos]; - const id = Object.keys(r.top_rows[0] || {})[0]; // the id comes first + const top = r.top_rows || []; + const id = Object.keys(top[0] || {})[0]; // the id comes first const show = (args.show || []).filter((c) => c !== id).slice(0, 2); S.show = show; - const k = r.top_rows.length; const sentence = - (mean != null ? `About ${fmt(Math.round(mean * r.rows))} of the ${fmt(r.rows)} ${esc(S.d.current)} are likely ${esc(S.d.outcome)}. ` : "") + - `These ${words(k)} are the most likely.`; - const rows = r.top_rows + (mean != null ? `About ${fmt(Math.round(mean * r.rows))} of the ${fmt(r.rows)} ${esc(S.d.current)} are likely ${esc(S.d.outcome)}. ` : `The agent scored all ${fmt(r.rows)} ${esc(S.d.current)}. `) + + (top.length ? `These ${words(top.length)} are ${proba ? "the most likely" : "at the top"}.` : ""); + const rows = top .map((row, i) => { const p = Number(row[key]); S.top.set(String(row[id]), row); + const score = proba && Number.isFinite(p) + ? `${pct(p)}` + : `${esc(num(row[key]))}`; return `${i + 1}${esc(row[id])} - ${pct(p)} + ${score} ${esc(show.filter((c) => c in row).map((c) => said(c, row[c])).join(" · "))}`; }) .join(""); - write("predict", sentence, `${rows}
`); + write("predict", sentence, rows ? `${rows}
` : ""); } // ---- 04 · what drives it? ------------------------------------------------------- function explain(r) { - const top = r.importance.filter((i) => i.score_drop > 0).slice(0, 6); + const top = (r.importance || []).filter((i) => i.score_drop > 0).slice(0, 6); if (!top.length) return write("explain", "No single column stands out.", ""); const max = top[0].score_drop; const names = top.map((i) => lower(label(i.column))); - const sentence = `Mostly ${esc(names[0])}${names.length > 2 ? `, then ${esc(list(names.slice(1, 3)))}` : ""}. That is what the predictions lean on, not proof of what causes it.`; + const sentence = `Mostly ${esc(names[0])}${names.length > 1 ? `, then ${esc(list(names.slice(1, 3)))}` : ""}. That is what the predictions lean on, not proof of what causes it.`; const viz = top .map( (i, k) => `
${esc(label(i.column))} @@ -248,7 +259,8 @@ const Answer = (() => { function whatIf(r, args = {}) { const pos = S.d.positive || "Yes"; const res = r.results || []; - if (!res.length || !res[0].probabilities) return; + // The first row is the case as it is; a regression what-if has no chance to move. + if (!has("what_if") || res.length < 2 || res[0].probabilities?.[pos] == null) return; S.people++; const today = res[0].probabilities[pos]; const id = args.base_where ? Object.values(args.base_where).join(", ") : ""; @@ -256,10 +268,14 @@ const Answer = (() => { const facts = known ? (S.show || []).filter((c) => c in known).map((c) => said(c, known[c])).join(" · ") : ""; const at = (p) => `${Math.max(0, Math.min(1, p)) * 100}%`; const tries = res + .map((x, i) => [x, i]) .slice(1) - .map((x, i) => { + .filter(([x]) => x.probabilities?.[pos] != null) + .map(([x, i]) => { const p = x.probabilities[pos]; - const what = Object.entries(x.changes ?? args.rows?.[i + 1] ?? {}).map(([c, v]) => changed(c, v)).join(" + ") || "As it is"; + // The change itself; args.rows are whole rows unless based on one. + const change = x.changes ?? (args.base_where ? args.rows?.[i] : null) ?? {}; + const what = Object.entries(change).map(([c, v]) => changed(c, v)).join(" + ") || "As it is"; const dir = p < today ? "down" : "up"; const lo = Math.min(p, today), hi = Math.max(p, today); return `
${esc(what)} @@ -268,20 +284,40 @@ const Answer = (() => { }) .join(""); const person = `
${esc(id)}${esc(facts)}today ${pct(today)}
${tries}
`; - const viz = S.steps.get("what_if").el.querySelector(".viz").innerHTML + person; - write("what_if", `For the ${words(S.people)} most at risk, the agent changed one thing at a time and asked the model again. These are patterns the model learned, worth a pilot, not proven effects.`, viz); + const who = S.people === 1 ? `one of the ${esc(S.d.entity || "rows")}` : `${words(S.people)} ${esc(S.d.entity || "rows")}`; + write("what_if", `For ${who} most at risk, the agent changed one thing at a time and asked the model again. These are patterns the model learned, worth a pilot, not proven effects.`, person, true); } // ---- the memo and the check ------------------------------------------------------- + // Steps the agent started but never answered say so, rather than look done. + function unanswered(note) { + for (const [k, s] of S.steps) { + const lede = s.el.querySelector(".lede"); + if (s.state === "idle" || (lede && lede.dataset.final)) continue; + if (k === "report" && !lede) continue; + if (lede) { + lede.textContent = note; + lede.classList.remove("doing"); + } + setStep(k, "missing"); + } + } + + function fail(text) { + if (!S) return; + unanswered("The agent didn't get to an answer here."); + $("#agent-status").textContent = text; + } + function outcome(o, stats) { if (!S) return; - for (const [k, s] of S.steps) if (s.state === "working") setStep(k, "done"); if (o.report && has("report")) { const step = S.steps.get("report"); step.el.querySelector(".lede")?.remove(); step.el.querySelector(".viz").innerHTML = memo(o.report); setStep("report", "done"); } + unanswered("The agent didn't get to an answer here."); const secs = stats?.seconds; $("#agent-status").innerHTML = o.report ? `Answered${secs ? ` in ${Math.floor(secs / 60)} min ${Math.round(secs % 60)} s` : ""}, from one question and two raw files.` @@ -321,5 +357,5 @@ const Answer = (() => { flip(t, !t.classList.contains("tidy")); }); - return { reset, event, outcome }; + return { reset, event, outcome, fail }; })(); diff --git a/examples/agent_demo/ui/static/app.js b/examples/agent_demo/ui/static/app.js index f55548f..fc787d3 100644 --- a/examples/agent_demo/ui/static/app.js +++ b/examples/agent_demo/ui/static/app.js @@ -123,9 +123,13 @@ async function init() { state.agent = setup.running || setup.runs.at(-1)?.agent || setup.agents.find((a) => a.available)?.id; renderAgents(); $("#reset-prompt").onclick = () => ($("#prompt").value = setup.prompt); - $("#run").onclick = () => (state.setup.hosted && !state.setup.signed_in ? openSettings(true) : start()); + $("#run").onclick = (e) => { + if (!state.setup.hosted || state.setup.signed_in) return start(); + e.stopPropagation(); // or the click, bubbling on, would close it again + openSettings(true); + }; $("#replay").onclick = () => watch(state.agent, true); - $("#again-replay").onclick = () => watch(state.agent, true); + $("#again-replay").onclick = () => watch(state.shown, true); $("#again-new").onclick = newQuestion; $("#login").onsubmit = signIn; $("#logout").onclick = signOut; @@ -211,10 +215,16 @@ function initPanels() { e.stopPropagation(); openSettings($("#settings").hidden); }; + // A click inside (even on a control it has since re-rendered) keeps it open. document.addEventListener("click", (e) => { - if (!$("#settings").hidden && !e.target.closest("#settings")) openSettings(false); + if (!$("#settings").hidden && e.target.isConnected && !$("#settings").contains(e.target)) openSettings(false); + }); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && !$("#settings").hidden) { + openSettings(false); + $("#settings-btn").focus(); + } }); - document.addEventListener("keydown", (e) => e.key === "Escape" && openSettings(false)); let work = false; try { work = localStorage.getItem("work") === "1"; @@ -232,6 +242,7 @@ function initPanels() { function openSettings(open) { $("#settings").hidden = !open; $("#settings-btn").setAttribute("aria-expanded", String(open)); + if (open) $("#settings").querySelector("input:not([hidden]), button:not(:disabled)")?.focus(); } function newQuestion() { state.source?.close(); @@ -306,7 +317,7 @@ function notice(text) { function watch(agent, replay) { state.source?.close(); clearInterval(state.timer); - Object.assign(state, { calls: new Map(), tabctxCalls: 0, used: new Set(), queue: [], replaying: replay, stats: null }); + Object.assign(state, { calls: new Map(), tabctxCalls: 0, used: new Set(), queue: [], replaying: replay, stats: null, shown: agent }); const info = state.setup.agents.find((a) => a.id === agent); $("#empty").hidden = true; $("#thread").hidden = false; @@ -335,7 +346,13 @@ function watch(agent, replay) { }); source.onerror = () => { source.close(); - if ($("#outcome").hidden && !state.queue.length) setStatus("failed", "Connection lost"); + if (!$("#outcome").hidden) return; + // A replay still draining its queue ends when the queue does. + enqueue(() => { + setStatus("failed", "Connection lost"); + Answer.fail("The connection to the agent was lost before it finished."); + $("#again").hidden = false; + }); }; $("#run").disabled = true; $("#replay").disabled = true; @@ -350,11 +367,16 @@ function enqueue(fn) { function drain() { const fn = state.queue[0]; if (!fn) return; - const delay = fn() === "slow" ? 420 : 60; - setTimeout(() => { - state.queue.shift(); - drain(); - }, delay); + let delay = 60; + try { + if (fn() === "slow") delay = 420; + } finally { + // One event that fails to draw must not stop the rest of the replay. + setTimeout(() => { + state.queue.shift(); + drain(); + }, delay); + } } // An agent's progress note: bold and `code` as the agent wrote them, @@ -364,7 +386,11 @@ function inlineMd(text) { } function handle(ev, info) { - Answer.event(ev); + try { + Answer.event(ev); + } catch (err) { + console.error("the brief could not draw an event", ev, err); // the work panel still does + } const tl = $("#timeline"); if (ev.type === "message") { const el = document.createElement("div"); @@ -629,14 +655,13 @@ function whatIfCard(r, args) { const results = r.results; if (!results.length) return ""; const probs = results[0].probabilities; - const who = ""; // the card's header names the row if (!probs) { - return `${who}
${results.map((x, i) => `
${caseLabel(x, args, i)}
${fmt(x.prediction, 2)}
`).join("")}
`; + return `
${results.map((x, i) => `
${caseLabel(x, args, i)}
${fmt(x.prediction, 2)}
`).join("")}
`; } const classes = Object.keys(probs); const cls = positiveOf(classes); const first = probs[cls]; - return `${who}
P(${esc(cls)})
${results + return `
P(${esc(cls)})
${results .map((x, i) => { const p = x.probabilities[cls]; const d = p - first; diff --git a/examples/agent_demo/ui/static/index.html b/examples/agent_demo/ui/static/index.html index 974efa6..da34bb2 100644 --- a/examples/agent_demo/ui/static/index.html +++ b/examples/agent_demo/ui/static/index.html @@ -24,7 +24,7 @@ diff --git a/examples/agent_demo/ui/static/style.css b/examples/agent_demo/ui/static/style.css index 364f0f2..bb2cc54 100644 --- a/examples/agent_demo/ui/static/style.css +++ b/examples/agent_demo/ui/static/style.css @@ -11,7 +11,7 @@ --border-strong: #cfcbc1; --text: #191817; --text-2: #57544e; - --text-3: #8f8b83; + --text-3: #6f6b64; /* 4.9:1 on --bg, 5.2:1 on --surface */ --accent: #191817; --accent-soft: rgba(25, 24, 23, 0.05); --accent-line: rgba(25, 24, 23, 0.22); @@ -39,7 +39,7 @@ --border-strong: #3b3a36; --text: #ecebe6; --text-2: #b9b6ad; - --text-3: #807d76; + --text-3: #8f8c84; --accent: #ecebe6; --accent-soft: rgba(236, 235, 230, 0.06); --accent-line: rgba(236, 235, 230, 0.25); @@ -63,7 +63,7 @@ --border-strong: #3b3a36; --text: #ecebe6; --text-2: #b9b6ad; - --text-3: #807d76; + --text-3: #8f8c84; --accent: #ecebe6; --accent-soft: rgba(236, 235, 230, 0.06); --accent-line: rgba(236, 235, 230, 0.25); @@ -89,15 +89,14 @@ code, .mono { font-family: var(--mono); font-size: 0.9em; } /* ---- top bar ---------------------------------------------------------- */ .topbar { - position: sticky; top: 0; z-index: 10; height: 56px; padding: 0 24px; + position: sticky; top: 0; z-index: 10; height: 56px; padding: 0 28px; display: flex; align-items: center; justify-content: space-between; - background: color-mix(in srgb, var(--bg) 85%, transparent); backdrop-filter: blur(10px); - border-bottom: 1px solid var(--border); + background: color-mix(in srgb, var(--bg) 88%, transparent); backdrop-filter: blur(10px); } -.brand { display: flex; align-items: center; gap: 10px; } -.brand-name { font-weight: 650; letter-spacing: -0.01em; } -.brand-sub { color: var(--text-2); } -.top-right { display: flex; align-items: center; gap: 8px; } +.brand { display: flex; align-items: center; gap: 12px; } +.brand-name { font: 600 17px var(--serif); letter-spacing: -0.005em; } +.brand-sub { font-size: 13px; color: var(--text-3); } +.top-right { display: flex; align-items: center; gap: 18px; } .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--good); } .dot.off { background: var(--bad); } @@ -132,13 +131,14 @@ textarea[readonly] { color: var(--text-2); cursor: default; } .agent-tools { display: flex; gap: 4px; margin-top: 8px; } .chip-sm { display: inline-flex; align-items: center; height: 18px; padding: 0 6px; border-radius: 5px; font: 500 10.5px var(--mono); color: var(--text-2); background: var(--surface); border: 1px solid var(--border); } -.prompt-box { margin-top: 12px; } -.prompt-box summary { font-size: 12px; color: var(--text-3); cursor: pointer; list-style: none; } +.prompt-box { margin: 0 0 16px; } +.prompt-box summary { font-size: 12.5px; color: var(--text-3); cursor: pointer; list-style: none; } .prompt-box summary::-webkit-details-marker { display: none; } .prompt-box summary::before { content: "▸ "; } .prompt-box[open] summary::before { content: "▾ "; } .prompt-box summary:hover { color: var(--text); } .prompt-box textarea { margin-top: 8px; } +.prompt-box .link { margin-top: 6px; } textarea { width: 100%; resize: vertical; min-height: 110px; font: 12px/1.55 var(--mono); color: var(--text); @@ -270,8 +270,6 @@ td.id { font-family: var(--mono); font-size: 12px; } .delta { display: inline-flex; align-items: center; justify-content: flex-end; gap: 3px; font-size: 12px; font-weight: 600; font-variant-numeric: tabular-nums; } .delta.down { color: var(--good); } .delta.up { color: var(--bad); } -.who { display: inline-flex; align-items: center; gap: 8px; margin-bottom: 12px; font-size: 12.5px; color: var(--text-2); } -.who code { font-size: 12.5px; color: var(--text); padding: 1px 7px; border: 1px solid var(--border); border-radius: 6px; background: var(--surface-2); } details.more { margin-top: 12px; } details.more summary { cursor: pointer; color: var(--text-3); font-size: 12px; list-style: none; display: inline-flex; align-items: center; gap: 4px; } diff --git a/tests/unit/test_agent_demo_cleaning.py b/tests/unit/test_agent_demo_cleaning.py index 8c2bc6a..f61e844 100644 --- a/tests/unit/test_agent_demo_cleaning.py +++ b/tests/unit/test_agent_demo_cleaning.py @@ -2,6 +2,7 @@ cleaning.py): a raw export set beside the clean table an agent wrote, and the words the page uses for each scenario. No agent, no model.""" +import importlib import json import sys from pathlib import Path @@ -100,11 +101,72 @@ def test_the_sample_puts_the_most_fixed_rows_first_and_shows_cells_plainly(run): assert len(sample["rows"]) == cleaning.SAMPLE_ROWS -def test_a_run_prefers_its_saved_view_then_its_files(run): +def test_yes_no_columns_are_categories_whatever_the_agent_wrote_them_as(tmp_path): + raw = "id,Adv\n1,Y\n2,TRUE\n3,0\n4,no\n" + for written in ("True\nFalse\nFalse\nFalse", "1\n1\n0\n0"): + (tmp_path / "raw.csv").write_text(raw) + clean = "id,adv\n" + "\n".join( + f"{i},{v}" for i, v in zip("1234", written.split("\n"), strict=True) + ) + (tmp_path / "clean.csv").write_text(clean + "\n") + v = cleaning.summarize( + tmp_path / "raw.csv", tmp_path / "clean.csv", "id", "id", {"Adv": "adv"} + ) + assert v["columns"][0]["kind"] == "category" + assert (v["columns"][0]["before"], v["columns"][0]["after"]) == (4, 2) + + +def test_blanks_are_not_a_merged_value_and_empty_columns_are_skipped(tmp_path): + (tmp_path / "raw.csv").write_text( + "id,Prov,Note\n1,ON,x\n2,N/A,y\n3,-,z\n4,unknown,w\n5,ont,v\n" + ) + (tmp_path / "clean.csv").write_text( + "id,province,note\n1,ON,\n2,,\n3,,\n4,,\n5,ON,\n" + ) + v = cleaning.summarize( + tmp_path / "raw.csv", + tmp_path / "clean.csv", + "id", + "id", + {"Prov": "province", "Note": "note"}, + ) + assert [c["clean"] for c in v["columns"]] == ["province"] + prov = v["columns"][0] + assert prov["example"] == {"value": "ON", "spellings": ["ON", "ont"]} + assert (prov["before"], prov["after"]) == (2, 1) + + +def test_duplicates_the_agent_left_dont_repeat_rows_in_the_sample(run): + (run / "clean.csv").write_text(CLEAN + "A-3,ON,900,,Yes\n") + rows = view(run)["sample"]["rows"] + assert len(rows) == len({json.dumps(r) for r in rows}) + + +def test_a_saved_view_is_read_only_for_a_showcase_run(run): insurance = scenarios.load("insurance") - assert cleaning.for_run(run, insurance) is None # not the scenario's files (run / cleaning.SAVED).write_text(json.dumps({"saved": True})) - assert cleaning.for_run(run, insurance) == {"saved": True} + # A visitor's agent could write that file: it is never read for them. + assert cleaning.for_run(run, insurance) is None + assert cleaning.for_run(run, insurance, saved=True) == {"saved": True} + for bad in ("not json", "[1, 2]"): + (run / cleaning.SAVED).write_text(bad) + assert cleaning.for_run(run, insurance, saved=True) is None + + +def test_links_and_oversized_files_are_not_read(tmp_path, monkeypatch): + insurance = scenarios.load("insurance") + task = insurance.task + outside = tmp_path / "outside.csv" + outside.write_text(CLEAN.replace("policy_id", task.ID)) + run = tmp_path / "run" + run.mkdir() + (run / task.HISTORY).write_text(RAW.replace("Policy ID", insurance.raw_id)) + (run / task.CLEAN_HISTORY).symlink_to(outside) + assert cleaning.for_run(run, insurance) is None + (run / task.CLEAN_HISTORY).unlink() + (run / task.CLEAN_HISTORY).write_text(outside.read_text()) + monkeypatch.setattr(cleaning, "MAX_BYTES", 10) + assert cleaning.for_run(run, insurance) is None def test_a_run_whose_agent_wrote_a_different_table_has_no_view(tmp_path): @@ -126,3 +188,36 @@ def test_every_scenario_tells_the_page_its_question_and_column_names(name): assert task.ID in names and task.TARGET in names for clean in getattr(task, "RAW_COLUMNS", {}).values(): assert clean in names # the tidy view names every column it shows + + +@pytest.fixture +def page(tmp_path, monkeypatch): + """ui/app.py for the insurance scenario, its runs in tmp_path.""" + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + monkeypatch.setenv("DEMO_SCENARIO", "insurance") + monkeypatch.setenv("DEMO_RUNS", str(tmp_path)) + monkeypatch.delenv("DEMO_HOSTED", raising=False) + for name in ("app", "runner"): + monkeypatch.delitem(sys.modules, name, raising=False) + yield TestClient(importlib.import_module("app").app) + for name in ("app", "runner"): + sys.modules.pop(name, None) + + +def test_the_page_serves_a_runs_cleaning_view_or_a_404(page, tmp_path): + insurance = scenarios.load("insurance") + task = insurance.task + run = tmp_path / "claude" + run.mkdir() + (run / "transcript.jsonl").write_text("") + assert page.get("/api/runs/claude/cleaning").status_code == 404 + (run / task.HISTORY).write_text(RAW.replace("Policy ID", insurance.raw_id)) + (run / task.CLEAN_HISTORY).write_text( + CLEAN.replace("policy_id", task.ID).replace("premium", "annual_premium") + ) + (run / cleaning.SAVED).write_text("not json") # the agent's, ignored + res = page.get("/api/runs/claude/cleaning") + assert res.status_code == 200 and res.json()["rows_after"] == 5 + assert page.get("/api/runs/nobody/cleaning").status_code == 404 diff --git a/tests/unit/test_agent_demo_scenarios.py b/tests/unit/test_agent_demo_scenarios.py index 9f35e52..50ae81d 100644 --- a/tests/unit/test_agent_demo_scenarios.py +++ b/tests/unit/test_agent_demo_scenarios.py @@ -205,6 +205,17 @@ def test_hosted_claude_runs_bash_as_its_run_user_without_the_sandbox( assert settings["permissions"]["deny"] == [f"Read(/{INSURANCE.answers}/**)"] +def test_the_page_says_when_a_hosted_agent_has_no_sandbox( + monkeypatch, tmp_path, fresh_app +): + monkeypatch.setenv("DEMO_SCENARIO", "insurance") + monkeypatch.setenv("DEMO_RUNS", str(tmp_path)) + monkeypatch.setenv("CLAUDE_SANDBOX", "off") + monkeypatch.setenv("CODEX_SANDBOX", "danger-full-access") + agents = fresh_app().AGENTS + assert agents["claude"]["tools"] == agents["codex"]["tools"] == "MCP · shell" + + def test_gemini_keeps_its_shell_only_when_the_task_runs_code(tmp_path, fake_agents): run_dir = _run("insurance", "gemini", tmp_path, fake_agents) settings = json.loads((run_dir / ".gemini" / "settings.json").read_text()) @@ -298,6 +309,7 @@ def test_the_page_reads_a_raw_export_through_the_scenario( monkeypatch.setenv("DEMO_WORKSPACE", str(tmp_path)) monkeypatch.setenv("DEMO_RUNS", str(tmp_path / "runs")) monkeypatch.delenv("DEMO_HOSTED", raising=False) + monkeypatch.delenv("CLAUDE_SANDBOX", raising=False) app = fresh_app() data = app.dataset() assert data["history_rows"] == len(history) @@ -429,3 +441,22 @@ def test_main_writes_the_workspace_and_keeps_the_answers_apart(tmp_path, monkeyp assert len(answers) == 80 history = pd.read_csv(scenario.workspace / insurance.HISTORY) assert 0.05 < scenario.positive_rate(history) < 0.3 + + +def test_a_build_seed_changes_the_withheld_outcomes(tmp_path, monkeypatch): + """The hosted image draws DEMO_SEED and keeps it nowhere: the generator + is public, so the fixed seed would let an agent rebuild the answers.""" + monkeypatch.delenv("DEMO_WORKSPACE", raising=False) + + def answers(seed: str | None) -> pd.DataFrame: + if seed is None: + monkeypatch.delenv("DEMO_SEED", raising=False) + else: + monkeypatch.setenv("DEMO_SEED", seed) + scenario = replace(INSURANCE, root=tmp_path / (seed or "fixed")) + insurance_data.main(scenario, history_rows=200, current_rows=50) + return pd.read_csv(scenario.answers / insurance.ANSWERS) + + fixed = answers(None) + assert fixed.equals(answers(str(insurance_data.SEED))) + assert not fixed[insurance.ID].equals(answers("18446744073709551557")[insurance.ID])