From 266b85c37a22c1f2dcec19bfce49b31a10e254d3 Mon Sep 17 00:00:00 2001 From: Edward Cheung Date: Wed, 28 Oct 2026 18:42:08 +0000 Subject: [PATCH] feat: add beta Designer App runtime plugin Signed-off-by: Edward Cheung --- docs/docs/api/appkit/Variable.designer.md | 7 + docs/docs/api/appkit/index.md | 1 + docs/docs/api/appkit/typedoc-sidebar.ts | 5 + docs/docs/plugins/designer.md | 132 +++ packages/appkit/package.json | 9 +- packages/appkit/src/designer.ts | 11 + .../src/plugins/beta-exports.generated.ts | 1 + packages/appkit/src/plugins/designer/index.ts | 1 + .../appkit/src/plugins/designer/manifest.json | 25 + .../appkit/src/plugins/designer/plugin.ts | 35 + .../designer/server/browserRequestOrigin.ts | 33 + .../designer/server/exportedRunOutput.ts | 340 ++++++ .../plugins/designer/server/fileDownloads.ts | 51 + .../plugins/designer/server/fileUploads.ts | 251 +++++ .../src/plugins/designer/server/guardrails.ts | 125 +++ .../plugins/designer/server/jobParameters.ts | 28 + .../designer/server/outputFileStore.ts | 39 + .../plugins/designer/server/outputFiles.ts | 242 +++++ .../plugins/designer/server/runAttribution.ts | 38 + .../src/plugins/designer/server/runHistory.ts | 43 + .../src/plugins/designer/server/runOutputs.ts | 278 +++++ .../plugins/designer/server/runParameters.ts | 210 ++++ .../plugins/designer/server/runRevision.ts | 58 ++ .../src/plugins/designer/server/runtime.ts | 980 ++++++++++++++++++ .../plugins/designer/server/storageVolume.ts | 6 + .../plugins/designer/server/uploadStore.ts | 81 ++ .../src/plugins/designer/server/uploads.ts | 194 ++++ .../plugins/designer/shared/appManifest.ts | 236 +++++ .../plugins/designer/shared/fileFormats.ts | 76 ++ .../plugins/designer/shared/fileOutputs.ts | 115 ++ .../src/plugins/designer/shared/guardrails.ts | 54 + .../plugins/designer/shared/htmlOutputs.ts | 5 + .../plugins/designer/shared/multiselect.ts | 46 + .../designer/shared/publishedRunner.ts | 37 + .../plugins/designer/shared/resultPreview.ts | 37 + .../plugins/designer/shared/storageConfig.ts | 58 ++ .../src/plugins/designer/shared/values.ts | 37 + .../designer/tests/browser-origin.test.mjs | 44 + .../designer/tests/output-files.test.mjs | 440 ++++++++ .../plugins/designer/tests/results.test.mjs | 492 +++++++++ .../tests/runtime.integration.test.mjs | 397 +++++++ .../plugins/designer/tests/storage.test.mjs | 309 ++++++ .../tests/uploads.integration.test.mjs | 177 ++++ .../plugins/designer/tests/uploads.test.mjs | 730 +++++++++++++ packages/appkit/tsdown.config.ts | 1 + template/appkit.plugins.json | 35 + 46 files changed, 6548 insertions(+), 2 deletions(-) create mode 100644 docs/docs/api/appkit/Variable.designer.md create mode 100644 docs/docs/plugins/designer.md create mode 100644 packages/appkit/src/designer.ts create mode 100644 packages/appkit/src/plugins/designer/index.ts create mode 100644 packages/appkit/src/plugins/designer/manifest.json create mode 100644 packages/appkit/src/plugins/designer/plugin.ts create mode 100644 packages/appkit/src/plugins/designer/server/browserRequestOrigin.ts create mode 100644 packages/appkit/src/plugins/designer/server/exportedRunOutput.ts create mode 100644 packages/appkit/src/plugins/designer/server/fileDownloads.ts create mode 100644 packages/appkit/src/plugins/designer/server/fileUploads.ts create mode 100644 packages/appkit/src/plugins/designer/server/guardrails.ts create mode 100644 packages/appkit/src/plugins/designer/server/jobParameters.ts create mode 100644 packages/appkit/src/plugins/designer/server/outputFileStore.ts create mode 100644 packages/appkit/src/plugins/designer/server/outputFiles.ts create mode 100644 packages/appkit/src/plugins/designer/server/runAttribution.ts create mode 100644 packages/appkit/src/plugins/designer/server/runHistory.ts create mode 100644 packages/appkit/src/plugins/designer/server/runOutputs.ts create mode 100644 packages/appkit/src/plugins/designer/server/runParameters.ts create mode 100644 packages/appkit/src/plugins/designer/server/runRevision.ts create mode 100644 packages/appkit/src/plugins/designer/server/runtime.ts create mode 100644 packages/appkit/src/plugins/designer/server/storageVolume.ts create mode 100644 packages/appkit/src/plugins/designer/server/uploadStore.ts create mode 100644 packages/appkit/src/plugins/designer/server/uploads.ts create mode 100644 packages/appkit/src/plugins/designer/shared/appManifest.ts create mode 100644 packages/appkit/src/plugins/designer/shared/fileFormats.ts create mode 100644 packages/appkit/src/plugins/designer/shared/fileOutputs.ts create mode 100644 packages/appkit/src/plugins/designer/shared/guardrails.ts create mode 100644 packages/appkit/src/plugins/designer/shared/htmlOutputs.ts create mode 100644 packages/appkit/src/plugins/designer/shared/multiselect.ts create mode 100644 packages/appkit/src/plugins/designer/shared/publishedRunner.ts create mode 100644 packages/appkit/src/plugins/designer/shared/resultPreview.ts create mode 100644 packages/appkit/src/plugins/designer/shared/storageConfig.ts create mode 100644 packages/appkit/src/plugins/designer/shared/values.ts create mode 100644 packages/appkit/src/plugins/designer/tests/browser-origin.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/output-files.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/results.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/runtime.integration.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/storage.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/uploads.integration.test.mjs create mode 100644 packages/appkit/src/plugins/designer/tests/uploads.test.mjs diff --git a/docs/docs/api/appkit/Variable.designer.md b/docs/docs/api/appkit/Variable.designer.md new file mode 100644 index 000000000..847cb52ee --- /dev/null +++ b/docs/docs/api/appkit/Variable.designer.md @@ -0,0 +1,7 @@ +# Variable: designer + +```ts +const designer: ToPlugin; +``` + +Published Lakeflow Designer App runtime (beta). diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index ee6db8b67..f7f3b94dc 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -190,6 +190,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | ------ | ------ | | [agents](Variable.agents.md) | Plugin factory for the agents plugin. Discovers agents from `server/agents//agent.{ts,md}` by default (markdown still in `config/agents/` is read as a deprecated fallback), resolves toolkits/tools from registered plugins, exposes the `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | | [aiSearch](Variable.aiSearch.md) | - | +| [designer](Variable.designer.md) | Published Lakeflow Designer App runtime (beta). | | [READ\_ACTIONS](Variable.READ_ACTIONS.md) | Actions that only read data. | | [sql](Variable.sql.md) | SQL helper namespace | | [SUPERVISOR\_EXTENSION\_KEY](Variable.SUPERVISOR_EXTENSION_KEY.md) | Namespace key under which the adapter reads its hosted-tool payload from [AgentInput.extensions](Interface.AgentInput.md#extensions). Exported so the agents plugin and standalone `runAgent` (the producers) can write under the same key the adapter reads. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index dca9af23b..7f6603376 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -847,6 +847,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Variable.aiSearch", label: "aiSearch" }, + { + type: "doc", + id: "api/appkit/Variable.designer", + label: "designer" + }, { type: "doc", id: "api/appkit/Variable.READ_ACTIONS", diff --git a/docs/docs/plugins/designer.md b/docs/docs/plugins/designer.md new file mode 100644 index 000000000..1bdf11e7e --- /dev/null +++ b/docs/docs/plugins/designer.md @@ -0,0 +1,132 @@ +--- +sidebar_position: 6 +--- + +# Lakeflow Designer + + +:::warning Beta plugin +This plugin is currently **beta**. APIs may change between minor releases. Import from `@databricks/appkit/beta`. See [Plugin Stability Tiers](./stability.md). +::: + + +The beta `designer()` plugin runs the backend of a published Lakeflow Designer App. It serves the publisher's manifest, submits and monitors its runner Job, decodes notebook results, and transfers uploaded and generated files. The template owns the React interface, charts, tables, HTML sandbox, and browser recovery. + +```typescript +import { createApp, server } from "@databricks/appkit"; +import { designer } from "@databricks/appkit/beta"; + +await createApp({ plugins: [server(), designer()] }); +``` + +The Designer publisher provisions the Job and resource permissions. Register this plugin in that published template; it does not create a flow or grant permissions. No generic Jobs or Files plugins are required. + +## Runtime contract + +The plugin preserves the Designer template's `/api/designer` API and manifest version 8: + +| Endpoint | Behavior | +| --- | --- | +| `GET /config` | Read the current manifest and check whether its runner is ready | +| `POST /run` | Validate inputs and submit the bound Job | +| `GET /run/:jobRunId` | Read an authorized run and its output blocks | +| `DELETE /run/:jobRunId` | Cancel an authorized run | +| `GET /last-run`, `GET /runs` | Read the viewer's active run and history | +| `POST /uploads/:parameterName` | Stream an immutable parameter upload | +| `GET /uploads/:parameterName/:reference/download` | Download an owned upload | +| `GET /run/:jobRunId/files/:outputId/:fileIndex/download` | Download a declared file from an authorized run | + +All paths in the table are relative to `/api/designer`. File uploads use `application/octet-stream` with the URL-encoded filename in `X-File-Name`. Run submissions use JSON `{ params: { ... } }`. + +`DATABRICKS_JOB_ID` identifies the publisher-owned Job. The plugin discovers `designerApp.json` beside its runner notebook; the manifest is outside the publicly served app assets. Missing or incompatible manifests make the App unrunnable. Republish refreshes the manifest without requiring a redeploy. Submission remains unavailable while the manifest and runner revision disagree. + +`DATABRICKS_APP_ID`, falling back to `DATABRICKS_CLIENT_ID`, identifies the App in run attribution. Databricks Apps ingress supplies the trusted `X-Forwarded-User` and email headers. Jobs, workspace reads, and file storage use the App service principal. An optional forwarded access token is used only to look up the submitting user's display name. A custom deployment must provide an equally trusted ingress; these headers are not credentials supplied by an end user. + +The Job resource retains the publisher's `CAN_MANAGE` permission. Optional upload and output volumes come from the validated manifest, including changes made by republishing. Their grants remain the publisher's responsibility; a plugin manifest cannot discover these resources synchronously. Storage uses the Files connector directly, without nested AppKit instances, volume environment mutations, or generic `/api/files` routes. + +Every run read, cancellation, and download checks Job and viewer ownership. Download checks also validate the current publication, output declaration, and recorded receipt. Origin checks and private response headers apply throughout. Completed-output caches belong to the plugin instance and are cleared on shutdown; authorization precedes cache reads. Disabled nodes and groups use existing exported notebook metadata and manifest hints, without a codegen change. + +## Browser contracts + +Import schemas, types, and browser helpers through the dedicated entry point: + +```typescript +import { + appManifestSchema, + type AppManifest, + fileDownloadRoute, +} from "@databricks/appkit/beta/designer"; +``` + +This entry contains no server code or Node built-ins. Do not import the server plugin from a browser entry. Like `designer()`, these contracts are beta and should be consumed from an exact AppKit version. + +## Iterate locally with the Designer template + +Start from a template whose server uses the bootstrap above and whose shared imports use `@databricks/appkit/beta/designer`. In an AppKit checkout, install dependencies and build once: + +```bash +pnpm install --frozen-lockfile +pnpm build +``` + +In the template checkout, link the package: + +```bash +APPKIT_DIR=/absolute/path/to/appkit +pnpm link "$APPKIT_DIR/packages/appkit" +``` + +Run the template with the development export condition and explicit watches for linked sources: + +```bash +NODE_ENV=development NODE_OPTIONS=--conditions=development \ + DISABLE_REMOTE_SERVING=true pnpm exec tsx watch \ + --include "$APPKIT_DIR/packages/appkit/src/**/*.ts" \ + --include "$APPKIT_DIR/packages/shared/src/**/*.ts" \ + --tsconfig ./tsconfig.server.json --env-file-if-exists=./.env \ + ./server/server.ts +``` + +`NODE_ENV` alone does not select Node's `development` exports. Plugin and shared-source edits restart the server; Vite continues to provide client HMR. Configure the usual local Databricks credentials and a test runner Job in `.env`. Use `APPKIT_DEV_OBO=false` only if your local CLI does not support AppKit's development OBO token flow. + +Run backend tests from the AppKit checkout: + +```bash +pnpm exec vitest run --project appkit src/plugins/designer +``` + +These tests boot the real AppKit server with simulated Databricks services. They cover submission, ownership, history, streaming files, manifest refresh, output decoding, and shutdown isolation. They do not replace a Databricks deployment test. + +After local work, restore the template's pinned registry dependency and lockfile. Do not commit the local link or its pnpm override. + +## Test the packaged integration before release + +From AppKit, build and package the SDK through its normal distribution process: + +```bash +pnpm build +pnpm docs:build +pnpm --filter @databricks/appkit tarball +``` + +The tarball is written under `packages/appkit/tmp/`. In a separate test template branch, copy it into the template and install it: + +```bash +mkdir -p vendor +cp "$APPKIT_DIR"/packages/appkit/tmp/databricks-appkit-*.tgz vendor/ +pnpm add ./vendor/databricks-appkit-.tgz +pnpm build +``` + +Remove the `overrides["@databricks/appkit"]` source-link entry in `pnpm-workspace.yaml` first. Commit the test template, tarball, and generated pnpm lockfile together. This makes the cloud build consume the same package you tested locally, including its compiled exports and bundled shared code. Keep existing template dependency overrides and patches in place. + +Publish a separate test App from that template commit using the Designer environment's template-reference override. Verify the deployment build log resolves the tarball, then exercise: + +1. Manifest loading and an enabled Run button. +2. A successful run with table, chart, and HTML output. +3. Both parameter-controlled branches, including disabled groups. +4. Upload, run, and repeated input/output downloads. +5. History selection, cancellation, and access from a second viewer. +6. Republish with changed inputs or storage, followed by a fresh run. + +The test tarball and template-reference override are for validation. The production sequence is: merge the AppKit PR, release AppKit, update the template's exact dependency and lockfile, advance the publisher's template revision, and redeploy Apps that need the new server. Do not pin an unpublished registry version or ship a local filesystem link. diff --git a/packages/appkit/package.json b/packages/appkit/package.json index 968531019..2cf7ca139 100644 --- a/packages/appkit/package.json +++ b/packages/appkit/package.json @@ -56,7 +56,11 @@ "types": "./dist/shared/src/plugin.d.ts", "default": "./dist/shared/src/plugin.d.ts" }, - "./package.json": "./package.json" + "./package.json": "./package.json", + "./beta/designer": { + "development": "./src/designer.ts", + "default": "./dist/designer.js" + } }, "scripts": { "build:package": "pnpm exec tsx ../../tools/generate-registry-types.ts && pnpm exec tsx ../../tools/generate-plugin-entries.ts && tsdown --config tsdown.config.ts", @@ -169,7 +173,8 @@ "types": "./dist/tsdown/index.d.ts", "default": "./dist/tsdown/index.js" }, - "./package.json": "./package.json" + "./package.json": "./package.json", + "./beta/designer": "./dist/designer.js" } } } diff --git a/packages/appkit/src/designer.ts b/packages/appkit/src/designer.ts new file mode 100644 index 000000000..2472aa246 --- /dev/null +++ b/packages/appkit/src/designer.ts @@ -0,0 +1,11 @@ +// Browser-safe contracts for the beta Designer plugin. +export * from "./plugins/designer/shared/appManifest"; +export * from "./plugins/designer/shared/fileFormats"; +export * from "./plugins/designer/shared/fileOutputs"; +export * from "./plugins/designer/shared/guardrails"; +export * from "./plugins/designer/shared/htmlOutputs"; +export * from "./plugins/designer/shared/multiselect"; +export * from "./plugins/designer/shared/publishedRunner"; +export * from "./plugins/designer/shared/resultPreview"; +export * from "./plugins/designer/shared/storageConfig"; +export * from "./plugins/designer/shared/values"; diff --git a/packages/appkit/src/plugins/beta-exports.generated.ts b/packages/appkit/src/plugins/beta-exports.generated.ts index 7e556ebd9..1c41c2d4c 100644 --- a/packages/appkit/src/plugins/beta-exports.generated.ts +++ b/packages/appkit/src/plugins/beta-exports.generated.ts @@ -7,3 +7,4 @@ export { agents } from "./agents"; export { aiSearch } from "./ai-search"; +export { designer } from "./designer"; diff --git a/packages/appkit/src/plugins/designer/index.ts b/packages/appkit/src/plugins/designer/index.ts new file mode 100644 index 000000000..e65fe057f --- /dev/null +++ b/packages/appkit/src/plugins/designer/index.ts @@ -0,0 +1 @@ +export { designer } from "./plugin"; diff --git a/packages/appkit/src/plugins/designer/manifest.json b/packages/appkit/src/plugins/designer/manifest.json new file mode 100644 index 000000000..4666f2f40 --- /dev/null +++ b/packages/appkit/src/plugins/designer/manifest.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://databricks.github.io/appkit/schemas/plugin-manifest.schema.json", + "name": "designer", + "displayName": "Lakeflow Designer", + "stability": "beta", + "description": "Serve published Lakeflow Designer Apps with viewer-scoped runs, results, and file transfers.", + "resources": { + "required": [], + "optional": [ + { + "type": "job", + "alias": "Designer runner", + "resourceKey": "job", + "description": "Publisher-owned Designer runner Job", + "permission": "CAN_MANAGE", + "fields": { + "id": { + "env": "DATABRICKS_JOB_ID", + "description": "Job bound by the Designer publisher" + } + } + } + ] + } +} diff --git a/packages/appkit/src/plugins/designer/plugin.ts b/packages/appkit/src/plugins/designer/plugin.ts new file mode 100644 index 000000000..cb2563cbc --- /dev/null +++ b/packages/appkit/src/plugins/designer/plugin.ts @@ -0,0 +1,35 @@ +import type { IAppRouter } from "shared"; + +import { ServiceContext } from "../../context/service-context"; +import { Plugin, toPlugin } from "../../plugin"; +import { defineManifest } from "../../registry"; +import manifest from "./manifest.json"; +import { createDesignerRuntime } from "./server/runtime"; + +class DesignerPlugin extends Plugin { + static manifest = defineManifest<"designer">(manifest); + private runtime?: ReturnType; + + async setup() { + // Designer Jobs and files are App-owned. Forwarded identity is checked by + // each route; it must not switch these operations to a viewer's credentials. + this.runtime = createDesignerRuntime( + ServiceContext.get().client, + this.config.telemetry, + ); + } + + injectRoutes(router: IAppRouter) { + this.runtime?.registerRoutes(router, (config) => + this.route(router, config), + ); + } + + async shutdown() { + this.runtime?.close(); + this.runtime = undefined; + } +} + +/** Published Lakeflow Designer App runtime (beta). */ +export const designer = toPlugin(DesignerPlugin); diff --git a/packages/appkit/src/plugins/designer/server/browserRequestOrigin.ts b/packages/appkit/src/plugins/designer/server/browserRequestOrigin.ts new file mode 100644 index 000000000..53f67649c --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/browserRequestOrigin.ts @@ -0,0 +1,33 @@ +interface BrowserRequestOrigin { + origin?: string; + appUrl?: string; + fetchSite?: string; +} + +// CORS protects response reads, not side effects. HTML frames must not act through +// the viewer's App session, including when an App endpoint accepts an empty body. +export function isSameOriginBrowserRequest({ + origin, + appUrl, + fetchSite, +}: BrowserRequestOrigin): boolean { + if ( + fetchSite !== undefined && + fetchSite !== "same-origin" && + fetchSite !== "none" + ) + return false; + // Non-browser API clients do not send browser origin metadata; ingress authenticates them. + if (origin === undefined) return true; + if (origin === "null" || appUrl === undefined) return false; + try { + const url = new URL(origin); + return ( + (url.protocol === "http:" || url.protocol === "https:") && + url.origin === origin && + url.origin === new URL(appUrl).origin + ); + } catch { + return false; + } +} diff --git a/packages/appkit/src/plugins/designer/server/exportedRunOutput.ts b/packages/appkit/src/plugins/designer/server/exportedRunOutput.ts new file mode 100644 index 000000000..5a1133736 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/exportedRunOutput.ts @@ -0,0 +1,340 @@ +import { + FILE_OUTPUTS_MIME_TYPE, + MAX_OUTPUT_FILES, + parseFileOutputBehavior, + type WrittenFiles, +} from "../shared/fileOutputs"; +import type { GuardrailDiagnostic } from "../shared/guardrails"; +import { isHtmlOutput } from "../shared/htmlOutputs"; +import { + DISPLAY_ROW_LIMIT, + summarizeResultPreview, +} from "../shared/resultPreview"; +import { isRecord } from "../shared/values"; +import { + commandGuardrailDiagnostics, + mergeGuardrailDiagnostics, +} from "./guardrails"; + +const RUNNER_PAYLOAD_VERSION = 2; +const NOTEBOOK_MODEL_ASSIGNMENT = /__DATABRICKS_NOTEBOOK_MODEL = '([^']*)'/; +const ROW_COUNTS_MIME_TYPE = + "application/vnd.databricks.lakeflow-designer.row-counts+json"; +const RUNTIME_NODE_STATE_MIME_TYPE = + "application/vnd.databricks.lakebuilder.runtime-node-state+json"; +// Each cell's display(ctx["."]) lines name the ports it renders, in source order. +const DISPLAY_CTX_KEY = /display\(ctx\["([^"]+)"\]\)/g; + +export function findNotebookModelValue( + exportedHtml: unknown, +): string | undefined { + if (typeof exportedHtml !== "string") { + return undefined; + } + const direct = exportedHtml.match(NOTEBOOK_MODEL_ASSIGNMENT); + if (direct !== null) { + return direct[1]; + } + try { + return Buffer.from(exportedHtml, "base64") + .toString() + .match(NOTEBOOK_MODEL_ASSIGNMENT)?.[1]; + } catch { + return undefined; + } +} + +function decodeNotebookModel( + exportedHtml: unknown, +): Record | undefined { + const value = findNotebookModelValue(exportedHtml); + if (value === undefined) { + return undefined; + } + try { + const parsed: unknown = JSON.parse( + decodeURIComponent(Buffer.from(value, "base64").toString()), + ); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function displayedCtxKeys(source: unknown): string[] { + if (typeof source !== "string") { + return []; + } + return Array.from(source.matchAll(DISPLAY_CTX_KEY), (match) => match[1]); +} + +// Node names carry no dots, so the last dot separates node and port. +function splitCtxKey(key: string): { node: string; port: string } { + const dot = key.lastIndexOf("."); + return dot === -1 + ? { node: key, port: "" } + : { node: key.slice(0, dot), port: key.slice(dot + 1) }; +} + +function simpleType(rawType: unknown): string { + if (typeof rawType !== "string") { + return "string"; + } + try { + const parsed: unknown = JSON.parse(rawType); + return typeof parsed === "string" ? parsed : rawType; + } catch { + return rawType; + } +} + +function toDisplayTable(entry: unknown) { + if ( + !isRecord(entry) || + entry.type !== "table" || + !Array.isArray(entry.schema) || + !Array.isArray(entry.data) + ) { + return undefined; + } + const schema = entry.schema.filter(isRecord).map((field) => ({ + name: typeof field.name === "string" ? field.name : "", + type: simpleType(field.type), + nullable: typeof field.nullable === "boolean" ? field.nullable : true, + })); + const columnNames = schema.map((field) => field.name); + const data = entry.data.filter((row) => Array.isArray(row)); + const rows = data + .slice(0, DISPLAY_ROW_LIMIT) + .map((row) => + Object.fromEntries( + columnNames.map((name, index) => [name, row[index] ?? null]), + ), + ); + // Notebook overflow does not distinguish row and byte limits or carry an exact total. + // Preserve unknown overflow instead of interpreting absent metadata as a complete result. + return { + schema, + rows, + ...summarizeResultPreview(data.length, entry.overflow), + }; +} + +function resultEntries(results: unknown): unknown[] { + if ( + isRecord(results) && + (results.type === "table" || + results.type === "mimeBundle" || + results.type === "htmlSandbox" || + results.type === "html" || + results.type === "ansi" || + results.type === "text") + ) + return [results]; + return isRecord(results) && Array.isArray(results.data) ? results.data : []; +} + +// A MIME bundle may carry its payload as an object or a JSON string; unparseable strings are skipped. +function mimePayloads(results: unknown, mimeType: string): unknown[] { + return resultEntries(results).flatMap((entry) => { + if ( + !isRecord(entry) || + entry.type !== "mimeBundle" || + !isRecord(entry.data) + ) + return []; + const payload = entry.data[mimeType]; + if (typeof payload !== "string") return [payload]; + try { + const parsed: unknown = JSON.parse(payload); + return [parsed]; + } catch { + return []; + } + }); +} + +const isPortCount = (entry: [string, unknown]): entry is [string, number] => + entry[0] !== "" && + typeof entry[1] === "number" && + Number.isSafeInteger(entry[1]) && + entry[1] >= 0; + +function exactRowCounts(results: unknown): Map { + const counts = new Map(); + for (const payload of mimePayloads(results, ROW_COUNTS_MIME_TYPE)) { + if ( + !isRecord(payload) || + typeof payload.node !== "string" || + payload.node === "" || + !isRecord(payload.counts) + ) { + continue; + } + // One invalid count discards the node's whole payload. + const portCounts = Object.entries(payload.counts); + if (portCounts.length === 0 || !portCounts.every(isPortCount)) { + continue; + } + for (const [port, count] of portCounts) { + counts.set(`${payload.node}\0${port}`, count); + } + } + return counts; +} + +function runtimeNodeStates( + results: unknown, +): Map { + const states = new Map(); + for (const payload of mimePayloads(results, RUNTIME_NODE_STATE_MIME_TYPE)) { + if ( + isRecord(payload) && + typeof payload.nodeId === "string" && + payload.nodeId !== "" && + typeof payload.disabled === "boolean" + ) { + states.set(payload.nodeId, { disabled: payload.disabled }); + } + } + return states; +} + +function writtenFiles(results: unknown): WrittenFiles[] { + return mimePayloads(results, FILE_OUTPUTS_MIME_TYPE).flatMap((receipt) => { + if ( + !isRecord(receipt) || + typeof receipt.node !== "string" || + receipt.node === "" || + !Array.isArray(receipt.files) || + receipt.files.length > MAX_OUTPUT_FILES || + !receipt.files.every( + (file): file is { path: string } => + isRecord(file) && typeof file.path === "string", + ) + ) { + return []; + } + const behavior = parseFileOutputBehavior(receipt.behavior); + return [ + { + node: receipt.node, + files: receipt.files.map((file) => ({ path: file.path })), + ...(behavior ? { behavior } : {}), + }, + ]; + }); +} + +// Only runner-marked HTML cells can become App HTML blocks. Other rich displays stay private. +function htmlOutputs( + source: unknown, + results: unknown, +): Record[] { + if (typeof source !== "string") return []; + const markers = [...source.matchAll(/^# designer-app-html-output: (.+)$/gm)]; + if (markers.length !== 1) return []; + let node: unknown; + try { + node = JSON.parse(markers[0][1]); + } catch { + return []; + } + if (typeof node !== "string" || node === "") return []; + const entries = resultEntries(results).flatMap((entry) => { + if (!isRecord(entry)) return []; + if ( + entry.type === "mimeBundle" && + isRecord(entry.data) && + "text/html" in entry.data + ) { + return [entry.data["text/html"]]; + } + return entry.type === "htmlSandbox" || entry.type === "html" + ? [entry.data] + : []; + }); + if (entries.length === 0) return []; + const binding = { kind: "html", target_node: node }; + if (entries.length !== 1 || !isHtmlOutput(entries[0])) { + return [ + { + ...binding, + status: "error", + error: { + type: "HtmlOutputError", + message: + entries.length !== 1 + ? "The HTML operator must display exactly one HTML page." + : "The HTML output is invalid or exceeds the 4,194,304-character limit. Reduce the page size or its input data.", + }, + }, + ]; + } + return [{ ...binding, status: "ok", html: entries[0] }]; +} + +// MIME metadata must not consume a table ordinal when pairing results with display calls. +export function exportedModelToRunPayload( + exportedHtml: unknown, + taskErrorTexts: string[] = [], +): string | undefined { + const model = decodeNotebookModel(exportedHtml); + if ( + (model === undefined || !Array.isArray(model.commands)) && + taskErrorTexts.length === 0 + ) { + return undefined; + } + const outputs: Record[] = []; + const files: WrittenFiles[] = []; + const diagnostics: GuardrailDiagnostic[] = []; + const states = new Map(); + const commands = Array.isArray(model?.commands) ? model.commands : []; + for (const command of commands) { + if (!isRecord(command)) { + continue; + } + outputs.push(...htmlOutputs(command.command, command.results)); + diagnostics.push( + ...commandGuardrailDiagnostics(command, resultEntries(command.results)), + ); + const keys = displayedCtxKeys(command.command); + for (const [nodeId, state] of runtimeNodeStates(command.results)) { + states.set(nodeId, state); + } + // Receipts are independent of display results. A successful write must stay downloadable + // even if the preview/count fails or this cell deliberately has no tabular preview. + files.push(...writtenFiles(command.results)); + const rowCounts = exactRowCounts(command.results); + const tables = resultEntries(command.results).flatMap( + (entry) => toDisplayTable(entry) ?? [], + ); + tables.forEach((table, index) => { + const key = keys[index]; + if (key === undefined) { + return; + } + const parts = splitCtxKey(key); + const totalRowCount = rowCounts.get(`${parts.node}\0${parts.port}`); + outputs.push({ + status: "ok", + target_node: parts.node, + target_port: parts.port, + ...table, + ...(totalRowCount === undefined + ? {} + : { total_row_count: totalRowCount }), + }); + }); + } + return JSON.stringify({ + version: RUNNER_PAYLOAD_VERSION, + outputs, + files, + diagnostics: mergeGuardrailDiagnostics(diagnostics, taskErrorTexts), + ...(states.size > 0 + ? { runtimeNodeStates: Object.fromEntries(states) } + : {}), + }); +} diff --git a/packages/appkit/src/plugins/designer/server/fileDownloads.ts b/packages/appkit/src/plugins/designer/server/fileDownloads.ts new file mode 100644 index 000000000..4d63c904f --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/fileDownloads.ts @@ -0,0 +1,51 @@ +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +import type { Response } from "express"; + +export async function streamFileDownload( + res: Response, + stream: ReadableStream, + filename: string, + size: number, + report: (error: unknown) => void, +) { + const fallback = filename.replace(/[^A-Za-z0-9._ -]/g, "_"); + const encoded = encodeURIComponent(filename).replace( + /['()*]/g, + (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`, + ); + res.setHeader("Cache-Control", "no-store"); + res.setHeader("Content-Type", "application/octet-stream"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${fallback}"; filename*=UTF-8''${encoded}`, + ); + res.setHeader("Content-Length", String(size)); + res.setHeader("X-Content-Type-Options", "nosniff"); + const reader = stream.getReader(); + const cancelRead = () => { + void reader.cancel().catch(report); + }; + res.once("close", cancelRead); + async function* bytes() { + let transferred = 0; + try { + for (;;) { + const next = await reader.read(); + if (next.done) break; + transferred += next.value.byteLength; + if (transferred > size) + throw new Error("File changed during transfer."); + yield next.value; + } + if (transferred !== size) + throw new Error("File transfer was incomplete."); + } finally { + res.off("close", cancelRead); + await reader.cancel().catch(report); + reader.releaseLock(); + } + } + await pipeline(Readable.from(bytes()), res); +} diff --git a/packages/appkit/src/plugins/designer/server/fileUploads.ts b/packages/appkit/src/plugins/designer/server/fileUploads.ts new file mode 100644 index 000000000..fd0d926dc --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/fileUploads.ts @@ -0,0 +1,251 @@ +import { createHash, randomUUID } from "node:crypto"; + +import { + MAX_UPLOAD_SIZE_LABEL, + UPLOAD_REFERENCE, + UPLOAD_UNAVAILABLE, + uploadStoragePath, + type AppStorage, +} from "../shared/storageConfig"; +import { isRecord } from "../shared/values"; +import { runJobParameters } from "./jobParameters"; + +export const APP_VIEWER_PARAM = "_lb_app_viewer"; + +export class UploadError extends Error { + status: number; + code?: typeof UPLOAD_UNAVAILABLE; + constructor( + status: number, + message: string, + options?: ErrorOptions & { code?: typeof UPLOAD_UNAVAILABLE }, + ) { + super(message, options); + this.status = status; + this.code = options?.code; + } +} + +const hash = (value: string) => + createHash("sha256").update(value).digest("hex"); + +// Databricks Apps supplies this header at its authenticated ingress. Never take identity from a body/query. +export function viewerKey( + forwardedUser: string | undefined, + jobId: string | undefined, +): string | undefined { + return forwardedUser?.trim() && jobId + ? hash(JSON.stringify([jobId, forwardedUser.trim()])) + : undefined; +} + +export function canAccessRun( + run: unknown, + viewer: string | undefined, +): boolean { + const owner = runJobParameters(run)[APP_VIEWER_PARAM]; + return viewer !== undefined && viewer !== "" && owner === viewer; +} + +interface StoredUpload { + reference: string; + filename: string; + size: number; + createdAt: number; +} + +// Keep storage operations behind a small boundary so failures/partial writes can be exercised without UC. +export interface UploadStore { + mkdir(path: string): Promise; + put(path: string, bytes: Uint8Array): Promise; + putStream(path: string, stream: ReadableStream): Promise; + read(path: string): Promise; + download(path: string): Promise>; + size(path: string): Promise; + delete(path: string): Promise; +} + +function uploadFolder( + config: AppStorage, + viewer: string, + parameterName: string, +): string { + return `${uploadStoragePath(config)}/${viewer}/${hash(parameterName)}`; +} + +function uploadId(reference: string): string { + const id = UPLOAD_REFERENCE.exec(reference)?.[1]; + if (!id) + throw new UploadError(400, "Choose an uploaded file before running."); + return id; +} + +function isValidFilename(filename: string): boolean { + return ( + filename.trim() !== "" && + filename !== "." && + filename !== ".." && + Buffer.byteLength(filename, "utf8") <= 255 && + ![...filename].some( + (character) => + character === "/" || + character === "\\" || + character.charCodeAt(0) < 32 || + character.charCodeAt(0) === 127, + ) + ); +} + +function parseStoredUpload( + raw: unknown, + reference: string, + maxBytes: number, +): StoredUpload { + if ( + !isRecord(raw) || + raw.reference !== reference || + typeof raw.filename !== "string" || + !isValidFilename(raw.filename) || + typeof raw.size !== "number" || + !Number.isSafeInteger(raw.size) || + raw.size <= 0 || + raw.size > maxBytes || + typeof raw.createdAt !== "number" || + !Number.isFinite(raw.createdAt) + ) { + throw new UploadError( + 409, + "This upload is incomplete or unreadable. Upload the file again.", + { + code: UPLOAD_UNAVAILABLE, + }, + ); + } + return { + reference, + filename: raw.filename, + size: raw.size, + createdAt: raw.createdAt, + }; +} + +export async function resolveUpload( + store: UploadStore, + config: AppStorage, + viewer: string, + parameterName: string, + reference: string, +) { + const id = uploadId(reference); + const folder = uploadFolder(config, viewer, parameterName); + const upload = parseStoredUpload( + await store.read(`${folder}/${id}.json`), + reference, + config.maxUploadFileSizeBytes, + ); + const path = `${folder}/${id}/${upload.filename}`; + if ((await store.size(path)) !== upload.size) + throw new UploadError( + 409, + "The uploaded file is missing or has changed. Upload the file again.", + { + code: UPLOAD_UNAVAILABLE, + }, + ); + return { path, upload }; +} + +export async function saveUploadStream( + store: UploadStore, + config: AppStorage, + viewer: string, + parameterName: string, + filename: string, + stream: ReadableStream, + declaredSize?: number, +): Promise { + if (!isValidFilename(filename)) + throw new UploadError(400, "Choose a file with a valid filename."); + if ( + declaredSize !== undefined && + declaredSize > config.maxUploadFileSizeBytes + ) + throw new UploadError( + 413, + `Files must be at most ${MAX_UPLOAD_SIZE_LABEL}.`, + ); + const folder = uploadFolder(config, viewer, parameterName); + const id = randomUUID(); + // Isolate each upload so its original extension survives and JSON data cannot collide with its sidecar. + const path = `${folder}/${id}/${filename}`; + await store.mkdir(`${folder}/${id}`); + let size = 0; + let validationError: UploadError | undefined; + const boundedStream = stream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + size += chunk.byteLength; + if (size > config.maxUploadFileSizeBytes) { + validationError = new UploadError( + 413, + `Files must be at most ${MAX_UPLOAD_SIZE_LABEL}.`, + ); + throw validationError; + } + if (declaredSize !== undefined && size > declaredSize) { + validationError = new UploadError( + 400, + "The upload exceeded its declared size.", + ); + throw validationError; + } + controller.enqueue(chunk); + }, + }), + ); + try { + await store.putStream(path, boundedStream); + if (size === 0) throw new UploadError(400, "Choose a non-empty file."); + if (declaredSize !== undefined && size !== declaredSize) + throw new UploadError(400, "The upload did not match its declared size."); + const upload: StoredUpload = { + reference: `upload:${id}`, + filename, + size, + createdAt: Date.now(), + }; + // The sidecar is the completion marker; partially written files can never be selected for a run. + await store.put( + `${folder}/${id}.json`, + Buffer.from(JSON.stringify(upload)), + ); + return upload; + } catch (error) { + await store.delete(path).catch(() => undefined); + throw validationError ?? error; + } +} + +export function saveUpload( + store: UploadStore, + config: AppStorage, + viewer: string, + parameterName: string, + filename: string, + bytes: Uint8Array, +): Promise { + return saveUploadStream( + store, + config, + viewer, + parameterName, + filename, + new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + bytes.byteLength, + ); +} diff --git a/packages/appkit/src/plugins/designer/server/guardrails.ts b/packages/appkit/src/plugins/designer/server/guardrails.ts new file mode 100644 index 000000000..0f1bc4203 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/guardrails.ts @@ -0,0 +1,125 @@ +import { isMap, parseDocument } from "yaml"; + +import type { GuardrailDiagnostic } from "../shared/guardrails"; +import { isRecord, nonEmptyString } from "../shared/values"; + +const ANSI_ESCAPE_SEQUENCE = new RegExp( + `${String.fromCharCode(0x1b)}\\[[0-9;]*m`, + "g", +); +const GUARDRAILS_MARKER = + /\[guardrails\]\s*(?:(WARNING|FAILURE):\s*)?checks\[(\d+)\]\s*(?:\(([^)]*)\))?\s*:\s*(.+)/; + +function guardrailOperator( + source: unknown, +): { nodeId: string; operatorName: string } | undefined { + if (typeof source !== "string") return undefined; + // Restrict metadata to the generated leading docstring, never config/code that mentions Guardrails. + const annotation = source.match(/^\s*"""\r?\n([\s\S]*?)\r?\n"""/); + if (!annotation) return undefined; + const doc = parseDocument(annotation[1], { schema: "failsafe" }); + if ( + doc.errors.length > 0 || + !isMap(doc.contents) || + doc.get("template") !== "guardrails" + ) + return undefined; + const nodeId = nonEmptyString(doc.get("id")); + if (!nodeId) return undefined; + return { nodeId, operatorName: nonEmptyString(doc.get("name")) ?? nodeId }; +} + +function markers( + texts: string[], + operator?: { nodeId: string; operatorName: string }, +): GuardrailDiagnostic[] { + return texts.flatMap((text) => + text + .replace(ANSI_ESCAPE_SEQUENCE, "") + .split(/\r?\n/) + .flatMap((line) => { + const match = line.match(GUARDRAILS_MARKER); + if (!match || !Number.isSafeInteger(Number(match[2]))) return []; + return [ + { + ...operator, + operatorName: operator?.operatorName ?? "Guardrails", + checkIndex: Number(match[2]), + ...(match[3] ? { checkType: match[3] } : {}), + severity: + match[1] === "WARNING" + ? ("warning" as const) + : ("failure" as const), + message: match[4].trim(), + }, + ]; + }), + ); +} + +function summaryText(value: string): string { + return value + .replace(/|<\/(?:div|p)>/gi, "\n") + .replace(/<[^>]*>/g, " ") + .replace( + /&(lt|gt|amp|quot|#39);/g, + (entity) => + ({ + "<": "<", + ">": ">", + "&": "&", + """: '"', + "'": "'", + })[entity] ?? entity, + ); +} + +export function commandGuardrailDiagnostics( + command: Record, + entries: unknown[], +): GuardrailDiagnostic[] { + const operator = guardrailOperator(command.command); + if (!operator) return []; + const texts = entries.flatMap((entry) => + isRecord(entry) && + (entry.type === "ansi" || entry.type === "text") && + typeof entry.data === "string" + ? [entry.data] + : [], + ); + if (typeof command.errorTrace === "string") texts.push(command.errorTrace); + if (typeof command.errorSummaryHtml === "string") + texts.push(summaryText(command.errorSummaryHtml)); + return markers(texts, operator); +} + +export function mergeGuardrailDiagnostics( + diagnostics: GuardrailDiagnostic[], + taskErrorTexts: string[] = [], +): GuardrailDiagnostic[] { + const merged = new Map(); + for (const diagnostic of diagnostics) { + const key = `${diagnostic.nodeId}\0${diagnostic.checkIndex}`; + const previous = merged.get(key); + if (!previous || diagnostic.severity === "failure") + merged.set(key, diagnostic); + } + // Jobs can retain the raised exception even when the failed cell's stdout was lost. + // Without cell metadata, keep an unattributed check rather than guessing an operator. + for (const diagnostic of markers(taskErrorTexts)) { + const existing = [...merged.values()].some( + (entry) => + entry.checkIndex === diagnostic.checkIndex && + entry.checkType === diagnostic.checkType && + entry.message === diagnostic.message && + entry.severity === diagnostic.severity, + ); + if (!existing) { + merged.set( + `task\0${diagnostic.checkIndex}\0${diagnostic.checkType}\0${diagnostic.severity}\0${diagnostic.message}`, + diagnostic, + ); + } + } + return [...merged.values()]; +} diff --git a/packages/appkit/src/plugins/designer/server/jobParameters.ts b/packages/appkit/src/plugins/designer/server/jobParameters.ts new file mode 100644 index 000000000..71f2fe0ac --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/jobParameters.ts @@ -0,0 +1,28 @@ +import { isRecord } from "../shared/values"; + +export function runJobParameters(run: unknown): Record { + if ( + typeof run !== "object" || + run === null || + !("job_parameters" in run) || + !Array.isArray(run.job_parameters) + ) { + return {}; + } + const parameters: Record = {}; + for (const parameter of run.job_parameters) { + if ( + isRecord(parameter) && + typeof parameter.name === "string" && + typeof parameter.value === "string" + ) { + Object.defineProperty(parameters, parameter.name, { + value: parameter.value, + enumerable: true, + configurable: true, + writable: true, + }); + } + } + return parameters; +} diff --git a/packages/appkit/src/plugins/designer/server/outputFileStore.ts b/packages/appkit/src/plugins/designer/server/outputFileStore.ts new file mode 100644 index 000000000..18ca4aaf7 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/outputFileStore.ts @@ -0,0 +1,39 @@ +import type { FilesConnector } from "../../../connectors/files"; +import { ApiError, type WorkspaceClient } from "../../../workspace-client"; +import { isOutputFilePath } from "../shared/fileOutputs"; +import { parseStorageFileSize } from "./storageVolume"; + +export interface OutputFileStore { + size(path: string): Promise; + download(path: string): Promise>; +} + +export function createOutputFileStore( + client: WorkspaceClient, + connector: FilesConnector, + volume: string, +): OutputFileStore { + return { + async size(path) { + if (!isOutputFilePath(path, { volumes: [volume] })) + throw new Error("Invalid output file destination."); + try { + return parseStorageFileSize( + (await connector.metadata(client, path)).contentLength, + ); + } catch (error) { + if (error instanceof ApiError && error.statusCode === 404) + return undefined; + throw error; + } + }, + async download(path) { + if (!isOutputFilePath(path, { volumes: [volume] })) + throw new Error("Invalid output file destination."); + const result = await connector.download(client, path); + if (!result.contents) + throw new Error("The output file has no readable contents."); + return result.contents; + }, + }; +} diff --git a/packages/appkit/src/plugins/designer/server/outputFiles.ts b/packages/appkit/src/plugins/designer/server/outputFiles.ts new file mode 100644 index 000000000..baa97ecb5 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/outputFiles.ts @@ -0,0 +1,242 @@ +import type { Request } from "express"; +import type { RouteConfig } from "shared"; + +import { + APP_REVISION_PARAM, + FILE_OUTPUTS_PARAM, + isOutputFilePath, + MAX_OUTPUT_FILES, + parseFileOutput, + parseFileOutputBehavior, + type FileOutputConfig, + type WrittenFiles, +} from "../shared/fileOutputs"; +import { RUNNER_REVISION_PARAM } from "../shared/publishedRunner"; +import { idString, isRecord } from "../shared/values"; +import { streamFileDownload } from "./fileDownloads"; +import { canAccessRun } from "./fileUploads"; +import { runJobParameters } from "./jobParameters"; +import type { OutputFileStore } from "./outputFileStore"; +import { + isLegacyExportRun, + manifestRevision, + type FileOutputManifest, +} from "./runRevision"; + +export function recordedFileOutput( + payload: string | undefined, + node: string, + config: FileOutputConfig, +): WrittenFiles | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(payload ?? ""); + } catch { + return undefined; + } + if (!isRecord(parsed) || !Array.isArray(parsed.files)) return undefined; + const matches = parsed.files.filter( + (entry): entry is Record => + isRecord(entry) && entry.node === node, + ); + if (matches.length !== 1) return undefined; + const [receipt] = matches; + if ( + !Array.isArray(receipt.files) || + receipt.files.length > MAX_OUTPUT_FILES + ) { + return undefined; + } + const paths = receipt.files.map((file) => + isRecord(file) ? file.path : undefined, + ); + if ( + !paths.every((path): path is string => isOutputFilePath(path, config)) || + new Set(paths).size !== paths.length + ) { + return undefined; + } + const behavior = parseFileOutputBehavior(receipt.behavior); + return { + node, + files: paths.map((path) => ({ path })), + ...(behavior ? { behavior } : {}), + }; +} + +interface Dependencies { + jobId: string | undefined; + manifest(): Promise; + viewer(req: Request): string | undefined; + notebookPath(): Promise; + getRun(id: number): Promise; + readPayload(taskId: string): Promise; + store(volume: string): OutputFileStore; + report: (error: unknown) => void; +} + +class DownloadError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +export function registerOutputFileRoutes( + route: (config: RouteConfig) => void, + deps: Dependencies, +) { + route({ + name: "downloadOutput", + method: "get", + path: "/run/:jobRunId/files/:outputId/:fileIndex/download", + handler: async (req, res) => { + try { + if (req.method !== "GET") + throw new DownloadError(405, "Use GET to download the file."); + if (req.get("range")) + throw new DownloadError( + 416, + "Partial downloads are not supported. Retry the full download.", + ); + const viewer = deps.viewer(req); + if (!viewer) + throw new DownloadError( + 401, + "Sign in through Databricks Apps to download files.", + ); + const runId = String(req.params.jobRunId); + if ( + !/^[1-9][0-9]*$/.test(runId) || + !Number.isSafeInteger(Number(runId)) + ) { + throw new DownloadError(404, "Run not found."); + } + const manifest = await deps.manifest(); + if (!manifest || !deps.jobId) + throw new DownloadError(409, "File outputs are not configured."); + const block = manifest.blocks.find( + (entry) => + entry.type === "output" && entry.id === req.params.outputId, + ); + if (!block?.fileOutput || !block.nodeId) + throw new DownloadError(404, "File output not found."); + const run = await deps.getRun(Number(runId)); + if ( + !isRecord(run) || + String(run.job_id) !== deps.jobId || + !canAccessRun(run, viewer) || + isLegacyExportRun(run) + ) { + throw new DownloadError(404, "Run not found."); + } + const params = runJobParameters(run); + if (params[APP_REVISION_PARAM] !== manifestRevision(manifest)) { + throw new DownloadError( + 409, + "This run uses a different file-output configuration. Run the app again to download files.", + ); + } + let declared: unknown; + try { + declared = JSON.parse(String(params[FILE_OUTPUTS_PARAM])); + } catch { + throw new DownloadError( + 409, + "This run did not declare file outputs. Run the app again.", + ); + } + const declaredOutput = parseFileOutput( + isRecord(declared) ? declared[block.nodeId] : undefined, + ); + if ( + !declaredOutput || + JSON.stringify(declaredOutput) !== + JSON.stringify(parseFileOutput(block.fileOutput)) + ) { + throw new DownloadError( + 409, + "This run did not declare this file output. Run the app again.", + ); + } + const task: unknown = + Array.isArray(run.tasks) && run.tasks.length === 1 + ? run.tasks[0] + : undefined; + const taskId = isRecord(task) ? idString(task.run_id) : undefined; + const currentNotebookPath = await deps.notebookPath(); + const revision = params[RUNNER_REVISION_PARAM]; + const folder = currentNotebookPath?.slice( + 0, + currentNotebookPath.lastIndexOf("/"), + ); + if ( + !isRecord(task) || + !isRecord(task.notebook_task) || + !taskId || + !folder || + typeof revision !== "string" || + !/^[0-9a-f]{64}$/.test(revision) || + task.notebook_task.notebook_path !== + `${folder}/runner-${revision}.designer.py` + ) { + throw new DownloadError( + 409, + "This run’s published workflow could not be verified. Run the app again to download files.", + ); + } + const state = isRecord(run.state) ? run.state : {}; + if ( + !state.result_state && + !["TERMINATED", "SKIPPED", "INTERNAL_ERROR"].includes( + String(state.life_cycle_state), + ) + ) { + throw new DownloadError( + 409, + "Wait for the run to finish before downloading files.", + ); + } + const receipt = recordedFileOutput( + await deps.readPayload(taskId), + block.nodeId, + block.fileOutput, + ); + const indexText = String(req.params.fileIndex); + const index = /^(0|[1-9][0-9]*)$/.test(indexText) + ? Number(indexText) + : -1; + const file = receipt?.files[index]; + if (!file) + throw new DownloadError( + 404, + "The run has no completed file at this output.", + ); + const store = deps.store(file.path.split("/").slice(2, 5).join(".")); + const size = await store.size(file.path); + if (size === undefined) { + throw new DownloadError( + 404, + "This output file no longer exists at its recorded destination.", + ); + } + const stream = await store.download(file.path); + const filename = file.path.slice(file.path.lastIndexOf("/") + 1); + await streamFileDownload(res, stream, filename, size, deps.report); + } catch (error) { + if (!(error instanceof DownloadError)) deps.report(error); + if (res.headersSent) { + res.destroy(error instanceof Error ? error : undefined); + } else { + res.status(error instanceof DownloadError ? error.status : 502).json({ + error: + error instanceof DownloadError + ? error.message + : "Could not read the output file. Check the App’s volume permissions and try again.", + }); + } + } + }, + }); +} diff --git a/packages/appkit/src/plugins/designer/server/runAttribution.ts b/packages/appkit/src/plugins/designer/server/runAttribution.ts new file mode 100644 index 000000000..701edbadf --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runAttribution.ts @@ -0,0 +1,38 @@ +import { randomUUID } from "node:crypto"; + +export const APP_PARAMETERS_PARAM = "_lb_app_parameters"; + +interface ForwardedHeaders { + get(name: string): string | undefined; +} + +// Apps ingress supplies these headers. Body parameters never establish the submitting user. +export async function runAttribution( + headers: ForwardedHeaders, + appId: string, + readUserProfile?: ( + token: string, + ) => Promise<{ id?: string; displayName?: string }>, +): Promise | undefined> { + const subject = headers.get("x-forwarded-user")?.trim(); + if (!subject) return undefined; + const userId = /^(\d+)@\d+$/.exec(subject)?.[1] ?? subject; + const email = headers.get("x-forwarded-email")?.trim() ?? ""; + let name = headers.get("x-forwarded-preferred-username")?.trim(); + const token = headers.get("x-forwarded-access-token")?.trim(); + if ((!name || name === email) && token && readUserProfile) { + try { + const profile = await readUserProfile(token); + if (profile.id === userId) name = profile.displayName?.trim() || name; + } catch { + // Profile availability must not block a run whose ingress identity is already verified. + } + } + return { + _lb_app_id: appId, + _lb_app_user_id: userId, + _lb_app_user_name: name || email || userId, + _lb_app_user_email: email, + _lb_app_submission_id: randomUUID(), + }; +} diff --git a/packages/appkit/src/plugins/designer/server/runHistory.ts b/packages/appkit/src/plugins/designer/server/runHistory.ts new file mode 100644 index 000000000..d9fbb7394 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runHistory.ts @@ -0,0 +1,43 @@ +import { isRecord, nonEmptyString } from "../shared/values"; + +export interface VisibleRunPage { + runs: Record[]; + nextPageToken?: string; +} + +// Bound each HTTP request, but preserve the Jobs cursor when other viewers fill the scan. +export async function scanVisibleRuns({ + pageToken, + readPage, + accessible, + isComplete, +}: { + pageToken?: string; + readPage: (pageToken?: string) => Promise; + accessible: (run: unknown) => Promise | undefined>; + isComplete: (runs: Record[]) => boolean; +}): Promise { + const runs: Record[] = []; + const seen = new Set(); + if (pageToken) seen.add(pageToken); + for (let page = 0; page < 8; page += 1) { + const response = await readPage(pageToken); + if ( + !isRecord(response) || + (response.runs !== undefined && !Array.isArray(response.runs)) + ) { + throw new Error("The Job returned an unreadable run-history page."); + } + for (const run of Array.isArray(response.runs) ? response.runs : []) { + const visible = await accessible(run); + if (visible) runs.push(visible); + } + pageToken = nonEmptyString(response.next_page_token); + if (!pageToken) return { runs }; + if (seen.has(pageToken)) + throw new Error("The Job returned a repeated run-history cursor."); + seen.add(pageToken); + if (isComplete(runs)) break; + } + return { runs, nextPageToken: pageToken }; +} diff --git a/packages/appkit/src/plugins/designer/server/runOutputs.ts b/packages/appkit/src/plugins/designer/server/runOutputs.ts new file mode 100644 index 000000000..a3cd8d613 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runOutputs.ts @@ -0,0 +1,278 @@ +import { outputBlockTitle, type AppOutputBlock } from "../shared/appManifest"; +import type { FileOutputBehavior } from "../shared/fileOutputs"; +import { parseRunDiagnostics, type RunDiagnostics } from "../shared/guardrails"; +import { isHtmlOutput } from "../shared/htmlOutputs"; +import { errorMessage, isRecord, nonEmptyString } from "../shared/values"; +import { recordedFileOutput } from "./outputFiles"; + +const NO_OUTPUT_REASON = + "The run finished but returned no output. The notebook did not call dbutils.notebook.exit()."; +const NO_OUTPUTS_REASON = + "The run finished but returned a payload with no outputs. The notebook was published with nothing to return."; +const MISSING_OUTPUT_REASON = + "This output is published with the app, but the run did not return it. That happens when the run predates the output being added, or when the app and its runner have drifted apart."; + +const isReadableError = ( + value: unknown, +): value is { type: string; message: string } => + isRecord(value) && + typeof value.type === "string" && + typeof value.message === "string"; +const isSchemaField = ( + value: unknown, +): value is { name: string; type: string } => + isRecord(value) && + typeof value.name === "string" && + typeof value.type === "string"; + +type ClassifiedEntry = + | { + outcome: "malformed"; + index: number; + id: string | undefined; + reason: string; + } + | { + outcome: "computeError" | "result" | "html"; + index: number; + id: string | undefined; + payload: Record; + }; + +function classifyOutputEntry(entry: unknown, index: number): ClassifiedEntry { + if (!isRecord(entry)) { + return { + outcome: "malformed", + index, + id: undefined, + reason: "This output is not an object.", + }; + } + const id = nonEmptyString(entry.id); + const malformed = (reason: string): ClassifiedEntry => ({ + outcome: "malformed", + index, + id, + reason, + }); + if (entry.status === "error") { + return isReadableError(entry.error) + ? { outcome: "computeError", index, id, payload: entry } + : malformed( + "This output reported an error, but the error was not readable.", + ); + } + if (entry.status !== "ok") { + return malformed( + `This output has an unrecognised status: ${JSON.stringify(entry.status)}`, + ); + } + if (entry.kind === "html") { + return isHtmlOutput(entry.html) && + typeof entry.target_node === "string" && + entry.target_port === undefined + ? { outcome: "html", index, id, payload: entry } + : malformed("This HTML output is invalid or too large."); + } + if (!Array.isArray(entry.schema) || !Array.isArray(entry.rows)) { + return malformed( + "This output reported success but is missing the schema or rows.", + ); + } + if (!entry.schema.every(isSchemaField) || !entry.rows.every(isRecord)) { + return malformed( + "This output reported success but its schema or rows are malformed.", + ); + } + const fieldNames = entry.schema.map((field) => field.name); + // Rows are keyed by column name downstream, so blank or duplicate names silently collapse + // columns; reject the output rather than render data that has lost or merged a column. + if ( + fieldNames.some((name) => name === "") || + new Set(fieldNames).size !== fieldNames.length + ) { + return malformed( + "This output reported success but its columns have blank or duplicate names.", + ); + } + return { outcome: "result", index, id, payload: entry }; +} + +function classifyMultiRunOutput(raw: string | undefined): ( + | { outcome: "noPayload"; reason: string } + | { + outcome: "outputs"; + outputs: ClassifiedEntry[]; + runtimeNodeStates: Map; + } +) & + RunDiagnostics { + if (raw === undefined || raw.trim() === "") { + return { outcome: "noPayload", reason: NO_OUTPUT_REASON }; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + return { + outcome: "noPayload", + reason: `The run returned output that is not valid JSON: ${errorMessage(err)}`, + }; + } + if (!isRecord(parsed) || Object.keys(parsed).length === 0) { + return { + outcome: "noPayload", + reason: "The run returned an empty output object.", + }; + } + const diagnostics = parseRunDiagnostics(parsed); + if (!Array.isArray(parsed.outputs)) { + return { + ...diagnostics, + outcome: "noPayload", + reason: + "The run returned output with no outputs list, so it is not a multi-output payload.", + }; + } + const runtimeNodeStates = new Map(); + if (isRecord(parsed.runtimeNodeStates)) { + for (const [nodeId, state] of Object.entries(parsed.runtimeNodeStates)) { + if ( + nodeId !== "" && + isRecord(state) && + typeof state.disabled === "boolean" + ) { + runtimeNodeStates.set(nodeId, state.disabled); + } + } + } + if (parsed.outputs.length === 0 && runtimeNodeStates.size === 0) { + return { outcome: "noPayload", reason: NO_OUTPUTS_REASON, ...diagnostics }; + } + // Classify entries independently so one malformed output does not discard valid siblings. + return { + outcome: "outputs", + outputs: parsed.outputs.map((entry, index) => + classifyOutputEntry(entry, index), + ), + runtimeNodeStates, + ...diagnostics, + }; +} + +const joinSource = ( + node: string | undefined, + port: string | undefined, +): string | undefined => + node === undefined + ? undefined + : port === undefined + ? node + : `${node}.${port}`; + +type MatchOutcome = + | { outcome: "malformed"; reason: string } + | { outcome: "computeError" | "result"; payload: Record } + | { outcome: "html"; html: string } + | { outcome: "missing"; reason: string }; +type OutputSection = { + key: string; + id?: string; + title: string; + source?: string; + chartSpec?: Record; + files?: { path: string }[]; + fileBehavior?: FileOutputBehavior; + disabled?: boolean; + outcome: MatchOutcome; +}; + +function matchedOutcome(entry: ClassifiedEntry): MatchOutcome { + if (entry.outcome === "html") { + return isHtmlOutput(entry.payload.html) + ? { outcome: "html", html: entry.payload.html } + : { + outcome: "malformed", + reason: "This HTML output is invalid or too large.", + }; + } + return entry.outcome === "malformed" + ? { outcome: "malformed", reason: entry.reason } + : { outcome: entry.outcome, payload: entry.payload }; +} + +export function matchRunOutputs( + declared: AppOutputBlock[], + raw: string | undefined, +) { + const multi = classifyMultiRunOutput(raw); + const diagnostics = parseRunDiagnostics(multi); + if ( + multi.outcome !== "outputs" && + !declared.some((block) => block.fileOutput || block.disabled === true) + ) { + return { outcome: "noPayload", reason: multi.reason, ...diagnostics }; + } + const classified = multi.outcome === "outputs" ? multi.outputs : []; + const runtimeNodeStates = + multi.outcome === "outputs" ? multi.runtimeNodeStates : undefined; + const consumed = new Set(); + + // The run may include other nodes and ports, but only published output blocks are exposed. + const outputs: OutputSection[] = declared.map((output) => { + // Entries are keyed by the (node, port) of the cell's display(ctx[...]) line, not an output id. + // Payload empties read as undefined, so normalize declared empties the same way. + const declaredNode = output.nodeId === "" ? undefined : output.nodeId; + const declaredPort = output.port === "" ? undefined : output.port; + // A malformed entry has no (node, port), so it binds to its block by id and surfaces as malformed. + const matches = (entry: ClassifiedEntry) => + entry.outcome === "malformed" + ? entry.id !== undefined && entry.id === output.id + : nonEmptyString(entry.payload.target_node) === declaredNode && + (entry.payload.kind === "html") === (output.kind === "html") && + nonEmptyString(entry.payload.target_port) === declaredPort; + // Prefer an unconsumed entry; blocks published at the same (node, port) may share one. + const found = + classified.find( + (entry) => !consumed.has(entry.index) && matches(entry), + ) ?? classified.find(matches); + const source = joinSource(declaredNode, declaredPort); + const fileReceipt = output.fileOutput + ? recordedFileOutput(raw, output.nodeId, output.fileOutput) + : undefined; + const ownDisabled = + runtimeNodeStates?.get(output.nodeId) ?? output.disabled; + const groupDisabled = + output.parentGroupId === undefined + ? undefined + : runtimeNodeStates?.get(output.parentGroupId); + const disabled = + ownDisabled === true || groupDisabled === true + ? true + : ownDisabled === undefined && groupDisabled === undefined + ? undefined + : false; + const section = { + key: `declared:${output.id}`, + id: output.id, + title: outputBlockTitle(output), + ...(source === undefined ? {} : { source }), + ...(output.chartSpec === undefined + ? {} + : { chartSpec: output.chartSpec }), + ...(output.fileOutput ? { files: fileReceipt?.files ?? [] } : {}), + ...(fileReceipt?.behavior ? { fileBehavior: fileReceipt.behavior } : {}), + ...(disabled === undefined ? {} : { disabled }), + }; + if (found === undefined) { + return { + ...section, + outcome: { outcome: "missing", reason: MISSING_OUTPUT_REASON }, + }; + } + consumed.add(found.index); + return { ...section, outcome: matchedOutcome(found) }; + }); + + return { outcome: "outputs", outputs, ...diagnostics }; +} diff --git a/packages/appkit/src/plugins/designer/server/runParameters.ts b/packages/appkit/src/plugins/designer/server/runParameters.ts new file mode 100644 index 000000000..e57e2f211 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runParameters.ts @@ -0,0 +1,210 @@ +import { randomUUID } from "node:crypto"; + +import { isReservedParameter } from "../shared/appManifest"; +import { validateFileFormat } from "../shared/fileFormats"; +import { + FILE_OUTPUTS_PARAM, + OUTPUT_NAMESPACE_PARAM, + type FileOutputConfig, +} from "../shared/fileOutputs"; +import { + isValidMultiselectConfig, + isValidMultiselectValue, +} from "../shared/multiselect"; +import { + RUNNER_REVISION_PARAM, + type PublishedRunner, +} from "../shared/publishedRunner"; +import type { AppStorage } from "../shared/storageConfig"; +import { + UPLOAD_UNAVAILABLE, + UPLOAD_UNAVAILABLE_MESSAGE, +} from "../shared/storageConfig"; +import { + APP_VIEWER_PARAM, + UploadError, + resolveUpload, + type UploadStore, +} from "./fileUploads"; +import { APP_PARAMETERS_PARAM } from "./runAttribution"; + +export { isReservedParameter }; + +interface ParameterManifest { + runner?: PublishedRunner; + storage?: AppStorage; + blocks?: { type: string; nodeId?: string; fileOutput?: FileOutputConfig }[]; + parameters: { + name: string; + label: string; + type: string; + defaultValue: string; + choices?: string[]; + fileFormats?: string[]; + }[]; +} + +const DISPLAY_OUTPUTS_FOR_PARAM = "ld_display_outputs_for"; +const COLLECT_ROW_COUNTS_PARAM = "_lb_collect_row_counts"; + +function displayOutputsFor(manifest: ParameterManifest): string { + return Array.from( + new Set( + (manifest.blocks ?? []).flatMap((block) => + block.type === "output" && + typeof block.nodeId === "string" && + block.nodeId !== "" + ? [block.nodeId] + : [], + ), + ), + ).join(","); +} + +export async function resolveRunParameters( + manifest: ParameterManifest, + submitted: Record, + viewer: string | undefined, + store: UploadStore, +): Promise< + | { ok: true; params: Record } + | { ok: false; error: string; code?: typeof UPLOAD_UNAVAILABLE } +> { + // Pin hidden defaults as well as exposed inputs to the selected publication. A Job reset can + // race submission, so inheriting the Job's newer defaults would mix publication versions. + const params: Record = { ...manifest.runner?.parameters }; + const filenames = new Map(); + const fileOutputs = Object.fromEntries( + (manifest.blocks ?? []).flatMap((block) => + block.type === "output" && block.nodeId && block.fileOutput + ? [[block.nodeId, block.fileOutput]] + : [], + ), + ); + if (!viewer) + return { + ok: false, + error: "Sign in through Databricks Apps to run this app.", + }; + for (const parameter of manifest.parameters) { + if (isReservedParameter(parameter.name)) continue; + const raw = submitted[parameter.name]; + if (parameter.type === "multiselect") { + const resolved = raw === undefined ? parameter.defaultValue : raw; + if ( + !isValidMultiselectConfig(parameter.choices, parameter.defaultValue) || + !isValidMultiselectValue(resolved, parameter.choices ?? []) + ) { + return { + ok: false, + error: `"${parameter.label}" must contain only the offered choices.`, + }; + } + params[parameter.name] = resolved; + continue; + } + if ( + raw !== undefined && + raw !== null && + typeof raw !== "string" && + typeof raw !== "number" && + typeof raw !== "boolean" + ) { + return { + ok: false, + error: `"${parameter.label}" must be a string, number or boolean.`, + }; + } + const value = raw === undefined || raw === null ? "" : String(raw); + const resolved = value.trim() === "" ? parameter.defaultValue : value; + if (parameter.type === "file") { + if (!manifest.storage || !viewer) + return { + ok: false, + error: UPLOAD_UNAVAILABLE_MESSAGE, + code: UPLOAD_UNAVAILABLE, + }; + try { + const { path, upload } = await resolveUpload( + store, + manifest.storage, + viewer, + parameter.name, + resolved, + ); + const formatError = validateFileFormat( + upload.filename, + parameter.fileFormats, + ); + if (formatError) return { ok: false, error: formatError }; + params[parameter.name] = path; + filenames.set(parameter.name, upload.filename); + } catch (error) { + if (!(error instanceof UploadError)) { + return { + ok: false, + error: + "Could not verify the uploaded file. Try Run again after storage is available.", + }; + } + if (error.code === UPLOAD_UNAVAILABLE) + return { ok: false, error: error.message, code: UPLOAD_UNAVAILABLE }; + if ([400, 404, 409].includes(error.status)) { + return { + ok: false, + error: `${error.message} Upload the file again.`, + code: UPLOAD_UNAVAILABLE, + }; + } + return { ok: false, error: error.message }; + } + continue; + } + if ( + parameter.type === "dropdown" && + Array.isArray(parameter.choices) && + !parameter.choices.includes(resolved) + ) { + return { + ok: false, + error: `"${parameter.label}" must be one of the offered choices.`, + }; + } + params[parameter.name] = resolved; + } + // Override all server-owned controls so a partially completed republish cannot change this + // manifest's output policy through newer Job defaults. + const displayedNodes = displayOutputsFor(manifest); + if (displayedNodes !== "") { + params[DISPLAY_OUTPUTS_FOR_PARAM] = displayedNodes; + params[COLLECT_ROW_COUNTS_PARAM] = "true"; + } + // An empty override is intentional: a partially completed republish may leave newer + // file-writing Job defaults behind the current manifest. Never inherit that policy. + params[FILE_OUTPUTS_PARAM] = JSON.stringify(fileOutputs); + params._lb_file_output_volume = + manifest.runner?.parameters._lb_file_output_volume ?? ""; + if (manifest.runner) params[RUNNER_REVISION_PARAM] = manifest.runner.revision; + // The runtime adds an attempt ID and node ID beneath this server-owned submission namespace. + // Consumer parameters must not choose/reuse a previous run's artifact directory. + if (Object.keys(fileOutputs).length > 0) + params[OUTPUT_NAMESPACE_PARAM] = randomUUID(); + params[APP_VIEWER_PARAM] = viewer; + // Preserve labels and resolved values at submission time, including inputs later removed or + // renamed on republish. File reporting exposes the filename rather than its private storage path. + params[APP_PARAMETERS_PARAM] = JSON.stringify( + Object.fromEntries( + manifest.parameters + .filter(({ name }) => !isReservedParameter(name)) + .map((parameter) => [ + parameter.name, + { + label: parameter.label, + type: parameter.type, + value: filenames.get(parameter.name) ?? params[parameter.name], + }, + ]), + ), + ); + return { ok: true, params }; +} diff --git a/packages/appkit/src/plugins/designer/server/runRevision.ts b/packages/appkit/src/plugins/designer/server/runRevision.ts new file mode 100644 index 000000000..cac688d83 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runRevision.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; + +import type { FileOutputConfig } from "../shared/fileOutputs"; +import type { AppStorage } from "../shared/storageConfig"; +import { runJobParameters } from "./jobParameters"; + +export interface FileOutputManifest { + storage?: AppStorage; + blocks: { + type: string; + id?: string; + nodeId?: string; + port?: string; + fileOutput?: FileOutputConfig; + }[]; + parameters: { name: string }[]; +} + +export function manifestRevision(manifest: FileOutputManifest): string { + // Runner notebook identity is verified separately. Labels, ordering, defaults and publication + // timestamps do not change a completed write's destination or ownership. + const outputs = manifest.blocks + .filter((block) => block.type === "output") + .map((block) => ({ + id: block.id ?? "", + nodeId: block.nodeId, + port: block.port, + ...(block.fileOutput + ? { + fileOutput: { + volumes: [...new Set(block.fileOutput.volumes)].sort(), + }, + } + : {}), + })) + .sort((left, right) => + left.id < right.id ? -1 : left.id > right.id ? 1 : 0, + ); + const revision = createHash("sha256") + .update( + JSON.stringify({ + storage: manifest.storage && { + volume: manifest.storage.volume, + path: manifest.storage.path, + }, + parameters: manifest.parameters + .map((parameter) => parameter.name) + .sort(), + outputs, + }), + ) + .digest("hex"); + return `file-outputs-v1:${revision}`; +} + +// Old converter runs can still be present in this Job's history after republishing. +export const isLegacyExportRun = (run: unknown) => + typeof runJobParameters(run)._lb_export_request === "string"; diff --git a/packages/appkit/src/plugins/designer/server/runtime.ts b/packages/appkit/src/plugins/designer/server/runtime.ts new file mode 100644 index 000000000..214f5d5c5 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/runtime.ts @@ -0,0 +1,980 @@ +import type { Request, Response } from "express"; +import type { IAppRouter, RouteConfig, TelemetryOptions } from "shared"; + +import { FilesConnector } from "../../../connectors/files"; +import { createLogger } from "../../../logging/logger"; +import { + createWorkspaceClient, + type WorkspaceClient, +} from "../../../workspace-client"; +import { + appManifestSchema, + isReservedParameter, + type AppManifest, + type AppOutputBlock, +} from "../shared/appManifest"; +import { APP_REVISION_PARAM } from "../shared/fileOutputs"; +import { + APP_UPDATING_MESSAGE, + isPublishedRunner, +} from "../shared/publishedRunner"; +import { UPLOAD_REFERENCE } from "../shared/storageConfig"; +import { + errorMessage, + idString, + isRecord, + nonEmptyString, + positiveInt, +} from "../shared/values"; +import { isSameOriginBrowserRequest } from "./browserRequestOrigin"; +import { + exportedModelToRunPayload, + findNotebookModelValue, +} from "./exportedRunOutput"; +import { canAccessRun, viewerKey } from "./fileUploads"; +import { runJobParameters } from "./jobParameters"; +import { registerOutputFileRoutes } from "./outputFiles"; +import { createOutputFileStore } from "./outputFileStore"; +import { runAttribution } from "./runAttribution"; +import { scanVisibleRuns, type VisibleRunPage } from "./runHistory"; +import { matchRunOutputs } from "./runOutputs"; +import { resolveRunParameters } from "./runParameters"; +import { isLegacyExportRun, manifestRevision } from "./runRevision"; +import { registerUploadRoutes } from "./uploads"; +import { createUploadStore } from "./uploadStore"; + +// Publishing writes the manifest beside the runner notebook in the app's publisher-owned folder, outside +// the git-backed source and client/dist, so it is never served publicly. +const logger = createLogger("designer"); +const MANIFEST_FILENAME = "designerApp.json"; +// SKIPPED and INTERNAL_ERROR are terminal even when no task ran. +const TERMINAL_LIFE_CYCLE_STATES = new Set([ + "TERMINATED", + "SKIPPED", + "INTERNAL_ERROR", +]); + +type RunSummary = { + jobRunId: string; + taskRunId?: string; + endTime?: number; + startTime?: number; + setupDurationMs?: number; + executionDurationMs?: number; + runPageUrl?: string; + resultState?: string; + lifeCycleState?: string; + parameters?: Record; + parameterDisplayValues?: Record; +}; + +export function createDesignerRuntime( + client: WorkspaceClient, + telemetry?: TelemetryOptions, +) { + const JOB_ID = process.env.DATABRICKS_JOB_ID?.trim() || undefined; + const APP_ID = + process.env.DATABRICKS_APP_ID?.trim() || + process.env.DATABRICKS_CLIENT_ID?.trim(); + const appUrl = process.env.DATABRICKS_APP_URL; + const connector = new FilesConnector({ telemetry }); + const wsClient = () => client; + const appKitUploadStore = ( + storage: Parameters[2], + ) => createUploadStore(client, connector, storage); + const appKitOutputFileStore = (volume: string) => + createOutputFileStore(client, connector, volume); + + // Runner/access checks consume the Jobs REST shape, including recorded parameter maps. + async function readWorkspaceJson( + path: string, + query: Record, + ): Promise { + const response = await wsClient().request({ + method: "GET", + path, + query: Object.fromEntries( + Object.entries(query) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => [key, String(value)]), + ), + }); + return response.json(); + } + + const readJob = () => + readWorkspaceJson("/api/2.2/jobs/get", { job_id: JOB_ID }); + const readRun = (runId: string | number) => + readWorkspaceJson("/api/2.2/jobs/runs/get", { run_id: runId }); + + function jobNotebookPaths(job: unknown): (string | undefined)[] { + const tasks = + isRecord(job) && + isRecord(job.settings) && + Array.isArray(job.settings.tasks) + ? job.settings.tasks + : []; + return tasks.map((task: unknown) => + isRecord(task) && isRecord(task.notebook_task) + ? nonEmptyString(task.notebook_task.notebook_path) + : undefined, + ); + } + + const READ_RETRY_ATTEMPTS = 3; + const READ_RETRY_BACKOFF_MS = 1000; + + const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); + + // Only reads get application-level retries; writes rely on the SDK transport. + async function retryRead(read: () => Promise): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= READ_RETRY_ATTEMPTS; attempt += 1) { + try { + return await read(); + } catch (err) { + lastError = err; + if (attempt < READ_RETRY_ATTEMPTS) { + await sleep(READ_RETRY_BACKOFF_MS); + } + } + } + throw lastError; + } + + // Polled run-status reads reuse the cached manifest; other routes pass forceFresh, so a republish + // takes effect without a redeploy. + let cachedManifest: AppManifest | undefined; + const loadManifest = async ( + forceFresh = false, + ): Promise => { + if (!forceFresh && cachedManifest !== undefined) { + return cachedManifest; + } + const manifest = await readManifest(); + // A failed fresh read must not keep stale execution or file policies active. + if (manifest !== undefined || forceFresh) { + cachedManifest = manifest; + } + return manifest; + }; + + function manifestFolderFromNotebookPath(notebookPath: string): string { + const lastSlash = notebookPath.lastIndexOf("/"); + return lastSlash <= 0 ? notebookPath : notebookPath.slice(0, lastSlash); + } + + // All immutable runner revisions share the App folder, so resolve it from the bound Job once. + let cachedManifestFolder: string | undefined; + + async function resolveManifestFolder(): Promise { + if (cachedManifestFolder !== undefined) { + return cachedManifestFolder; + } + if (JOB_ID === undefined) { + return undefined; + } + try { + const notebookPath = jobNotebookPaths(await readJob()).find( + (path) => path !== undefined, + ); + if (notebookPath === undefined) { + logger.error( + "Could not find the runner notebook path on job", + JOB_ID, + "to locate the app manifest", + ); + return undefined; + } + cachedManifestFolder = manifestFolderFromNotebookPath(notebookPath); + return cachedManifestFolder; + } catch (err) { + logger.error( + "Could not read the runner job to locate the app manifest", + err, + ); + return undefined; + } + } + + async function readManifest() { + const folder = await resolveManifestFolder(); + if (folder === undefined) { + return undefined; + } + const manifestPath = `${folder}/${MANIFEST_FILENAME}`; + try { + const exported = await readWorkspaceJson("/api/2.0/workspace/export", { + path: manifestPath, + format: "SOURCE", + }); + if (!isRecord(exported) || typeof exported.content !== "string") { + return undefined; + } + return parseManifest( + Buffer.from(exported.content, "base64").toString("utf8"), + ); + } catch (err) { + logger.error("Could not read the app manifest from", manifestPath, err); + return undefined; + } + } + + function parseManifest(raw: string): AppManifest | undefined { + try { + return appManifestSchema.safeParse(JSON.parse(raw)).data; + } catch { + return undefined; + } + } + + const requestViewer = (req: Request) => + viewerKey(req.get("x-forwarded-user"), JOB_ID); + + async function accessibleRun( + run: unknown, + req: Request, + hydrate = false, + ): Promise | undefined> { + if (!runBelongsToJob(run, JOB_ID)) return undefined; + const manifest = await loadManifest(); + if (!manifest) return undefined; + let detail = run; + if (hydrate && isRecord(run) && !Array.isArray(run.job_parameters)) { + const id = idString(run.run_id); + if (!id) return undefined; + detail = await readRun(id); + } + return isRecord(detail) && + !isLegacyExportRun(detail) && + runBelongsToJob(detail, JOB_ID) && + canAccessRun(detail, requestViewer(req)) + ? detail + : undefined; + } + + // Guard every client-supplied run ID before reading output or cancelling it. + async function readViewerRun(req: Request, res: Response) { + const jobRunId = idString(req.params.jobRunId); + if (jobRunId === undefined) { + res.status(400).json({ error: "jobRunId must be a positive integer." }); + return undefined; + } + let run; + try { + run = await retryRead(() => readRun(jobRunId)); + } catch (err) { + res.status(502).json({ + error: `Could not read run ${jobRunId}: ${errorMessage(err)}`, + }); + return undefined; + } + const visibleRun = await accessibleRun(run, req); + if (visibleRun === undefined) { + res + .status(404) + .json({ error: `Run ${jobRunId} is not a run of this app.` }); + return undefined; + } + return { jobRunId, run: visibleRun }; + } + + async function declaredOutputs() { + return ( + (await loadManifest())?.blocks.filter( + (block): block is AppOutputBlock => block.type === "output", + ) ?? [] + ); + } + + // The runs/export CODE view carries the displayed command results. + async function readRunExport(taskRunId: string): Promise { + const exported = await retryRead(() => + wsClient().jobs.exportRun({ + runId: BigInt(taskRunId), + viewsToExport: "CODE", + }), + ); + const views = + isRecord(exported) && Array.isArray(exported.views) ? exported.views : []; + // Probe with the decoder rather than a substring: a base64-wrapped view never contains the literal marker. + const view: unknown = views.find( + (candidate) => + isRecord(candidate) && + findNotebookModelValue(candidate.content) !== undefined, + ); + return isRecord(view) ? view.content : undefined; + } + + interface CompletedTaskOutput { + payload?: string; + failureReason?: string; + readError?: string; + } + + const COMPLETED_RESULT_CACHE_LIMIT = 20; + const COMPLETED_RESULT_CACHE_TTL_MS = 5 * 60 * 1000; + const completedTaskOutputs = new Map< + string, + { expiresAt: number; value: Promise } + >(); + + async function readCompletedTaskOutput( + taskRunId: string, + failed = false, + ): Promise { + const key = `${taskRunId}:${failed}`; + const now = Date.now(); + const cached = completedTaskOutputs.get(key); + if (cached && cached.expiresAt > now) return cached.value; + for (const [id, entry] of completedTaskOutputs) { + if (entry.expiresAt <= now) completedTaskOutputs.delete(id); + } + const value = Promise.allSettled([ + readRunExport(taskRunId), + failed + ? retryRead(() => + wsClient().jobs.getRunOutput({ runId: BigInt(taskRunId) }), + ) + : Promise.resolve(undefined), + ]).then(([exported, taskOutput]) => { + const taskError = + taskOutput.status === "fulfilled" ? taskOutput.value : undefined; + const taskErrorTexts = [taskError?.error, taskError?.errorTrace].filter( + (text): text is string => typeof text === "string", + ); + const html = exported.status === "fulfilled" ? exported.value : undefined; + // Retention gaps and transient read errors can recover; do not retain them for the full cache TTL. + if ( + exported.status === "rejected" || + taskOutput.status === "rejected" || + html === undefined + ) { + const entry = completedTaskOutputs.get(key); + if (entry?.value === value) entry.expiresAt = Date.now() + 5000; + } + return { + payload: exportedModelToRunPayload(html, taskErrorTexts), + failureReason: + nonEmptyString(taskError?.error) ?? + nonEmptyString(taskError?.errorTrace), + ...(exported.status === "rejected" + ? { readError: errorMessage(exported.reason) } + : {}), + }; + }); + // Cache only after access checks, and share the in-flight read across concurrent viewers/pollers. + completedTaskOutputs.set(key, { + expiresAt: now + COMPLETED_RESULT_CACHE_TTL_MS, + value, + }); + while (completedTaskOutputs.size > COMPLETED_RESULT_CACHE_LIMIT) { + const oldest = completedTaskOutputs.keys().next().value; + if (oldest === undefined) break; + completedTaskOutputs.delete(oldest); + } + return value; + } + + async function readRunOutputPayload( + taskRunId: string, + ): Promise { + return (await readCompletedTaskOutput(taskRunId)).payload; + } + + async function readRunResult(taskRunId: string | undefined, failed = false) { + if (taskRunId === undefined) { + return { + outcome: "noPayload", + reason: "The run reported no task, so it has no output to read.", + }; + } + try { + // Jobs requires the task run ID here, not the job run ID. + const { payload, failureReason, readError } = + await readCompletedTaskOutput(taskRunId, failed); + const result = matchRunOutputs(await declaredOutputs(), payload); + return { + ...result, + ...(failureReason ? { failureReason } : {}), + ...(result.outcome === "noPayload" && readError + ? { reason: `Could not read the run output: ${readError}` } + : {}), + }; + } catch (err) { + return { + outcome: "noPayload", + reason: `Could not read the run output: ${errorMessage(err)}`, + }; + } + } + + const LAST_RUN_PAGE_SIZE = 25; + const SUCCESSFUL_RESULT_STATES = new Set([ + "SUCCESS", + "SUCCESS_WITH_FAILURES", + ]); + + type ListRunsQuery = { + active_only?: boolean; + completed_only?: boolean; + expand_tasks?: boolean; + limit: number; + page_token?: string; + }; + + async function collectVisibleRuns( + req: Request, + options: ListRunsQuery, + isComplete: (visible: Record[]) => boolean, + ): Promise { + return scanVisibleRuns({ + pageToken: options.page_token, + // The SDK iterator consumes page tokens internally; use its authenticated transport to + // return a continuation when this request's scan budget is exhausted. + readPage: (pageToken) => + readWorkspaceJson("/api/2.2/jobs/runs/list", { + ...options, + job_id: JOB_ID, + page_token: pageToken, + }), + accessible: (run) => accessibleRun(run, req, true), + isComplete, + }); + } + + function isSuccessfulRun(run: unknown): run is Record { + return ( + isRecord(run) && + idString(run.run_id) !== undefined && + isRecord(run.state) && + typeof run.state.result_state === "string" && + SUCCESSFUL_RESULT_STATES.has(run.state.result_state) + ); + } + + const startedAt = (run: Record) => + positiveInt(run.start_time) ?? 0; + const newestFirst = ( + a: Record, + b: Record, + ) => startedAt(b) - startedAt(a); + + function selectLastSuccessfulRun( + runs: Record[], + ): Record | undefined { + return runs.filter(isSuccessfulRun).sort(newestFirst)[0]; + } + + function selectActiveRun( + runs: Record[], + ): Record | undefined { + return runs + .filter((run) => idString(run.run_id) !== undefined) + .sort(newestFirst)[0]; + } + + const lifeCycleStateOf = (run: Record) => + isRecord(run.state) + ? nonEmptyString(run.state.life_cycle_state) + : undefined; + + const resultStateOf = (run: Record) => + isRecord(run.state) ? nonEmptyString(run.state.result_state) : undefined; + + function summarizeLastRun( + run: Record, + ): RunSummary | undefined { + const jobRunId = idString(run.run_id); + if (jobRunId === undefined) { + return undefined; + } + const firstTask = (Array.isArray(run.tasks) ? run.tasks : []).find( + isRecord, + ); + const summary: RunSummary = { jobRunId }; + const taskRunId = idString(firstTask?.run_id); + if (taskRunId !== undefined) summary.taskRunId = taskRunId; + const endTime = positiveInt(run.end_time); + if (endTime !== undefined) summary.endTime = endTime; + const startTime = positiveInt(run.start_time); + if (startTime !== undefined) summary.startTime = startTime; + const setupDurationMs = positiveInt(run.setup_duration); + if (setupDurationMs !== undefined) + summary.setupDurationMs = setupDurationMs; + const executionDurationMs = positiveInt(run.execution_duration); + if (executionDurationMs !== undefined) + summary.executionDurationMs = executionDurationMs; + if (typeof run.run_page_url === "string" && run.run_page_url !== "") + summary.runPageUrl = run.run_page_url; + return summary; + } + + function runParameterData( + run: Record, + ): Pick | undefined { + if (!Array.isArray(run.job_parameters)) { + return undefined; + } + const params: Record = {}; + const displayValues: Record = {}; + for (const [name, value] of Object.entries(runJobParameters(run))) { + if ( + isReservedParameter(name) || + !cachedManifest?.parameters.some((parameter) => parameter.name === name) + ) { + continue; + } + const isFile = cachedManifest?.parameters.some( + (parameter) => parameter.name === name && parameter.type === "file", + ); + if (isFile) { + const parts = value.split("/"); + const reference = `upload:${parts.at(-2)}`; + if (UPLOAD_REFERENCE.test(reference)) { + params[name] = reference; + const filename = parts.at(-1); + if (filename) displayValues[name] = filename; + } else { + params[name] = ""; + } + } else { + params[name] = value; + } + } + return { + parameters: params, + ...(Object.keys(displayValues).length === 0 + ? {} + : { parameterDisplayValues: displayValues }), + }; + } + + function runBelongsToJob(run: unknown, jobId: string | undefined) { + if (jobId === undefined || !isRecord(run)) { + return false; + } + return String(run.job_id) === jobId; + } + + const RUN_HISTORY_DEFAULT_WINDOW = 100; + const RUN_HISTORY_MAX_WINDOW = 200; + // listRuns caps page size at 26. + const RUN_HISTORY_PAGE_SIZE = 25; + + function summarizeHistoryRun(run: Record) { + const summary = summarizeLastRun(run); + if (summary === undefined) { + return undefined; + } + delete summary.taskRunId; + const resultState = resultStateOf(run); + if (resultState !== undefined) summary.resultState = resultState; + const lifeCycleState = lifeCycleStateOf(run); + if (lifeCycleState !== undefined) summary.lifeCycleState = lifeCycleState; + const parameterData = runParameterData(run); + if (parameterData !== undefined) Object.assign(summary, parameterData); + return summary; + } + + function summarizeRunHistory(page: VisibleRunPage) { + return { + entries: [...page.runs] + .sort(newestFirst) + .flatMap((run) => summarizeHistoryRun(run) ?? []), + hasMore: page.nextPageToken !== undefined, + nextPageToken: page.nextPageToken, + }; + } + + function resolveHistoryWindow(raw: unknown) { + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + return RUN_HISTORY_DEFAULT_WINDOW; + } + return Math.min(parsed, RUN_HISTORY_MAX_WINDOW); + } + + function registerRoutes( + app: IAppRouter, + route: (config: RouteConfig) => void, + ) { + // These responses contain per-viewer filenames and results, never shared cache entries. + app.use((req, res, next) => { + res.setHeader("Cache-Control", "no-store"); + res.setHeader("X-Content-Type-Options", "nosniff"); + res.setHeader("Cross-Origin-Resource-Policy", "same-origin"); + if ( + !isSameOriginBrowserRequest({ + origin: req.get("origin"), + // Apps ingress can rewrite Host; the runtime URL names the public origin. + appUrl: appUrl || `${req.protocol}://${req.get("host")}`, + fetchSite: req.get("sec-fetch-site"), + }) + ) { + res + .status(403) + .json({ error: "Open the App directly to use its APIs." }); + return; + } + next(); + }); + + registerOutputFileRoutes(route, { + jobId: JOB_ID, + manifest: () => loadManifest(true), + viewer: requestViewer, + notebookPath: async () => { + const paths = jobNotebookPaths(await readJob()); + return paths.length === 1 ? paths[0] : undefined; + }, + getRun: readRun, + readPayload: readRunOutputPayload, + store: appKitOutputFileStore, + report: (error) => logger.error("Output file download failed", error), + }); + + registerUploadRoutes(route, { + manifest: () => loadManifest(true), + viewer: requestViewer, + store: appKitUploadStore, + report: (error) => logger.error("Uploaded file download failed", error), + }); + + route({ + name: "config", + method: "get", + path: "/config", + handler: async (_req, res) => { + try { + const manifest = await loadManifest(true); + if (manifest === undefined) { + res.json({ + manifest: null, + runnable: false, + notRunnableReason: "noManifest", + }); + return; + } + // Hidden runner defaults are execution policy, not consumer-visible configuration. + const { runner: _runner, ...publicManifest } = manifest; + if (JOB_ID === undefined) { + res.json({ + manifest: publicManifest, + runnable: false, + notRunnableReason: "noJob", + }); + return; + } + const runnable = isPublishedRunner(manifest.runner, await readJob()); + res.json({ + manifest: publicManifest, + runnable, + ...(runnable ? {} : { notRunnableReason: "updating" }), + }); + } catch (err) { + logger.error("Designer request failed: %O", err); + res + .status(500) + .json({ error: "The app could not read its own configuration." }); + } + }, + }); + + route({ + name: "lastRun", + method: "get", + path: "/last-run", + handler: async (req, res) => { + if (JOB_ID === undefined) { + res.json({ status: "noJob" }); + return; + } + + let listed: VisibleRunPage; + try { + // completed_only includes failures, so page until the newest success is in hand: a streak + // of recent failures must not hide an older successful run. + listed = await retryRead(() => + collectVisibleRuns( + req, + { + completed_only: true, + expand_tasks: true, + limit: LAST_RUN_PAGE_SIZE, + page_token: nonEmptyString(req.query.cursor), + }, + (visible) => visible.some(isSuccessfulRun), + ), + ); + } catch (err) { + logger.error("Designer request failed: %O", err); + res.json({ + status: "unavailable", + reason: `Could not read this app's run history: ${errorMessage(err)}`, + }); + return; + } + + let active; + try { + // Trust active_only instead of duplicating Jobs lifecycle-state logic. Not retried: this + // best-effort signal must not delay primary content. + const activeRun = selectActiveRun( + ( + await collectVisibleRuns( + req, + { + active_only: true, + expand_tasks: true, + limit: LAST_RUN_PAGE_SIZE, + }, + (visible) => visible.length > 0, + ) + ).runs, + ); + const activeSummary = + activeRun === undefined ? undefined : summarizeLastRun(activeRun); + if (activeRun !== undefined && activeSummary !== undefined) { + const lifeCycleState = lifeCycleStateOf(activeRun); + active = { + run: activeSummary, + ...runParameterData(activeRun), + ...(lifeCycleState === undefined ? {} : { lifeCycleState }), + }; + } + } catch (err) { + logger.error("Designer request failed: %O", err); + } + + const run = selectLastSuccessfulRun(listed.runs); + const summary = run === undefined ? undefined : summarizeLastRun(run); + if (run === undefined || summary === undefined) { + res.json({ + status: listed.nextPageToken === undefined ? "none" : "searching", + nextPageToken: listed.nextPageToken, + ...(active === undefined ? {} : { active }), + }); + return; + } + summary.resultState = resultStateOf(run); + res.json({ + status: "found", + run: summary, + ...runParameterData(run), + ...(active === undefined ? {} : { active }), + result: await readRunResult(summary.taskRunId), + }); + }, + }); + + route({ + name: "run", + method: "post", + path: "/run", + handler: async (req, res) => { + try { + const manifest = await loadManifest(true); + if (manifest === undefined) { + res.status(409).json({ + error: + "This app has no published configuration, so it cannot run.", + }); + return; + } + if (JOB_ID === undefined) { + res.status(409).json({ + error: + "This app is not connected to a job yet, so it cannot run.", + }); + return; + } + if (!APP_ID) { + res.status(409).json({ + error: + "This app has no runtime App ID, so it cannot attribute Job runs.", + }); + return; + } + if (!isPublishedRunner(manifest.runner, await readJob())) { + res.status(409).json({ error: APP_UPDATING_MESSAGE }); + return; + } + const attribution = await runAttribution(req, APP_ID, async (token) => + createWorkspaceClient({ + host: await wsClient().getHost(), + token, + }).currentUser.me({}), + ); + const userName = attribution?._lb_app_user_email; + if (!attribution || !userName) { + res.status(401).json({ + error: "Sign in through Databricks Apps to run this app.", + }); + return; + } + const submitted = + isRecord(req.body) && isRecord(req.body.params) + ? req.body.params + : {}; + const resolved = await resolveRunParameters( + manifest, + submitted, + requestViewer(req), + appKitUploadStore(manifest.storage), + ); + if (!resolved.ok) { + res.status(400).json({ + error: resolved.error, + ...(resolved.code ? { code: resolved.code } : {}), + }); + return; + } + + // The SDK can retry transient responses. Reuse the submission ID so those attempts + // cannot start duplicate compute; the browser does not replay submissions. + resolved.params[APP_REVISION_PARAM] = manifestRevision(manifest); + const run = await wsClient().jobs.runNow({ + jobId: BigInt(JOB_ID), + idempotencyToken: attribution._lb_app_submission_id, + jobParameters: { ...resolved.params, ...attribution }, + }); + const jobRunId = idString(run?.runId); + if (jobRunId === undefined) { + res.status(502).json({ + error: + "The platform accepted the request but returned no run id.", + }); + return; + } + res.json({ jobRunId }); + } catch (err) { + logger.error("Designer request failed: %O", err); + res + .status(502) + .json({ error: `Could not start the run: ${errorMessage(err)}` }); + } + }, + }); + + route({ + name: "runStatus", + method: "get", + path: "/run/:jobRunId", + handler: async (req, res) => { + const found = await readViewerRun(req, res); + if (found === undefined) return; + const { jobRunId, run } = found; + + const lifeCycleState = lifeCycleStateOf(run); + const resultState = resultStateOf(run); + // A run carrying a result_state has finished even if its lifecycle value is absent or one + // this list does not know yet, so treat that as terminal rather than polling it forever. + const terminal = + (lifeCycleState != null && + TERMINAL_LIFE_CYCLE_STATES.has(lifeCycleState)) || + resultState != null; + const snapshot = { + jobRunId, + taskRunId: summarizeLastRun(run)?.taskRunId, + lifeCycleState, + resultState, + stateMessage: isRecord(run.state) + ? nonEmptyString(run.state.state_message) + : undefined, + setupDurationMs: run.setup_duration, + executionDurationMs: run.execution_duration, + runPageUrl: run.run_page_url, + terminal, + ...runParameterData(run), + }; + + if (!terminal) { + res.json(snapshot); + return; + } + + const failed = + resultState === undefined || + !SUCCESSFUL_RESULT_STATES.has(resultState); + res.json({ + ...snapshot, + result: await readRunResult(snapshot.taskRunId, failed), + }); + }, + }); + + route({ + name: "cancelRun", + method: "delete", + path: "/run/:jobRunId", + handler: async (req, res) => { + const found = await readViewerRun(req, res); + if (found === undefined) return; + try { + // Polling settles cancellation from the run's own state. + await wsClient().jobs.cancelRun({ runId: BigInt(found.jobRunId) }); + res.json({ cancelled: true }); + } catch (err) { + logger.error("Designer request failed: %O", err); + res.status(502).json({ + error: `Could not cancel run ${found.jobRunId}: ${errorMessage(err)}`, + }); + } + }, + }); + + route({ + name: "runs", + method: "get", + path: "/runs", + handler: async (req, res) => { + if (JOB_ID === undefined) { + res.json({ status: "noJob" }); + return; + } + const windowSize = resolveHistoryWindow(req.query?.window); + + let history: VisibleRunPage; + try { + history = await retryRead(() => + collectVisibleRuns( + req, + { + completed_only: true, + limit: RUN_HISTORY_PAGE_SIZE, + page_token: nonEmptyString(req.query.cursor), + }, + (visible) => visible.length >= windowSize, + ), + ); + } catch (err) { + logger.error("Designer request failed: %O", err); + res.json({ + status: "unavailable", + reason: `Could not read this app's run history: ${errorMessage(err)}`, + }); + return; + } + + const page = summarizeRunHistory(history); + res.json({ + status: "found", + runs: page.entries, + hasMore: page.hasMore, + nextPageToken: page.nextPageToken, + window: windowSize, + }); + }, + }); + } + return { + registerRoutes, + close() { + cachedManifest = undefined; + cachedManifestFolder = undefined; + completedTaskOutputs.clear(); + }, + }; +} diff --git a/packages/appkit/src/plugins/designer/server/storageVolume.ts b/packages/appkit/src/plugins/designer/server/storageVolume.ts new file mode 100644 index 000000000..8d84a0079 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/storageVolume.ts @@ -0,0 +1,6 @@ +export function parseStorageFileSize( + contentLength: unknown, +): number | undefined { + const size = Number(contentLength); + return Number.isSafeInteger(size) && size >= 0 ? size : undefined; +} diff --git a/packages/appkit/src/plugins/designer/server/uploadStore.ts b/packages/appkit/src/plugins/designer/server/uploadStore.ts new file mode 100644 index 000000000..7b3bf30f3 --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/uploadStore.ts @@ -0,0 +1,81 @@ +import type { FilesConnector } from "../../../connectors/files"; +import { ApiError, type WorkspaceClient } from "../../../workspace-client"; +import type { AppStorage } from "../shared/storageConfig"; +import { + UPLOAD_UNAVAILABLE, + UPLOAD_UNAVAILABLE_MESSAGE, +} from "../shared/storageConfig"; +import { uploadStoragePath } from "../shared/storageConfig"; +import { UploadError, type UploadStore } from "./fileUploads"; +import { parseStorageFileSize } from "./storageVolume"; + +export function createUploadStore( + client: WorkspaceClient, + connector: FilesConnector, + config: AppStorage | undefined, +): UploadStore { + const access = async ( + path: string, + operation: () => Promise, + ): Promise => { + try { + if (!config) + throw new UploadError(409, "File uploads are not configured."); + if (!path.startsWith(`${uploadStoragePath(config)}/`)) + throw new Error("Invalid upload storage destination."); + return await operation(); + } catch (error) { + if (error instanceof UploadError) throw error; + const missing = error instanceof ApiError && error.statusCode === 404; + throw new UploadError( + missing ? 404 : 502, + missing + ? UPLOAD_UNAVAILABLE_MESSAGE + : "Could not access upload storage. Check the app volume resource and permissions.", + { cause: error, ...(missing ? { code: UPLOAD_UNAVAILABLE } : {}) }, + ); + } + }; + return { + mkdir: (path) => + access(path, () => connector.createDirectory(client, path)), + put: (path, bytes) => + access(path, () => + connector.upload(client, path, Buffer.from(bytes), { + overwrite: false, + }), + ), + putStream: (path, stream) => + access(path, () => + connector.upload(client, path, stream, { overwrite: false }), + ), + read: (path) => + access(path, async () => { + const contents = await connector.read(client, path, { + maxSize: 16 * 1024, + }); + try { + const record: unknown = JSON.parse(contents); + return record; + } catch { + throw new UploadError(409, "The upload record is unreadable."); + } + }), + download: (path) => + access(path, async () => { + const result = await connector.download(client, path); + if (!result.contents) + throw new UploadError( + 502, + "The uploaded file has no readable contents.", + ); + return result.contents; + }), + size: (path) => + access(path, async () => { + const metadata = await connector.metadata(client, path); + return parseStorageFileSize(metadata.contentLength); + }), + delete: (path) => access(path, () => connector.delete(client, path)), + }; +} diff --git a/packages/appkit/src/plugins/designer/server/uploads.ts b/packages/appkit/src/plugins/designer/server/uploads.ts new file mode 100644 index 000000000..007aa56ba --- /dev/null +++ b/packages/appkit/src/plugins/designer/server/uploads.ts @@ -0,0 +1,194 @@ +import { Readable } from "node:stream"; + +import type { Request, Response } from "express"; +import type { RouteConfig } from "shared"; + +import { validateFileFormat } from "../shared/fileFormats"; +import { + MAX_UPLOAD_SIZE_LABEL, + UPLOAD_REFERENCE, + UPLOAD_UNAVAILABLE, + type AppStorage, +} from "../shared/storageConfig"; +import { streamFileDownload } from "./fileDownloads"; +import { + UploadError, + resolveUpload, + saveUploadStream, + type UploadStore, +} from "./fileUploads"; + +interface UploadManifest { + storage?: AppStorage; + parameters: { name: string; type: string; fileFormats?: string[] }[]; +} + +interface UploadDependencies { + manifest(): Promise; + viewer(req: Request): string | undefined; + store(storage: AppStorage): UploadStore; + report: (error: unknown) => void; +} + +const fileParameter = (manifest: UploadManifest | undefined, name: string) => + manifest?.parameters.find( + (parameter) => parameter.name === name && parameter.type === "file", + ); + +function sendUploadError(res: Response, error: unknown, fallback: string) { + res + .status(error instanceof UploadError ? error.status : 502) + .json( + error instanceof UploadError + ? { ...(error.code ? { code: error.code } : {}), error: error.message } + : { error: fallback }, + ); +} + +export function registerUploadRoutes( + route: (config: RouteConfig) => void, + deps: UploadDependencies, +) { + let activeUploads = 0; + + route({ + name: "downloadUpload", + method: "get", + path: "/uploads/:parameterName/:reference/download", + handler: async (req, res) => { + res.setHeader("Cache-Control", "no-store"); + try { + if (req.method !== "GET") + throw new UploadError(405, "Use GET to download the file."); + if (req.get("range")) + throw new UploadError( + 416, + "Partial downloads are not supported. Retry the full download.", + ); + const viewer = deps.viewer(req); + if (!viewer) + throw new UploadError( + 401, + "Sign in through Databricks Apps to download files.", + ); + const manifest = await deps.manifest(); + const parameter = fileParameter(manifest, req.params.parameterName); + if (!parameter || !manifest?.storage) { + throw new UploadError( + 404, + "This file parameter is no longer available. Review the app inputs.", + { + code: UPLOAD_UNAVAILABLE, + }, + ); + } + const reference = String(req.params.reference); + if (!UPLOAD_REFERENCE.test(reference)) + throw new UploadError(400, "The upload reference is invalid."); + const store = deps.store(manifest.storage); + const { path, upload } = await resolveUpload( + store, + manifest.storage, + viewer, + parameter.name, + reference, + ); + await streamFileDownload( + res, + await store.download(path), + upload.filename, + upload.size, + deps.report, + ); + } catch (error) { + if (!(error instanceof UploadError)) deps.report(error); + if (res.headersSent) { + res.destroy(error instanceof Error ? error : undefined); + } else { + sendUploadError( + res, + error, + "Could not download the uploaded file. Check the app volume resource and permissions.", + ); + } + } + }, + }); + + route({ + name: "upload", + method: "post", + path: "/uploads/:parameterName", + skipBodyParsing: true, + handler: async (req, res) => { + let admitted = false; + try { + const manifest = await deps.manifest(); + const parameter = fileParameter(manifest, req.params.parameterName); + if (!parameter || !manifest?.storage) { + res.status(404).json({ + error: + "This file parameter is no longer available. Review the app inputs.", + code: UPLOAD_UNAVAILABLE, + }); + return; + } + const viewer = deps.viewer(req); + if (!viewer) { + res.status(401).json({ + error: "Sign in through Databricks Apps to upload files.", + }); + return; + } + const store = deps.store(manifest.storage); + if (req.get("content-type") !== "application/octet-stream") + throw new UploadError(415, "Upload the file as an octet stream."); + if (activeUploads >= 4) + throw new UploadError(429, "Uploads are busy. Try again shortly."); + const contentLength = req.get("content-length"); + if (contentLength !== undefined && !/^\d+$/.test(contentLength)) + throw new UploadError(400, "The Content-Length header is invalid."); + const declaredSize = + contentLength === undefined ? undefined : Number(contentLength); + if ( + declaredSize !== undefined && + declaredSize > manifest.storage.maxUploadFileSizeBytes + ) + throw new UploadError( + 413, + `The file exceeds the ${MAX_UPLOAD_SIZE_LABEL} upload limit.`, + ); + let filename: string; + try { + filename = decodeURIComponent(req.get("x-file-name") ?? ""); + } catch { + throw new UploadError(400, "The filename is invalid."); + } + const formatError = validateFileFormat(filename, parameter.fileFormats); + if (formatError) throw new UploadError(400, formatError); + activeUploads += 1; + admitted = true; + res.status(201).json({ + upload: await saveUploadStream( + store, + manifest.storage, + viewer, + parameter.name, + filename, + Readable.toWeb(req) as ReadableStream, + declaredSize, + ), + }); + } catch (error) { + req.resume(); + sendUploadError( + res, + error, + "Could not store the upload. Check the app volume resource and permissions.", + ); + } finally { + if (admitted) activeUploads -= 1; + } + }, + }); +} diff --git a/packages/appkit/src/plugins/designer/shared/appManifest.ts b/packages/appkit/src/plugins/designer/shared/appManifest.ts new file mode 100644 index 000000000..960de9a60 --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/appManifest.ts @@ -0,0 +1,236 @@ +import { z } from "zod/mini"; + +import { isFileFormats } from "./fileFormats"; +import { + hasInvalidFileOutputDeclaration, + parseFileOutput, +} from "./fileOutputs"; +import { isValidMultiselectConfig } from "./multiselect"; +import { RUNNER_REVISION_PARAM } from "./publishedRunner"; +import { parseAppStorage } from "./storageConfig"; + +const nonEmptyString = z.string().check(z.minLength(1)); +const stringOrEmpty = z.catch(z.string(), ""); +const record = z.record(z.string(), z.unknown()); +const optionalBoolean = z.catch(z.optional(z.boolean()), undefined); + +const RESERVED_PARAMETERS = new Set([ + "target_node", + "ld_display_outputs", + "ld_display_outputs_for", +]); + +export function isReservedParameter(name: string): boolean { + return RESERVED_PARAMETERS.has(name) || name.startsWith("_lb_"); +} + +const chartSpecSchema = z.looseObject({ widgetType: nonEmptyString }); +const provenanceSchema = z.object({ + publishedAt: z.number().check(z.positive()), + generatorVersion: z.catch( + z.optional(z.union([z.string(), z.number()])), + undefined, + ), +}); +const targetSchema = z.object({ nodeId: nonEmptyString, label: stringOrEmpty }); +const runnerSchema = z + .object({ + revision: z.string().check(z.regex(/^[0-9a-f]{64}$/)), + parameters: z.record(z.string(), z.string()), + }) + .check( + z.refine( + (runner) => runner.parameters[RUNNER_REVISION_PARAM] === runner.revision, + ), + ); + +const parameterSchema = z.pipe( + z.object({ + name: nonEmptyString.check(z.refine((name) => !isReservedParameter(name))), + label: z.string(), + type: z.catch( + z.enum(["text", "number", "dropdown", "combobox", "multiselect", "file"]), + "text", + ), + defaultValue: stringOrEmpty, + choices: z.catch(z.optional(z.array(z.string())), undefined), + fileFormats: z.optional(z.unknown()), + help: z.optional(z.unknown()), + }), + z.transform((entry) => { + const choices = + entry.type === "combobox" || (entry.choices?.length ?? 0) > 0 + ? entry.choices + : undefined; + const type = + entry.type === "dropdown" && choices === undefined ? "text" : entry.type; + return { + name: entry.name, + label: entry.label, + type, + defaultValue: type === "file" ? "" : entry.defaultValue, + ...((type === "dropdown" || + type === "combobox" || + type === "multiselect") && + choices !== undefined + ? { choices } + : {}), + ...(type === "file" && isFileFormats(entry.fileFormats) + ? { fileFormats: entry.fileFormats } + : {}), + ...(typeof entry.help === "string" && entry.help !== "" + ? { help: entry.help } + : {}), + }; + }), +); + +const markdownBlockSchema = z.pipe( + z.object({ + type: z.literal("markdown"), + id: z.catch(z.optional(nonEmptyString), undefined), + text: z.string(), + hideUntilResults: optionalBoolean, + }), + z.transform(({ type, id, text, hideUntilResults }) => ({ + type, + ...(id === undefined ? {} : { id }), + text, + ...(hideUntilResults === undefined ? {} : { hideUntilResults }), + })), +); + +const outputBlockSchema = z.pipe( + z + .object({ + type: z.literal("output"), + id: nonEmptyString, + label: stringOrEmpty, + nodeId: stringOrEmpty, + port: z.optional(nonEmptyString), + kind: z.optional(z.literal("html")), + chartSpec: z.catch(z.optional(chartSpecSchema), undefined), + hideUntilResults: optionalBoolean, + fileOutput: z.optional(z.transform(parseFileOutput)), + disabled: optionalBoolean, + parentGroupId: z.catch(z.optional(nonEmptyString), undefined), + }) + .check( + z.refine((block) => + block.kind === "html" + ? block.port === undefined && + block.chartSpec === undefined && + block.fileOutput === undefined + : block.port !== undefined, + ), + ), + z.transform( + ({ + chartSpec, + fileOutput, + hideUntilResults, + disabled, + parentGroupId, + ...block + }) => ({ + ...block, + ...(chartSpec === undefined ? {} : { chartSpec }), + ...(fileOutput === undefined ? {} : { fileOutput }), + ...(hideUntilResults === undefined ? {} : { hideUntilResults }), + ...(disabled === undefined ? {} : { disabled }), + ...(parentGroupId === undefined ? {} : { parentGroupId }), + }), + ), +); + +const blockSchema = z.union([markdownBlockSchema, outputBlockSchema]); + +function parseBlocks(raw: unknown): AppManifestBlock[] { + const seen = new Set(); + return (Array.isArray(raw) ? raw : []).flatMap((entry) => { + const block = blockSchema.safeParse(entry).data; + if (!block) return []; + if (block.type === "output") { + if (seen.has(block.id)) return []; + seen.add(block.id); + } + return [block]; + }); +} + +// Both the workspace manifest and /config response use this contract. Ordinary malformed display +// entries can be omitted; declarations governing file access must reject the entire manifest. +export const appManifestSchema = z.pipe( + z.object({ + version: z.literal(8), + runner: z.optional(runnerSchema), + appName: nonEmptyString, + subtitle: z.optional(z.unknown()), + provenance: z.optional(z.unknown()), + target: z.optional(z.unknown()), + storage: z.optional(z.unknown()), + blocks: z.unknown(), + parameters: z.array(z.unknown()), + }), + z.transform((raw, ctx) => { + const storage = parseAppStorage(raw.storage); + const blocks = parseBlocks(raw.blocks); + const entries = raw.parameters.flatMap((entry) => { + const parsed = record.safeParse(entry).data; + return parsed ? [parsed] : []; + }); + if ( + !blocks.some((block) => block.type === "output") || + hasInvalidFileOutputDeclaration(raw.blocks) || + (raw.storage !== undefined && !storage) || + entries.some( + (entry) => + (entry.type === "file" && + (!storage || + (entry.fileFormats !== undefined && + !isFileFormats(entry.fileFormats)))) || + (entry.type === "multiselect" && + !isValidMultiselectConfig(entry.choices, entry.defaultValue)), + ) + ) { + ctx.issues.push({ + code: "custom", + input: raw, + message: "Invalid published app configuration", + }); + return z.NEVER; + } + const provenance = provenanceSchema.safeParse(raw.provenance).data; + const target = targetSchema.safeParse(raw.target).data; + return { + ...(storage === undefined ? {} : { storage }), + ...(raw.runner === undefined ? {} : { runner: raw.runner }), + version: raw.version, + appName: raw.appName, + ...(typeof raw.subtitle === "string" && raw.subtitle !== "" + ? { subtitle: raw.subtitle } + : {}), + ...(provenance === undefined ? {} : { provenance }), + ...(target === undefined ? {} : { target }), + blocks, + parameters: entries.flatMap((entry) => { + const parameter = parameterSchema.safeParse(entry).data; + return parameter ? [parameter] : []; + }), + }; + }), +); + +export type AppChartSpec = z.output; +export type AppParameter = z.output; +export type AppMarkdownBlock = z.output; +export type AppOutputBlock = z.output; +export type AppManifestBlock = z.output; +export type AppManifest = z.output; + +export const outputBlockTitle = ({ + label, + nodeId, + id, +}: Pick): string => + label !== "" ? label : nodeId !== "" ? nodeId : id; diff --git a/packages/appkit/src/plugins/designer/shared/fileFormats.ts b/packages/appkit/src/plugins/designer/shared/fileFormats.ts new file mode 100644 index 000000000..518f2f01e --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/fileFormats.ts @@ -0,0 +1,76 @@ +const FORMAT_EXTENSIONS: Record = { + excel: [".xls", ".xlsx"], + csv: [".csv", ".tsv", ".tab", ".txt"], + json: [".json", ".jsonl", ".ndjson"], + parquet: [".parquet"], + avro: [".avro"], + orc: [".orc"], + xml: [".xml"], + pdf: [".pdf"], +}; +const COMPRESSION_EXTENSIONS = [ + ".gz", + ".bz2", + ".deflate", + ".lz4", + ".snappy", + ".zst", + ".zstd", +]; + +export const isFileFormats = (value: unknown): value is string[] => + Array.isArray(value) && + value.every( + (format) => + typeof format === "string" && + format !== "" && + format === format.trim().toLowerCase(), + ); + +function extensionsFor(format: string): readonly string[] | undefined { + const extensions = Object.hasOwn(FORMAT_EXTENSIONS, format) + ? FORMAT_EXTENSIONS[format] + : undefined; + if (!extensions || !["csv", "json", "xml"].includes(format)) + return extensions; + return extensions.flatMap((extension) => [ + extension, + ...COMPRESSION_EXTENSIONS.map((suffix) => extension + suffix), + ]); +} + +// Text, binary, inferred and runtime-parameterized formats have no reliable filename restriction. +export function uploadAccept( + fileFormats: readonly string[] = [], +): string | undefined { + const restrictions = fileFormats.flatMap((format) => { + const extensions = extensionsFor(format); + return extensions ? [extensions] : []; + }); + return ( + restrictions[0] + ?.filter((extension) => + restrictions.every((allowed) => allowed.includes(extension)), + ) + .join(",") || undefined + ); +} + +export function validateFileFormat( + filename: string, + fileFormats: readonly string[] = [], +): string | undefined { + for (const format of fileFormats) { + const extensions = extensionsFor(format); + if ( + extensions && + !extensions.some((extension) => + filename.toLowerCase().endsWith(extension), + ) + ) { + const label = format === "excel" ? "Excel" : format.toUpperCase(); + return `This Source expects ${label} (${FORMAT_EXTENSIONS[format].join(", ")}). Choose a matching file.`; + } + } + return undefined; +} diff --git a/packages/appkit/src/plugins/designer/shared/fileOutputs.ts b/packages/appkit/src/plugins/designer/shared/fileOutputs.ts new file mode 100644 index 000000000..20fd67472 --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/fileOutputs.ts @@ -0,0 +1,115 @@ +import { isVolumeName } from "./storageConfig"; +import { isRecord } from "./values"; + +export const FILE_OUTPUTS_PARAM = "_lb_file_outputs"; +export const OUTPUT_NAMESPACE_PARAM = "_lb_output_namespace"; +export const APP_REVISION_PARAM = "_lb_app_revision"; +export const FILE_OUTPUTS_MIME_TYPE = + "application/vnd.databricks.lakeflow-designer.files+json"; +export const MAX_OUTPUT_FILES = 50; + +export interface FileOutputConfig { + volumes: string[]; +} +export interface WrittenFile { + path: string; +} +export type FileOutputBehavior = + | "run_artifact" + | "shared_append" + | "shared_workbook_update"; + +export function parseFileOutputBehavior( + value: unknown, +): FileOutputBehavior | undefined { + return value === "run_artifact" || + value === "shared_append" || + value === "shared_workbook_update" + ? value + : undefined; +} + +export interface WrittenFiles { + node: string; + files: WrittenFile[]; + // Older receipts omit this; never infer run isolation from a path or the current manifest. + behavior?: FileOutputBehavior; +} + +export function parseFileOutput(value: unknown): FileOutputConfig | undefined { + if ( + typeof value !== "object" || + value === null || + !("volumes" in value) || + !Array.isArray(value.volumes) || + value.volumes.length === 0 || + value.volumes.length > 50 + ) { + return undefined; + } + if ( + !value.volumes.every( + (volume): volume is string => + typeof volume === "string" && isVolumeName(volume), + ) + ) { + return undefined; + } + return { volumes: [...new Set(value.volumes)].sort() }; +} + +export function hasInvalidFileOutputDeclaration(blocks: unknown): boolean { + if (!Array.isArray(blocks)) return false; + const records = blocks.filter(isRecord); + const outputIds = records + .filter((block) => block.type === "output") + .map((block) => block.id); + // Preview blocks may be omitted when malformed. File blocks must fail closed instead: + // dropping one could remove the ownership/policy controls while the runner still writes it. + return records.some( + (block) => + block.fileOutput !== undefined && + (block.type !== "output" || + typeof block.id !== "string" || + block.id.trim() === "" || + typeof block.nodeId !== "string" || + block.nodeId.trim() === "" || + outputIds.filter((id) => id === block.id).length !== 1 || + parseFileOutput(block.fileOutput) === undefined), + ); +} + +// Require literal, canonical paths. Never normalize an untrusted path into the approved scope. +export function isOutputFilePath( + path: unknown, + config: FileOutputConfig, +): path is string { + if ( + typeof path !== "string" || + path.length > 4096 || + [...path].some( + (character) => + character === "\\" || + character.charCodeAt(0) < 32 || + character.charCodeAt(0) === 127, + ) + ) + return false; + const root = config.volumes + .map((volume) => `/Volumes/${volume.replaceAll(".", "/")}/`) + .find((root) => path.startsWith(root)); + if (!root) return false; + const segments = path.slice(root.length).split("/"); + return ( + segments.every( + (segment) => segment !== "" && segment !== "." && segment !== "..", + ) && /\.(csv|json|xlsx)$/i.test(segments.at(-1) ?? "") + ); +} + +export const fileDownloadRoute = ( + runId: string, + outputId: string, + index: number, +) => + `/api/designer/run/${encodeURIComponent(runId)}/files/${encodeURIComponent(outputId)}/${index}/download`; diff --git a/packages/appkit/src/plugins/designer/shared/guardrails.ts b/packages/appkit/src/plugins/designer/shared/guardrails.ts new file mode 100644 index 000000000..9ccb4ed7e --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/guardrails.ts @@ -0,0 +1,54 @@ +import { isRecord, nonEmptyString } from "./values"; + +export interface GuardrailDiagnostic { + nodeId?: string; + operatorName: string; + checkIndex: number; + checkType?: string; + severity: "warning" | "failure"; + message: string; +} + +export interface RunDiagnostics { + diagnostics?: GuardrailDiagnostic[]; + failureReason?: string; +} + +export function parseRunDiagnostics(raw: { + diagnostics?: unknown; + failureReason?: unknown; +}): RunDiagnostics { + const diagnostics = Array.isArray(raw.diagnostics) + ? raw.diagnostics.flatMap((entry): GuardrailDiagnostic[] => { + if (!isRecord(entry)) return []; + const operatorName = nonEmptyString(entry.operatorName); + const message = nonEmptyString(entry.message); + const nodeId = nonEmptyString(entry.nodeId); + const checkType = nonEmptyString(entry.checkType); + if ( + !operatorName || + !message || + typeof entry.checkIndex !== "number" || + !Number.isSafeInteger(entry.checkIndex) || + entry.checkIndex < 0 || + (entry.severity !== "warning" && entry.severity !== "failure") + ) + return []; + return [ + { + operatorName, + checkIndex: entry.checkIndex, + severity: entry.severity, + message, + ...(nodeId ? { nodeId } : {}), + ...(checkType ? { checkType } : {}), + }, + ]; + }) + : []; + const failureReason = nonEmptyString(raw.failureReason); + return { + ...(diagnostics.length > 0 ? { diagnostics } : {}), + ...(failureReason ? { failureReason } : {}), + }; +} diff --git a/packages/appkit/src/plugins/designer/shared/htmlOutputs.ts b/packages/appkit/src/plugins/designer/shared/htmlOutputs.ts new file mode 100644 index 000000000..eaac5fe8a --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/htmlOutputs.ts @@ -0,0 +1,5 @@ +export const MAX_HTML_OUTPUT_CHARS = 4 * 1024 * 1024; + +export function isHtmlOutput(value: unknown): value is string { + return typeof value === "string" && value.length <= MAX_HTML_OUTPUT_CHARS; +} diff --git a/packages/appkit/src/plugins/designer/shared/multiselect.ts b/packages/appkit/src/plugins/designer/shared/multiselect.ts new file mode 100644 index 000000000..dc0d26a5e --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/multiselect.ts @@ -0,0 +1,46 @@ +const MULTISELECT_DELIMITER = ","; + +export function parseMultiselectValue(value: string): string[] { + return value === "" ? [] : value.split(MULTISELECT_DELIMITER); +} + +export function isValidMultiselectValue( + value: unknown, + choices: readonly string[], +): value is string { + return ( + typeof value === "string" && + parseMultiselectValue(value).every((choice) => choices.includes(choice)) + ); +} + +export function isValidMultiselectConfig( + choices: unknown, + defaultValue: unknown, +): boolean { + return ( + Array.isArray(choices) && + choices.length > 0 && + choices.every( + (choice): choice is string => + typeof choice === "string" && + choice !== "" && + !choice.includes(MULTISELECT_DELIMITER), + ) && + isValidMultiselectValue( + defaultValue === undefined ? "" : defaultValue, + choices, + ) + ); +} + +export function toggleMultiselectValue( + value: string, + choice: string, + selected: boolean, +): string { + const selections = new Set(parseMultiselectValue(value)); + if (selected) selections.add(choice); + else selections.delete(choice); + return [...selections].join(MULTISELECT_DELIMITER); +} diff --git a/packages/appkit/src/plugins/designer/shared/publishedRunner.ts b/packages/appkit/src/plugins/designer/shared/publishedRunner.ts new file mode 100644 index 000000000..a8b6237d3 --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/publishedRunner.ts @@ -0,0 +1,37 @@ +import { isRecord } from "./values"; + +export const RUNNER_REVISION_PARAM = "_lb_runner_revision"; +export const APP_UPDATING_MESSAGE = + "This App is updating. New runs are paused until publication finishes. Try again shortly."; + +export interface PublishedRunner { + revision: string; + parameters: Record; +} + +export function isPublishedRunner( + runner: PublishedRunner | undefined, + job: unknown, +): boolean { + if ( + !runner || + !isRecord(job) || + !isRecord(job.settings) || + !Array.isArray(job.settings.tasks) + ) + return false; + if (job.settings.tasks.length !== 1) return false; + const task: unknown = job.settings.tasks[0]; + if (!isRecord(task) || !isRecord(task.notebook_task)) return false; + const notebook = task.notebook_task; + const parameters = notebook.base_parameters; + return ( + typeof notebook.notebook_path === "string" && + notebook.notebook_path.endsWith(`/runner-${runner.revision}.designer.py`) && + isRecord(parameters) && + Object.keys(parameters).length === Object.keys(runner.parameters).length && + Object.entries(runner.parameters).every( + ([name, value]) => parameters[name] === value, + ) + ); +} diff --git a/packages/appkit/src/plugins/designer/shared/resultPreview.ts b/packages/appkit/src/plugins/designer/shared/resultPreview.ts new file mode 100644 index 000000000..ec0557285 --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/resultPreview.ts @@ -0,0 +1,37 @@ +export const DISPLAY_ROW_LIMIT = 1000; + +export interface ResultPreview { + // null means the source did not report whether its result was truncated. + truncated: boolean | null; + total_row_count?: number; +} + +export function summarizeResultPreview( + receivedRowCount: number, + truncated: unknown, + totalRowCount?: unknown, +): ResultPreview { + const validTotal = + typeof totalRowCount === "number" && + Number.isSafeInteger(totalRowCount) && + totalRowCount >= receivedRowCount && + (truncated !== true || totalRowCount > receivedRowCount); + // Only a positively complete source result lets us use its length as an exact total. + const total = validTotal + ? totalRowCount + : truncated === false + ? receivedRowCount + : undefined; + const hasMoreRows = + truncated === true || + receivedRowCount > DISPLAY_ROW_LIMIT || + (total !== undefined && total > receivedRowCount); + return { + truncated: hasMoreRows + ? true + : truncated === false || total !== undefined + ? false + : null, + ...(total === undefined ? {} : { total_row_count: total }), + }; +} diff --git a/packages/appkit/src/plugins/designer/shared/storageConfig.ts b/packages/appkit/src/plugins/designer/shared/storageConfig.ts new file mode 100644 index 000000000..6728425c1 --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/storageConfig.ts @@ -0,0 +1,58 @@ +export interface AppStorage { + volume: string; + path: string; + maxUploadFileSizeBytes: number; +} + +export const MAX_UPLOAD_BYTES = 5 * 1024 * 1024 * 1024; +export const MAX_UPLOAD_SIZE_LABEL = "5 GB"; +export const UPLOAD_UNAVAILABLE = "UPLOAD_UNAVAILABLE"; +export const UPLOAD_UNAVAILABLE_MESSAGE = + "This upload is no longer available. Upload the file again."; +export const UPLOAD_REFERENCE = + /^upload:([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/; + +export const uploadStoragePath = (storage: AppStorage): string => + `${storage.path}/uploads`; + +const SAFE_NAME = /^[A-Za-z0-9_-]+$/; + +export function isVolumeName(volume: string): boolean { + const names = volume.split("."); + return names.length === 3 && names.every((name) => SAFE_NAME.test(name)); +} + +export function parseAppStorage(value: unknown): AppStorage | undefined { + if ( + typeof value !== "object" || + value === null || + !("volume" in value) || + !("path" in value) || + !("maxUploadFileSizeBytes" in value) + ) + return undefined; + if ( + typeof value.volume !== "string" || + typeof value.path !== "string" || + !isVolumeName(value.volume) + ) + return undefined; + const prefix = `/Volumes/${value.volume.replaceAll(".", "/")}/designer_apps/`; + if ( + !value.path.startsWith(prefix) || + !SAFE_NAME.test(value.path.slice(prefix.length)) + ) + return undefined; + if ( + typeof value.maxUploadFileSizeBytes !== "number" || + !Number.isSafeInteger(value.maxUploadFileSizeBytes) || + value.maxUploadFileSizeBytes <= 0 || + value.maxUploadFileSizeBytes > MAX_UPLOAD_BYTES + ) + return undefined; + return { + volume: value.volume, + path: value.path, + maxUploadFileSizeBytes: value.maxUploadFileSizeBytes, + }; +} diff --git a/packages/appkit/src/plugins/designer/shared/values.ts b/packages/appkit/src/plugins/designer/shared/values.ts new file mode 100644 index 000000000..45bd69e9b --- /dev/null +++ b/packages/appkit/src/plugins/designer/shared/values.ts @@ -0,0 +1,37 @@ +export const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const nonEmptyString = (value: unknown): string | undefined => + typeof value === "string" && value !== "" ? value : undefined; + +export const positiveInt = (value: unknown): number | undefined => { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined; +}; + +// Run and job IDs are int64: keep them as decimal strings, since Number corrupts values past 2^53-1. +export const idString = (value: unknown): string | undefined => { + if ( + typeof value !== "string" && + typeof value !== "number" && + typeof value !== "bigint" + ) { + return undefined; + } + const text = String(value); + return /^[1-9][0-9]*$/.test(text) ? text : undefined; +}; + +export const stringValues = ( + value: unknown, +): Record | undefined => + isRecord(value) + ? Object.fromEntries( + Object.entries(value).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ) + : undefined; + +export const errorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); diff --git a/packages/appkit/src/plugins/designer/tests/browser-origin.test.mjs b/packages/appkit/src/plugins/designer/tests/browser-origin.test.mjs new file mode 100644 index 000000000..b259fb763 --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/browser-origin.test.mjs @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; + +import { test } from "vitest"; + +import { isSameOriginBrowserRequest } from "../server/browserRequestOrigin.ts"; + +test("allows the App page and authenticated non-browser API clients", () => { + for (const headers of [ + {}, + { fetchSite: "same-origin" }, + { fetchSite: "none" }, + { + origin: "https://example.test", + appUrl: "https://example.test", + fetchSite: "same-origin", + }, + { origin: "https://example.test", appUrl: "https://example.test:443/" }, + { origin: "http://localhost:5173", appUrl: "http://localhost:5173" }, + ]) { + assert.equal(isSameOriginBrowserRequest(headers), true); + } +}); + +test("rejects opaque frame origins, sibling sites, and external browser requests", () => { + for (const headers of [ + { origin: "null", appUrl: "https://example.test" }, + { fetchSite: "cross-site" }, + { fetchSite: "same-site" }, + { origin: "https://other.example.test", appUrl: "https://example.test" }, + { + origin: "https://example.test", + appUrl: "https://example.test", + fetchSite: "cross-site", + }, + { origin: "https://example.test:444", appUrl: "https://example.test" }, + { origin: "http://example.test", appUrl: "https://example.test" }, + { origin: "https://example.test/path", appUrl: "https://example.test" }, + { origin: "not-an-origin", appUrl: "https://example.test" }, + { origin: "https://example.test", appUrl: "not-a-url" }, + { origin: "https://example.test" }, + ]) { + assert.equal(isSameOriginBrowserRequest(headers), false); + } +}); diff --git a/packages/appkit/src/plugins/designer/tests/output-files.test.mjs b/packages/appkit/src/plugins/designer/tests/output-files.test.mjs new file mode 100644 index 000000000..3ad69f0ea --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/output-files.test.mjs @@ -0,0 +1,440 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; + +import express from "express"; +import { test } from "vitest"; + +import { + registerOutputFileRoutes, + recordedFileOutput, +} from "../server/outputFiles"; +import { manifestRevision } from "../server/runRevision"; + +async function harness(t) { + const manifest = { + version: 8, + appName: "Output app", + parameters: [{ name: "filename" }], + blocks: [ + { + type: "output", + id: "out", + nodeId: "output_0", + port: "result", + fileOutput: { volumes: ["main.apps.files", "main.apps.other"] }, + }, + { type: "output", id: "preview", nodeId: "source", port: "data" }, + ], + }; + const viewer = "a".repeat(64); + const runnerRevision = "b".repeat(64); + const notebookPath = `/Users/author/app/runner-${runnerRevision}.designer.py`; + const state = { + manifest, + notebookPath, + reports: [], + reads: [], + receipts: [ + { + node: "output_0", + files: [{ path: "/Volumes/main/apps/files/report.csv" }], + }, + ], + run: { + run_id: 1, + job_id: 10, + state: { life_cycle_state: "TERMINATED", result_state: "SUCCESS" }, + tasks: [{ run_id: 2, notebook_task: { notebook_path: notebookPath } }], + job_parameters: Object.entries({ + _lb_app_viewer: viewer, + _lb_runner_revision: runnerRevision, + _lb_app_revision: manifestRevision(manifest), + _lb_file_outputs: JSON.stringify({ + output_0: manifest.blocks[0].fileOutput, + }), + }).map(([name, value]) => ({ name, value })), + }, + }; + const files = new Map([ + ["/Volumes/main/apps/files/report.csv", Buffer.from("price\n42\n")], + ]); + const app = express(); + registerOutputFileRoutes( + ({ method, path, handler }) => app[method](`/api/designer${path}`, handler), + { + jobId: "10", + manifest: async () => state.manifest, + viewer: (req) => req.get("x-viewer"), + notebookPath: async () => state.notebookPath, + getRun: async (id) => (id === 1 ? state.run : undefined), + readPayload: async (taskId) => { + assert.equal(taskId, "2"); + return JSON.stringify({ outputs: [], files: state.receipts }); + }, + store: (volume) => ({ + size: async (path) => files.get(path)?.length, + download: async (path) => { + state.reads.push({ volume, path }); + return state.stream + ? state.stream() + : new ReadableStream({ + start(controller) { + controller.enqueue(files.get(path)); + controller.close(); + }, + }); + }, + }), + report: (error) => state.reports.push(error), + }, + ); + const server = app.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.onTestFinished(() => { + server.closeAllConnections(); + server.close(); + }); + const call = (path = "/1/files/out/0/download", options = {}) => + fetch(`http://127.0.0.1:${server.address().port}/api/designer/run${path}`, { + ...options, + headers: { "x-viewer": viewer, ...options.headers }, + }); + return { state, files, call }; +} + +test("downloads original bytes repeatedly without another Job, file copy or deletion", async (t) => { + const h = await harness(t); + for (let index = 0; index < 3; index += 1) { + const response = await h.call(); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.equal(response.headers.get("x-content-type-options"), "nosniff"); + assert.match( + response.headers.get("content-disposition"), + /attachment; filename="report.csv"/, + ); + assert.equal(await response.text(), "price\n42\n"); + } + assert.equal(h.files.size, 1); + assert.equal(h.state.reads.length, 3); +}); + +test("historical downloads survive a runner-code update with unchanged file policy", async (t) => { + const h = await harness(t); + h.state.notebookPath = `/Users/author/app/runner-${"c".repeat(64)}.designer.py`; + const response = await h.call(); + assert.equal(response.status, 200); + assert.equal(await response.text(), "price\n42\n"); +}); + +test("downloads the recorded split-file index from any author-approved volume", async (t) => { + const h = await harness(t); + const path = "/Volumes/main/apps/other/sub/データ #100%.xlsx"; + h.state.receipts[0].files.push({ path }); + h.files.set(path, Buffer.from([0, 255, 1, 2])); + const response = await h.call( + "/1/files/out/1/download?path=/Volumes/main/apps/files/stolen.csv", + ); + assert.equal(response.status, 200); + assert.match( + response.headers.get("content-disposition"), + /filename\*=UTF-8''%E3/, + ); + assert.deepEqual( + Buffer.from(await response.arrayBuffer()), + Buffer.from([0, 255, 1, 2]), + ); + assert.deepEqual(h.state.reads, [{ volume: "main.apps.other", path }]); +}); + +test("serves current destination bytes and retains files when another run overwrites them", async (t) => { + const h = await harness(t); + h.files.set("/Volumes/main/apps/files/report.csv", Buffer.from("updated\n")); + assert.equal(await (await h.call()).text(), "updated\n"); + assert.equal(h.files.size, 1); +}); + +test("completed write receipts survive later preview/count errors and failed sibling branches", async (t) => { + const h = await harness(t); + h.state.run.state.result_state = "FAILED"; + assert.equal((await h.call()).status, 200); + delete h.state.run.state.result_state; + h.state.run.state.life_cycle_state = "RUNNING"; + assert.equal((await h.call()).status, 409); +}); + +test("presentation-only changes preserve downloads while execution changes require a new run", async (t) => { + const h = await harness(t); + h.state.manifest = { + ...h.state.manifest, + appName: "Renamed", + provenance: { publishedAt: 5000 }, + parameters: [{ name: "filename", defaultValue: "changed" }], + blocks: h.state.manifest.blocks + .toReversed() + .map((block) => ({ ...block, label: "Changed" })), + }; + assert.equal((await h.call()).status, 200); + h.state.manifest.blocks + .find((block) => block.id === "out") + .fileOutput.volumes.push("main.apps.new"); + assert.equal((await h.call()).status, 409); +}); + +function setRunParameter(run, name, value) { + run.job_parameters = run.job_parameters.filter( + (parameter) => parameter.name !== name, + ); + if (value !== undefined) run.job_parameters.push({ name, value }); +} + +for (const [label, mutate, status] of [ + [ + "wrong job", + (h) => { + h.state.run.job_id = 11; + }, + 404, + ], + [ + "wrong owner", + (h) => { + setRunParameter(h.state.run, "_lb_app_viewer", "other"); + }, + 404, + ], + [ + "missing owner", + (h) => { + delete h.state.run.job_parameters; + }, + 404, + ], + [ + "legacy export run", + (h) => { + setRunParameter(h.state.run, "_lb_export_request", "{}"); + }, + 404, + ], + [ + "missing revision", + (h) => { + setRunParameter(h.state.run, "_lb_app_revision", undefined); + }, + 409, + ], + [ + "unverified runner path", + (h) => { + h.state.run.tasks[0].notebook_task.notebook_path = + "/Users/other/runner.designer.py"; + }, + 409, + ], + [ + "missing runner revision", + (h) => { + setRunParameter(h.state.run, "_lb_runner_revision", undefined); + }, + 409, + ], + [ + "missing run policy", + (h) => { + setRunParameter(h.state.run, "_lb_file_outputs", undefined); + }, + 409, + ], + [ + "changed run policy", + (h) => { + setRunParameter( + h.state.run, + "_lb_file_outputs", + JSON.stringify({ output_0: { volumes: ["main.apps.other"] } }), + ); + }, + 409, + ], + [ + "missing artifact", + (h) => { + h.files.clear(); + }, + 404, + ], + [ + "missing receipt", + (h) => { + h.state.receipts = []; + }, + 404, + ], + [ + "duplicate receipt", + (h) => { + h.state.receipts.push(h.state.receipts[0]); + }, + 404, + ], + [ + "another node receipt", + (h) => { + h.state.receipts[0].node = "source"; + }, + 404, + ], +]) { + test(`rejects ${label} before reading any file`, async (t) => { + const h = await harness(t); + mutate(h); + assert.equal((await h.call()).status, status); + assert.deepEqual(h.state.reads, []); + }); +} + +for (const path of [ + "/Volumes/main/apps/files_other/report.csv", + "/Volumes/main/apps/secret/report.csv", + "/Volumes/main/apps/files/../secret/report.csv", + "/Volumes/main/apps/files/a/./report.csv", + "/Volumes/main/apps/files//report.csv", + "/Volumes/main/apps/files/back\\slash.csv", + "/Volumes/main/apps/files/bad\nname.csv", + "/Volumes/main/apps/files/private.py", + "s3://bucket/report.csv", + "/Volumes/main/apps/files/report.xlsx/", +]) { + test(`rejects unsafe or undeclared artifact path ${JSON.stringify(path)}`, async (t) => { + const h = await harness(t); + h.state.receipts[0].files[0].path = path; + assert.equal((await h.call()).status, 404); + assert.deepEqual(h.state.reads, []); + }); +} + +test("rejects anonymous, preview-only, invalid index and partial download requests", async (t) => { + const h = await harness(t); + assert.equal( + (await h.call(undefined, { headers: { "x-viewer": "" } })).status, + 401, + ); + for (const path of [ + "/1/files/preview/0/download", + "/1/files/out/-1/download", + "/1/files/out/00/download", + "/2/files/out/0/download", + ]) + assert.equal((await h.call(path)).status, 404); + assert.equal((await h.call(undefined, { method: "HEAD" })).status, 405); + assert.equal( + (await h.call(undefined, { headers: { range: "bytes=0-3" } })).status, + 416, + ); + assert.deepEqual(h.state.reads, []); +}); + +test("rejects duplicate or too many files as an entire invalid receipt", () => { + for (const files of [ + [ + { path: "/Volumes/main/apps/files/a.csv" }, + { path: "/Volumes/main/apps/files/a.csv" }, + ], + Array.from({ length: 51 }, (_, index) => ({ + path: `/Volumes/main/apps/files/${index}.csv`, + })), + ]) { + assert.equal( + recordedFileOutput( + JSON.stringify({ files: [{ node: "out", files }] }), + "out", + { volumes: ["main.apps.files"] }, + ), + undefined, + ); + } + assert.deepEqual( + recordedFileOutput( + JSON.stringify({ files: [{ node: "out", files: [] }] }), + "out", + { + volumes: ["main.apps.files"], + }, + ), + { node: "out", files: [] }, + ); +}); + +test("retains recorded write behavior without inferring it from the file path or current policy", () => { + const files = [ + { + path: "/Volumes/main/apps/files/_designer_apps/old-namespace/report.csv", + }, + ]; + for (const behavior of [ + "run_artifact", + "shared_append", + "shared_workbook_update", + undefined, + "unknown", + null, + {}, + ]) { + const receipt = recordedFileOutput( + JSON.stringify({ files: [{ node: "out", files, behavior }] }), + "out", + { + volumes: ["main.apps.files"], + }, + ); + assert.deepEqual(receipt, { + node: "out", + files, + ...(["run_artifact", "shared_append", "shared_workbook_update"].includes( + behavior, + ) + ? { behavior } + : {}), + }); + } +}); + +test("each run downloads its recorded namespaced artifact rather than the latest output path", async (t) => { + const h = await harness(t); + const path = + "/Volumes/main/apps/files/_designer_apps/submission/attempt/output_0/report.csv"; + h.state.receipts[0] = { + node: "output_0", + behavior: "run_artifact", + files: [{ path }], + }; + h.files.set(path, Buffer.from("first run\n")); + h.files.set( + "/Volumes/main/apps/files/report.csv", + Buffer.from("latest shared file\n"), + ); + h.files.set( + "/Volumes/main/apps/files/_designer_apps/new-submission/attempt/output_0/report.csv", + Buffer.from("next run\n"), + ); + assert.equal(await (await h.call()).text(), "first run\n"); + assert.deepEqual(h.state.reads, [{ volume: "main.apps.files", path }]); +}); + +test("aborts incomplete transfers without deleting the source file", async (t) => { + const h = await harness(t); + h.state.stream = () => + new ReadableStream({ + start(controller) { + controller.enqueue(Buffer.from("p")); + controller.close(); + }, + }); + await assert.rejects(async () => { + const response = await h.call(); + await response.arrayBuffer(); + }); + assert.equal(h.files.size, 1); + assert.equal(h.state.reports.length, 1); +}); diff --git a/packages/appkit/src/plugins/designer/tests/results.test.mjs b/packages/appkit/src/plugins/designer/tests/results.test.mjs new file mode 100644 index 000000000..474de70d8 --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/results.test.mjs @@ -0,0 +1,492 @@ +import assert from "node:assert/strict"; + +import { test } from "vitest"; + +import { exportedModelToRunPayload } from "../server/exportedRunOutput"; +import { matchRunOutputs } from "../server/runOutputs"; +const ROW_COUNTS_MIME_TYPE = + "application/vnd.databricks.lakeflow-designer.row-counts+json"; +const FILES_MIME_TYPE = + "application/vnd.databricks.lakeflow-designer.files+json"; +const NODE_STATE_MIME_TYPE = + "application/vnd.databricks.lakebuilder.runtime-node-state+json"; +function displayTable(rowCount, overflow) { + return { + type: "table", + schema: [{ name: "value", type: '"long"', nullable: true }], + data: Array.from({ length: rowCount }, (_, i) => [i]), + ...(overflow === undefined ? {} : { overflow }), + }; +} + +function rowCounts(node, counts) { + return { + type: "mimeBundle", + data: { [ROW_COUNTS_MIME_TYPE]: { node, counts } }, + }; +} + +function notebookHtml(commands) { + const model = Buffer.from( + encodeURIComponent(JSON.stringify({ commands })), + ).toString("base64"); + return ``; +} + +function nodeState(nodeId, disabled) { + return { + type: "mimeBundle", + data: { [NODE_STATE_MIME_TYPE]: { nodeId, disabled } }, + }; +} + +function guardrailCommand(nodeId, name, text, extra = {}) { + return { + command: `"""\nid: ${nodeId}\ntemplate: guardrails\ntemplateVersion: 2.0.0\nname: ${name}\n"""\nrun(config, inputs, spark)`, + results: { type: "ansi", data: text }, + ...extra, + }; +} + +const warningText = + '[guardrails] WARNING: checks[0] (not_null): column "id" has 3 null values'; + +test("runtime states do not shift table ports, row counts, or file receipts", () => { + const files = { + node: "writer", + files: [{ path: "/Volumes/main/apps/files/report.csv" }], + }; + const html = notebookHtml([ + { + command: 'display(ctx["source.data"])\ndisplay(ctx["source.summary"])', + results: { + data: [ + nodeState("source", false), + displayTable(2, true), + { + type: "mimeBundle", + data: { + [NODE_STATE_MIME_TYPE]: JSON.stringify({ + nodeId: "writer", + disabled: true, + }), + }, + }, + { type: "mimeBundle", data: { [FILES_MIME_TYPE]: files } }, + displayTable(1, false), + rowCounts("source", { data: 4000, summary: 1 }), + ], + }, + }, + ]); + const parsed = JSON.parse(exportedModelToRunPayload(html)); + assert.deepEqual(parsed.runtimeNodeStates, { + source: { disabled: false }, + writer: { disabled: true }, + }); + assert.deepEqual( + parsed.outputs.map(({ target_node, target_port, total_row_count }) => ({ + target_node, + target_port, + total_row_count, + })), + [ + { target_node: "source", target_port: "data", total_row_count: 4000 }, + { target_node: "source", target_port: "summary", total_row_count: 1 }, + ], + ); + assert.deepEqual(parsed.files, [files]); +}); + +test("singleton runtime states survive missing tables and later states replace earlier ones", () => { + const parsed = JSON.parse( + exportedModelToRunPayload( + notebookHtml([ + { command: "run()", results: nodeState("writer", true) }, + { command: "run()", results: nodeState("writer", false) }, + { command: "group_gate()", results: nodeState("group", true) }, + ]), + ), + ); + assert.deepEqual(parsed.runtimeNodeStates, { + writer: { disabled: false }, + group: { disabled: true }, + }); + assert.deepEqual(parsed.outputs, []); +}); + +test("malformed runtime metadata preserves valid sibling state and results", () => { + const invalid = [ + null, + "not JSON", + [], + {}, + { nodeId: "", disabled: true }, + { nodeId: "source", disabled: "true" }, + { nodeId: "source", disabled: 1 }, + ]; + const parsed = JSON.parse( + exportedModelToRunPayload( + notebookHtml([ + { + command: 'display(ctx["source.data"])', + results: { + data: [ + nodeState("source", false), + ...invalid.map((payload) => ({ + type: "mimeBundle", + data: { [NODE_STATE_MIME_TYPE]: payload }, + })), + displayTable(1, false), + ], + }, + }, + ]), + ), + ); + assert.deepEqual(parsed.runtimeNodeStates, { source: { disabled: false } }); + assert.equal(parsed.outputs[0].target_port, "data"); +}); + +test("file receipts are decoded independently of tables and full counts without shifting display ports", () => { + const receipt = { + node: "output_0", + files: [{ path: "/Volumes/main/apps/files/sheet.xlsx" }], + }; + const html = notebookHtml([ + { + command: "write_output()", + results: { + type: "listResults", + data: [ + { + type: "mimeBundle", + data: { [FILES_MIME_TYPE]: JSON.stringify(receipt) }, + }, + ], + }, + }, + { + command: 'display(ctx["source_0.data"])', + results: { + type: "listResults", + data: [ + { + type: "mimeBundle", + data: { [FILES_MIME_TYPE]: { node: "output_1", files: [] } }, + }, + displayTable(2, true), + rowCounts("source_0", { data: 12345 }), + ], + }, + }, + ]); + const parsed = JSON.parse(exportedModelToRunPayload(html)); + assert.deepEqual(parsed.files, [receipt, { node: "output_1", files: [] }]); + assert.equal(parsed.outputs.length, 1); + assert.equal(parsed.outputs[0].target_node, "source_0"); + assert.equal(parsed.outputs[0].total_row_count, 12345); +}); + +test("a singleton file MIME receipt remains available when no preview was produced", () => { + const receipt = { + node: "output_0", + files: [{ path: "/Volumes/main/apps/files/report.csv" }], + }; + const parsed = JSON.parse( + exportedModelToRunPayload( + notebookHtml([ + { + command: "write_file()", + results: { type: "mimeBundle", data: { [FILES_MIME_TYPE]: receipt } }, + }, + ]), + ), + ); + assert.deepEqual(parsed.files, [receipt]); + assert.deepEqual(parsed.outputs, []); +}); + +test("file MIME receipts retain their effective behavior and ignore unrecognized metadata", () => { + for (const behavior of [ + "run_artifact", + "shared_append", + "shared_workbook_update", + undefined, + "overwrite", + null, + {}, + ]) { + const receipt = { + node: "output_0", + files: [{ path: "/Volumes/main/apps/files/report.csv" }], + behavior, + }; + const parsed = JSON.parse( + exportedModelToRunPayload( + notebookHtml([ + { + command: "write_file()", + results: { + type: "mimeBundle", + data: { [FILES_MIME_TYPE]: receipt }, + }, + }, + ]), + ), + ); + assert.deepEqual(parsed.files, [ + { + node: receipt.node, + files: receipt.files, + ...([ + "run_artifact", + "shared_append", + "shared_workbook_update", + ].includes(behavior) + ? { behavior } + : {}), + }, + ]); + } +}); + +test("malformed file receipts do not discard valid sibling receipts or table counts", () => { + const invalid = [ + "not JSON", + {}, + { node: "output", files: [{}] }, + { + node: "output", + files: Array.from({ length: 51 }, () => ({ + path: "/Volumes/a/b/c/data.csv", + })), + }, + ]; + const html = notebookHtml([ + { + command: 'display(ctx["source_0.data"])', + results: { + data: [ + ...invalid.map((receipt) => ({ + type: "mimeBundle", + data: { [FILES_MIME_TYPE]: receipt }, + })), + displayTable(1, false), + rowCounts("source_0", { data: 1 }), + ], + }, + }, + ]); + const parsed = JSON.parse(exportedModelToRunPayload(html)); + assert.deepEqual(parsed.files, []); + assert.equal(parsed.outputs[0].total_row_count, 1); +}); + +test("keeps overflow and counts isolated per displayed port, including encoded exports", () => { + const html = notebookHtml([ + { + command: + 'display(ctx["filter.filtered_data"])\ndisplay(ctx["filter.excluded_data"])', + results: { data: [displayTable(990, false), displayTable(1000, true)] }, + }, + { + command: 'display(ctx["viz.data"])', + results: { data: [displayTable(12, false)] }, + }, + ]); + const plain = exportedModelToRunPayload(html); + assert.equal( + exportedModelToRunPayload(Buffer.from(html).toString("base64")), + plain, + ); + assert.deepEqual( + JSON.parse(plain).outputs.map( + ({ target_node, target_port, rows, truncated, total_row_count }) => ({ + target_node, + target_port, + rows: rows.length, + truncated, + total_row_count, + }), + ), + [ + { + target_node: "filter", + target_port: "filtered_data", + rows: 990, + truncated: false, + total_row_count: 990, + }, + { + target_node: "filter", + target_port: "excluded_data", + rows: 1000, + truncated: true, + total_row_count: undefined, + }, + { + target_node: "viz", + target_port: "data", + rows: 12, + truncated: false, + total_row_count: 12, + }, + ], + ); +}); + +test("invalid notebook exports remain unreadable instead of fabricating an empty complete result", () => { + for (const html of [ + undefined, + null, + "", + "not a notebook", + "__DATABRICKS_NOTEBOOK_MODEL = 'invalid'", + ]) { + assert.equal(exportedModelToRunPayload(html), undefined); + } +}); + +test("ignores malformed count metadata and preserves the preview completeness signal", () => { + for (const marker of [ + rowCounts("", { data: 50 }), + rowCounts("source_0", {}), + rowCounts("source_0", { data: -1 }), + rowCounts("source_0", { data: 1.5 }), + rowCounts("source_0", { data: "50" }), + ]) { + const html = notebookHtml([ + { + command: 'display(ctx["source_0.data"])', + results: { data: [displayTable(10, true), marker] }, + }, + ]); + const output = JSON.parse(exportedModelToRunPayload(html)).outputs[0]; + assert.equal(output.total_row_count, undefined); + assert.equal(output.truncated, true); + } +}); + +const HTML_BLOCK = { + type: "output", + id: "page", + nodeId: "html_1", + kind: "html", + label: "Report", +}; + +const htmlCommand = (html, extra = {}) => ({ + command: '# designer-app-html-output: "html_1"', + results: { + type: "listResults", + data: [{ type: "mimeBundle", data: { "text/html": html } }], + }, + ...extra, +}); + +const branches = ["prod", "dev"].map((nodeId) => ({ + type: "output", + id: nodeId, + nodeId, + port: "data", + parentGroupId: `group_${nodeId}`, +})); +for (const environment of ["prod", "dev"]) { + test(`group state disables only the inactive ${environment} branch`, () => { + const commands = ["prod", "dev"].flatMap((nodeId) => [ + { + command: "group_gate()", + results: nodeState(`group_${nodeId}`, nodeId !== environment), + }, + { + command: `display(ctx["${nodeId}.data"])`, + results: displayTable(nodeId === environment ? 1 : 0, false), + }, + ]); + const result = matchRunOutputs( + branches, + exportedModelToRunPayload(notebookHtml(commands)), + ); + assert.deepEqual( + result.outputs.map(({ id, disabled }) => ({ id, disabled })), + [ + { id: "prod", disabled: environment !== "prod" }, + { id: "dev", disabled: environment !== "dev" }, + ], + ); + }); +} +for (const own of [true, false]) + for (const group of [true, false]) { + test(`combines node ${own} and group ${group} disablement`, () => { + const result = matchRunOutputs( + [branches[0]], + exportedModelToRunPayload( + notebookHtml([ + { + command: "group_gate()", + results: nodeState("group_prod", group), + }, + { + command: 'display(ctx["prod.data"])', + results: { + data: [nodeState("prod", own), displayTable(0, false)], + }, + }, + ]), + ), + ); + assert.equal(result.outputs[0].disabled, own || group); + }); + } + +test("disabled file outputs remain hidden without a preview or file receipt", () => { + const result = matchRunOutputs( + branches.map((block) => ({ + ...block, + fileOutput: { volumes: ["main.apps.files"] }, + })), + exportedModelToRunPayload( + notebookHtml([ + { command: "group_gate()", results: nodeState("group_prod", true) }, + { command: "group_gate()", results: nodeState("group_dev", false) }, + ]), + ), + ); + assert.deepEqual( + result.outputs.map(({ disabled, files }) => ({ disabled, files })), + [ + { disabled: true, files: [] }, + { disabled: false, files: [] }, + ], + ); +}); + +test("preserves HTML content and selects only the published HTML node", () => { + const result = matchRunOutputs( + [HTML_BLOCK], + exportedModelToRunPayload(notebookHtml([htmlCommand("

Report

")])), + ); + assert.deepEqual(result.outputs[0].outcome, { + outcome: "html", + html: "

Report

", + }); +}); + +test("returns upstream guardrail diagnostics alongside selected outputs", () => { + const result = matchRunOutputs( + [branches[0]], + exportedModelToRunPayload( + notebookHtml([ + guardrailCommand("checks", "Quality", warningText), + { + command: 'display(ctx["prod.data"])', + results: displayTable(1, false), + }, + ]), + ), + ); + assert.equal(result.outcome, "outputs"); + assert.equal(result.diagnostics[0].message, 'column "id" has 3 null values'); +}); diff --git a/packages/appkit/src/plugins/designer/tests/runtime.integration.test.mjs b/packages/appkit/src/plugins/designer/tests/runtime.integration.test.mjs new file mode 100644 index 000000000..515e5a09a --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/runtime.integration.test.mjs @@ -0,0 +1,397 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { deflateSync, gzipSync } from "node:zlib"; + +import { test } from "vitest"; + +import { createTestApp } from "../../../testing"; +import { createWorkspaceClient } from "../../../workspace-client"; +import { designer } from "../plugin"; + +async function listen(t, server) { + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.onTestFinished(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }); + return `http://127.0.0.1:${server.address().port}`; +} + +async function harness(t, env = {}) { + const revision = "a".repeat(64); + const parameters = { + _lb_runner_revision: revision, + hidden: "published default", + choice: "A", + }; + const state = { + requests: [], + submissions: [], + failAcceptedSubmission: false, + cancellations: [], + runs: [], + manifest: { + version: 8, + appName: "Report", + runner: { revision, parameters }, + parameters: [ + { name: "choice", label: "Choice", type: "text", defaultValue: "A" }, + ], + blocks: [ + { + type: "output", + id: "out", + nodeId: "source", + port: "data", + label: "Data", + }, + ], + }, + job: { + settings: { + tasks: [ + { + notebook_task: { + notebook_path: `/Users/author/app/runner-${revision}.designer.py`, + base_parameters: { ...parameters }, + }, + }, + ], + }, + }, + }; + const workspaceUrl = await listen( + t, + createServer(async (req, res) => { + let text = ""; + for await (const chunk of req) text += chunk; + const body = text ? JSON.parse(text) : undefined; + const url = new URL(req.url, "http://workspace.test"); + state.requests.push({ method: req.method, path: url.pathname, body }); + assert.equal(req.headers.authorization, "Bearer test-only"); + res.setHeader("Content-Type", "application/json"); + const send = (value) => res.end(JSON.stringify(value)); + if (url.pathname.endsWith("/Me")) + return send({ id: "app-service-principal" }); + if (url.pathname.endsWith("/jobs/get")) return send(state.job); + if (url.pathname.endsWith("/workspace/export")) { + return send({ + content: Buffer.from(JSON.stringify(state.manifest)).toString( + "base64", + ), + }); + } + if (url.pathname.endsWith("/jobs/run-now")) { + const previous = state.submissions.find( + (submission) => + submission.idempotency_token === body.idempotency_token, + ); + if (previous) { + const run = state.runs.find((run) => + run.job_parameters.some( + ({ name, value }) => + name === "_lb_app_submission_id" && + value === previous.job_parameters._lb_app_submission_id, + ), + ); + return send({ run_id: run.run_id }); + } + state.submissions.push(body); + const run = { + run_id: 1000 + state.submissions.length, + job_id: 10, + start_time: Date.now(), + job_parameters: Object.entries(body.job_parameters).map( + ([name, value]) => ({ name, value }), + ), + state: { life_cycle_state: "RUNNING" }, + }; + state.runs.unshift(run); + if (state.failAcceptedSubmission) { + state.failAcceptedSubmission = false; + res.statusCode = 500; + return send({ + error_code: "INTERNAL_ERROR", + message: "Unexpected error", + }); + } + return send({ run_id: run.run_id }); + } + if (url.pathname.endsWith("/jobs/runs/export")) + return send({ views: [{ content: state.exportedHtml }] }); + if (url.pathname.endsWith("/jobs/runs/get")) { + return send( + state.runs.find( + (run) => run.run_id === Number(url.searchParams.get("run_id")), + ), + ); + } + if (url.pathname.endsWith("/jobs/runs/cancel")) { + state.cancellations.push(body.run_id); + return send({}); + } + if (url.pathname.endsWith("/jobs/runs/list")) { + const completed = url.searchParams.get("completed_only") === "true"; + const runs = state.runs.filter( + (run) => Boolean(run.state.result_state) === completed, + ); + const offset = Number(url.searchParams.get("page_token") || 0); + const end = offset + Number(url.searchParams.get("limit") || 25); + return send({ + runs: runs.slice(offset, end), + ...(end < runs.length ? { next_page_token: String(end) } : {}), + }); + } + res.statusCode = 404; + return send({ error_code: "NOT_FOUND", message: url.pathname }); + }), + ); + const app = await createTestApp({ + plugins: [designer()], + client: createWorkspaceClient({ host: workspaceUrl, token: "test-only" }), + env: { DATABRICKS_JOB_ID: "10", DATABRICKS_APP_ID: "app-id", ...env }, + }); + t.onTestFinished(() => app.close()); + const appUrl = app.baseUrl; + const call = ( + path, + viewer = "alice", + method = "GET", + body, + contentEncoding, + ) => + fetch(`${appUrl}/api/designer${path}`, { + method, + headers: { + "content-type": "application/json", + "x-forwarded-user": viewer, + "x-forwarded-email": `${viewer}@example.test`, + ...(contentEncoding ? { "content-encoding": contentEncoding } : {}), + }, + ...(body === undefined + ? {} + : { + body: + contentEncoding === "gzip" + ? gzipSync(JSON.stringify(body)) + : contentEncoding === "deflate" + ? deflateSync(JSON.stringify(body)) + : JSON.stringify(body), + }), + }); + return { state, call, app }; +} + +test("ordinary App runs stay within the submitting viewer without granting Job permissions", async (t) => { + const { state, call } = await harness(t); + const config = await (await call("/config")).json(); + assert.equal(config.runnable, true); + assert.equal(config.manifest.runner, undefined); + const submitted = await call("/run", "alice", "POST", { + params: { choice: "B", hidden: "override", _lb_app_viewer: "bob" }, + }); + assert.equal(submitted.status, 200); + const { jobRunId } = await submitted.json(); + assert.equal(state.submissions[0].job_parameters.hidden, "published default"); + const run = await (await call(`/run/${jobRunId}`)).json(); + assert.deepEqual(run.parameters, { choice: "B" }); + assert.equal((await call(`/run/${jobRunId}`, "bob")).status, 404); + assert.equal((await call(`/run/${jobRunId}`, "bob", "DELETE")).status, 404); + assert.equal((await call("/run", "", "POST", {})).status, 401); + assert.equal((await call(`/run/${jobRunId}`, "alice", "DELETE")).status, 200); + assert.deepEqual(state.cancellations, [Number(jobRunId)]); + assert.deepEqual( + state.requests.filter(({ path }) => path.includes("/permissions/")), + [], + ); +}); + +test("submissions pause until runner code and every default match the manifest", async (t) => { + const { state, call } = await harness(t); + const notebook = state.job.settings.tasks[0].notebook_task; + const published = structuredClone(notebook); + for (const changed of [ + { + ...published, + notebook_path: `/Users/author/app/runner-${"b".repeat(64)}.designer.py`, + }, + { + ...published, + base_parameters: { ...published.base_parameters, hidden: "new default" }, + }, + { + ...published, + base_parameters: { ...published.base_parameters, added: "new input" }, + }, + ]) { + Object.assign(notebook, changed); + const config = await (await call("/config")).json(); + assert.equal(config.runnable, false); + assert.equal(config.notRunnableReason, "updating"); + const refused = await call("/run", "alice", "POST", {}); + assert.equal(refused.status, 409); + assert.match((await refused.json()).error, /updating/); + } + assert.deepEqual(state.submissions, []); + Object.assign(notebook, published); + assert.equal((await call("/run", "alice", "POST", {})).status, 200); +}); + +test("compressed JSON submissions preserve parameters through the HTTP parser", async (t) => { + const { state, call } = await harness(t); + for (const encoding of ["gzip", "deflate"]) { + const response = await call( + "/run", + "alice", + "POST", + { params: { choice: encoding } }, + encoding, + ); + assert.equal(response.status, 200); + const { jobRunId } = await response.json(); + const run = await (await call(`/run/${jobRunId}`)).json(); + assert.deepEqual(run.parameters, { choice: encoding }); + } + assert.equal(state.runs.length, 2); +}); + +test("SDK retries reuse the submission token without starting a second run", async (t) => { + const { state, call } = await harness(t); + state.failAcceptedSubmission = true; + const response = await call("/run", "alice", "POST", {}); + assert.equal(response.status, 200); + assert.equal((await response.json()).jobRunId, "1001"); + const attempts = state.requests.filter(({ path }) => + path.endsWith("/jobs/run-now"), + ); + assert.equal(attempts.length, 2); + const token = attempts[0].body.idempotency_token; + assert.equal(token, attempts[0].body.job_parameters._lb_app_submission_id); + assert.equal(attempts[1].body.idempotency_token, token); + assert.equal(state.runs.length, 1); +}); + +test("history preserves a continuation beyond 200 other viewers’ runs", async (t) => { + const { state, call } = await harness(t); + const response = await call("/run", "alice", "POST", {}); + const { jobRunId } = await response.json(); + const ownRun = state.runs[0]; + ownRun.state = { life_cycle_state: "TERMINATED", result_state: "SUCCESS" }; + state.runs.unshift( + ...Array.from({ length: 200 }, (_, index) => ({ + ...ownRun, + run_id: 2000 + index, + start_time: ownRun.start_time + 200 - index, + job_parameters: [{ name: "_lb_app_viewer", value: "another-viewer" }], + })), + ); + const first = await (await call("/runs")).json(); + assert.deepEqual(first.runs, []); + assert.equal(first.hasMore, true); + const older = await ( + await call(`/runs?cursor=${first.nextPageToken}`) + ).json(); + assert.deepEqual( + older.runs.map((run) => run.jobRunId), + [jobRunId], + ); + assert.equal(older.hasMore, false); + const search = await (await call("/last-run")).json(); + assert.equal(search.status, "searching"); + const last = await ( + await call(`/last-run?cursor=${search.nextPageToken}`) + ).json(); + assert.equal(last.status, "found"); + assert.equal(last.run.jobRunId, jobRunId); +}); + +test("configuration refresh invalidates incompatible manifests and stops new submissions", async (t) => { + const { state, call } = await harness(t); + assert.equal((await (await call("/config")).json()).runnable, true); + state.manifest.version = 9; + assert.deepEqual(await (await call("/config")).json(), { + manifest: null, + runnable: false, + notRunnableReason: "noManifest", + }); + assert.equal((await call("/run", "alice", "POST", {})).status, 409); + assert.equal(state.submissions.length, 0); +}); + +test("private responses enforce the public App origin and expose only Designer routes", async (t) => { + const { app, state } = await harness(t, { + DATABRICKS_APP_URL: "https://report.example.test", + }); + const response = await app.get("/api/designer/config", { + headers: { origin: "https://report.example.test" }, + }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.equal(response.headers.get("x-content-type-options"), "nosniff"); + assert.equal( + response.headers.get("cross-origin-resource-policy"), + "same-origin", + ); + state.requests.length = 0; + assert.equal( + ( + await app.get("/api/designer/config", { + headers: { origin: "https://other.example.test" }, + }) + ).status, + 403, + ); + assert.equal(state.requests.length, 0); + assert.equal((await app.get("/api/files/list")).status, 404); + assert.equal((await app.post("/api/jobs/default/run")).status, 404); +}); + +test("cached completed results still require current viewer and Job ownership", async (t) => { + const { state, call } = await harness(t); + const { jobRunId } = await (await call("/run", "alice", "POST", {})).json(); + const run = state.runs[0]; + run.state = { life_cycle_state: "TERMINATED", result_state: "SUCCESS" }; + run.tasks = [{ run_id: 11 }]; + const model = { + commands: [ + { + command: 'display(ctx["source.data"])', + results: { + type: "table", + schema: [{ name: "value", type: '"long"' }], + data: [[42]], + overflow: false, + }, + }, + ], + }; + state.exportedHtml = `__DATABRICKS_NOTEBOOK_MODEL = '${Buffer.from(encodeURIComponent(JSON.stringify(model))).toString("base64")}'`; + const first = await (await call(`/run/${jobRunId}`)).json(); + assert.equal(first.result.outputs[0].outcome.payload.rows[0].value, 42); + assert.deepEqual(await (await call(`/run/${jobRunId}`)).json(), first); + assert.equal((await call(`/run/${jobRunId}`, "bob")).status, 404); + run.job_id = 20; + assert.equal((await call(`/run/${jobRunId}`)).status, 404); + assert.equal( + state.requests.filter(({ path }) => path.endsWith("/jobs/runs/export")) + .length, + 1, + ); +}); + +test("a new plugin instance discovers its own manifest after shutdown", async (t) => { + const first = await harness(t); + first.state.manifest.appName = "First App"; + assert.equal( + (await (await first.call("/config")).json()).manifest.appName, + "First App", + ); + await first.app.close(); + const second = await harness(t); + assert.equal( + (await (await second.call("/config")).json()).manifest.appName, + "Report", + ); +}); diff --git a/packages/appkit/src/plugins/designer/tests/storage.test.mjs b/packages/appkit/src/plugins/designer/tests/storage.test.mjs new file mode 100644 index 000000000..7758f145e --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/storage.test.mjs @@ -0,0 +1,309 @@ +import assert from "node:assert/strict"; + +import { afterAll, beforeAll, test } from "vitest"; + +import { FilesConnector } from "../../../connectors/files"; +import { createTestApp } from "../../../testing"; +import { ApiError } from "../../../workspace-client"; +import { createOutputFileStore } from "../server/outputFileStore"; +import { createUploadStore } from "../server/uploadStore"; +let app, appKitUploadStore, appKitOutputFileStore; +const contents = new Map(); +const directories = new Set(); +const config = { + volume: "main.default.designer_app1", + path: "/Volumes/main/default/designer_app1/designer_apps/app1", + maxUploadFileSizeBytes: 5 * 1024 * 1024 * 1024, +}; +let cancelledReads = 0; + +function stored(path) { + const bytes = contents.get(path); + if (!bytes) throw new ApiError("File missing", "NOT_FOUND", 404); + return bytes; +} + +beforeAll(async () => { + // Exercise the real Files connector, replacing only the remote workspace boundary. + const upload = async (path, bytes, overwrite) => { + assert.equal(overwrite, false); + if (contents.has(path)) + throw new ApiError("Already exists", "ALREADY_EXISTS", 409); + contents.set( + path, + bytes instanceof ReadableStream + ? Buffer.from(await new Response(bytes).arrayBuffer()) + : Buffer.from(bytes), + ); + }; + app = await createTestApp({ + plugins: [], + server: false, + client: { + getHost: async () => "https://workspace.invalid", + currentUser: { me: async () => ({ id: "app-service-principal" }) }, + request: async ({ method, path, query, body }) => { + assert.ok(path.startsWith("/api/2.0/fs/files/Volumes/")); + const filePath = decodeURIComponent( + path.slice("/api/2.0/fs/files".length), + ); + if (method === "HEAD") { + return new Response(null, { + headers: { "content-length": String(stored(filePath).length) }, + }); + } + assert.equal(method, "PUT"); + await upload(filePath, body, query.overwrite === "true"); + return new Response(null, { status: 204 }); + }, + files: { + createDirectory: async ({ directoryPath }) => { + directories.add(directoryPath); + }, + downloadFile: async ({ filePath }) => ({ + contents: new ReadableStream( + { + start(controller) { + controller.enqueue(stored(filePath)); + }, + pull(controller) { + controller.close(); + }, + cancel() { + cancelledReads += 1; + }, + }, + { highWaterMark: 0 }, + ), + }), + uploadFile: async ({ filePath, contents: bytes, overwrite }) => + upload(filePath, bytes, overwrite), + listDirectoryContentsIter: async function* ({ directoryPath }) { + const prefix = `${directoryPath}/`; + for (const path of [...directories, ...contents.keys()]) { + if ( + path.startsWith(prefix) && + !path.slice(prefix.length).includes("/") + ) + yield { + name: path.slice(prefix.length), + isDirectory: directories.has(path), + }; + } + }, + deleteFile: async ({ filePath }) => { + contents.delete(filePath); + }, + }, + }, + }); + const connector = new FilesConnector({}); + appKitUploadStore = (config) => + createUploadStore(app.client, connector, config); + appKitOutputFileStore = (volume) => + createOutputFileStore(app.client, connector, volume); +}); + +afterAll(() => app.close()); + +test("native output Files store reads exact original bytes repeatedly across separate volumes", async () => { + await Promise.all( + ["main.default.output_one", "main.default.output_two"].map( + async (volume) => { + const store = appKitOutputFileStore(volume); + const root = `/Volumes/${volume.replaceAll(".", "/")}`; + for (const name of [ + "result.csv", + "multi sheet.xlsx", + "データ.json", + "data #1?100%.csv", + ]) { + const path = `${root}/reports/${name}`; + assert.equal(await store.size(path), undefined); + contents.set(path, Buffer.from("original bytes")); + assert.equal(await store.size(path), 14); + assert.equal( + await new Response(await store.download(path)).text(), + "original bytes", + ); + assert.equal( + await new Response(await store.download(path)).text(), + "original bytes", + ); + assert.equal(contents.get(path).toString(), "original bytes"); + } + for (const path of [ + "/Volumes/main/default/other/report.csv", + `${root}/../other/report.csv`, + `${root}/code.py`, + ]) + await assert.rejects(store.download(path)); + assert.deepEqual(Object.keys(store).sort(), ["download", "size"]); + }, + ), + ); +}); + +test("uses the Files connector for immutable storage, downloads, bounded records, metadata and deletion", async () => { + const store = appKitUploadStore(config); + const folder = `${config.path}/uploads/viewer/parameter`; + await store.mkdir(folder); + await store.mkdir(`${folder}/child`); + for (const filename of [ + "data.csv", + "data.json", + "data #1?100%.csv.gz", + "carmax_car_prices copy (1).xlsx", + "データ.xlsx", + ]) { + const path = `${folder}/${filename}`; + const bytes = Buffer.from(JSON.stringify({ filename })); + await store.put(path, bytes); + assert.deepEqual(contents.get(path), bytes); + assert.equal(await store.size(path), bytes.length); + assert.deepEqual(await store.read(path), { filename }); + assert.deepEqual( + Buffer.from(await new Response(await store.download(path)).arrayBuffer()), + bytes, + ); + await assert.rejects(store.put(path, Buffer.from("replacement")), { + status: 502, + }); + assert.deepEqual(contents.get(path), bytes); + } + const streamedPath = `${folder}/streamed.csv`; + await store.putStream( + streamedPath, + new ReadableStream({ + start(controller) { + controller.enqueue(Buffer.from("streamed")); + controller.close(); + }, + }), + ); + assert.equal(contents.get(streamedPath).toString(), "streamed"); + await store.delete(`${folder}/data.csv`); + await assert.rejects(store.read(`${folder}/data.csv`), { status: 404 }); + await assert.rejects(store.download(`${folder}/data.csv`), { status: 404 }); +}); + +test("uploads and native output reads can share one physical volume without sharing handle policies", async () => { + const uploadStore = appKitUploadStore(config); + const outputStore = appKitOutputFileStore(config.volume); + const outputPath = + "/Volumes/main/default/designer_app1/reports/finished.xlsx"; + const uploadPath = `${config.path}/uploads/viewer/parameter/incoming.csv`; + contents.set(outputPath, Buffer.from("original workbook")); + await Promise.all([ + uploadStore.put(uploadPath, Buffer.from("new upload")), + outputStore.size(outputPath).then((size) => assert.equal(size, 17)), + ]); + assert.equal( + await new Response(await outputStore.download(outputPath)).text(), + "original workbook", + ); + await assert.rejects(uploadStore.read(outputPath), { status: 502 }); + await assert.rejects(uploadStore.download(outputPath), { status: 502 }); + assert.equal(await uploadStore.size(uploadPath), 10); +}); + +test("keeps oversized and malformed completion records unreadable", async () => { + const store = appKitUploadStore(config); + const path = `${config.path}/uploads/viewer/oversized.json`; + contents.set(path, Buffer.alloc(16 * 1024 + 1)); + await assert.rejects(store.read(path), { status: 502 }); + assert.equal(cancelledReads, 1); + contents.set(path, Buffer.from("not JSON")); + await assert.rejects(store.read(path), { status: 409 }); +}); + +test("limits plugin access to the uploads subtree of the configured app storage", async () => { + const store = appKitUploadStore(config); + await assert.rejects( + store.put( + "/Volumes/main/default/uploads/other-app/data.csv", + Buffer.from("data"), + ), + { + status: 502, + }, + ); + for (const path of [ + `${config.path}/exports/results.xlsx`, + `${config.path}/uploads_other/data.csv`, + ]) { + const bytes = Buffer.from("reserved"); + contents.set(path, bytes); + await assert.rejects(store.read(path), { status: 502 }); + await assert.rejects(store.download(path), { status: 502 }); + await assert.rejects(store.put(path, Buffer.from("replacement")), { + status: 502, + }); + await assert.rejects(store.delete(path), { status: 502 }); + assert.deepEqual(contents.get(path), bytes); + } + await assert.rejects( + appKitUploadStore(undefined).read(`${config.path}/file`), + { status: 409 }, + ); +}); + +test("switches volumes in one process while overlapping requests keep their own storage policies", async () => { + const next = { + ...config, + volume: "main.default.designer_app2", + path: "/Volumes/main/default/designer_app2/designer_apps/app1", + }; + const before = appKitUploadStore(config); + const after = appKitUploadStore(next); + const beforePath = `${config.path}/uploads/viewer/parameter/old.csv`; + const afterPath = `${next.path}/uploads/viewer/parameter/new.csv`; + await Promise.all([ + before.put(beforePath, Buffer.from("old file")), + after.put(afterPath, Buffer.from("new file")), + ]); + assert.equal(await before.size(beforePath), 8); + assert.equal(await after.size(afterPath), 8); + await assert.rejects(after.read(beforePath), { status: 502 }); + await assert.rejects(before.read(afterPath), { status: 502 }); + await appKitUploadStore(config).put( + `${config.path}/uploads/viewer/parameter/back.csv`, + Buffer.from("fresh"), + ); + assert.equal(contents.get(beforePath).toString(), "old file"); + assert.equal(contents.get(afterPath).toString(), "new file"); +}); + +test("allows republishing a different app root without changing the volume or sharing plugin policies", async () => { + const previous = { ...config, path: `${config.path}_previous` }; + const previousStore = appKitUploadStore(previous); + const store = appKitUploadStore(config); + const beforePath = `${previous.path}/uploads/viewer/parameter/before.csv`; + const afterPath = `${config.path}/uploads/viewer/parameter/after.csv`; + + await previousStore.put(beforePath, Buffer.from("before")); + await store.put(afterPath, Buffer.from("after")); + assert.equal(contents.get(beforePath).toString(), "before"); + assert.equal(contents.get(afterPath).toString(), "after"); + + // In-flight requests keep their own scope even after another request reads the new manifest. + await assert.rejects(previousStore.read(afterPath), { status: 502 }); + await Promise.all([ + previousStore.put( + `${previous.path}/uploads/viewer/concurrent.csv`, + Buffer.from("previous scope"), + ), + store.put( + `${config.path}/uploads/viewer/concurrent.csv`, + Buffer.from("current scope"), + ), + ]); + assert.equal( + contents.get(`${previous.path}/uploads/viewer/concurrent.csv`).toString(), + "previous scope", + ); + assert.equal( + contents.get(`${config.path}/uploads/viewer/concurrent.csv`).toString(), + "current scope", + ); +}); diff --git a/packages/appkit/src/plugins/designer/tests/uploads.integration.test.mjs b/packages/appkit/src/plugins/designer/tests/uploads.integration.test.mjs new file mode 100644 index 000000000..22b8e8d5e --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/uploads.integration.test.mjs @@ -0,0 +1,177 @@ +import assert from "node:assert/strict"; + +import { test } from "vitest"; + +import { createTestApp } from "../../../testing"; +import { ApiError } from "../../../workspace-client"; +import { designer } from "../plugin"; + +async function harness(t) { + const revision = "a".repeat(64); + const defaults = { _lb_runner_revision: revision, path: "" }; + const manifest = { + version: 8, + appName: "Upload report", + runner: { revision, parameters: defaults }, + storage: { + volume: "main.apps.files", + path: "/Volumes/main/apps/files/designer_apps/app1", + maxUploadFileSizeBytes: 1024, + }, + parameters: [ + { + name: "path", + label: "File", + type: "file", + defaultValue: "", + fileFormats: ["csv"], + }, + ], + blocks: [{ type: "output", id: "out", nodeId: "source", port: "data" }], + }; + const files = new Map(); + const stored = (path) => { + const value = files.get(path); + if (!value) throw new ApiError("Not found", "NOT_FOUND", 404); + return value; + }; + const put = async (path, body, overwrite) => { + assert.equal(overwrite, false); + assert.equal(files.has(path), false); + files.set(path, Buffer.from(await new Response(body).arrayBuffer())); + }; + const submissions = []; + // Replace only Databricks services; AppKit routes, SDK adapter, and streams are real. + const app = await createTestApp({ + plugins: [designer()], + env: { DATABRICKS_JOB_ID: "10", DATABRICKS_APP_ID: "app-id" }, + responses: { + request: async ({ method, path, query, body }) => { + if (path.endsWith("/jobs/get")) + return Response.json({ + settings: { + tasks: [ + { + notebook_task: { + notebook_path: `/Users/author/app/runner-${revision}.designer.py`, + base_parameters: defaults, + }, + }, + ], + }, + }); + if (path.endsWith("/workspace/export")) + return Response.json({ + content: Buffer.from(JSON.stringify(manifest)).toString("base64"), + }); + assert.ok(path.startsWith("/api/2.0/fs/files/Volumes/")); + const filePath = decodeURIComponent( + path.slice("/api/2.0/fs/files".length), + ); + if (method === "HEAD") + return new Response(null, { + headers: { "content-length": String(stored(filePath).length) }, + }); + assert.equal(method, "PUT"); + await put(filePath, body, query.overwrite === "true"); + return new Response(null, { status: 204 }); + }, + "files.createDirectory": async () => {}, + "files.uploadFile": async ({ filePath, contents, overwrite }) => + put(filePath, contents, overwrite), + "files.downloadFile": async ({ filePath }) => ({ + contents: new Response(stored(filePath)).body, + }), + "files.deleteFile": async ({ filePath }) => { + files.delete(filePath); + }, + "jobs.runNow": async (submission) => { + submissions.push(submission); + return { runId: 42 }; + }, + }, + }); + t.onTestFinished(() => app.close()); + const headers = { + "x-forwarded-user": "alice", + "x-forwarded-email": "alice@example.test", + }; + const upload = (body) => + app.post("/api/designer/uploads/path", { + body, + headers: { + ...headers, + "content-type": "application/octet-stream", + "x-file-name": "sales.csv", + }, + }); + return { app, headers, upload, files, manifest, submissions }; +} + +test("streamed uploads can be downloaded and passed to runs through the plugin", async (t) => { + const { app, headers, upload, files, manifest, submissions } = + await harness(t); + const before = Object.entries(process.env).filter(([key]) => + key.startsWith("DATABRICKS_VOLUME_"), + ); + const response = await upload("price\n42\n"); + assert.equal(response.status, 201); + const receipt = (await response.json()).upload; + const path = `/api/designer/uploads/path/${encodeURIComponent(receipt.reference)}/download`; + assert.equal(await (await app.get(path, { headers })).text(), "price\n42\n"); + assert.equal( + (await app.get(path, { headers: { "x-forwarded-user": "bob" } })).status, + 404, + ); + const run = await app.post("/api/designer/run", { + headers, + body: { params: { path: receipt.reference } }, + }); + assert.deepEqual(await run.json(), { jobRunId: "42" }); + assert.equal( + files.get(submissions[0].jobParameters.path).toString(), + "price\n42\n", + ); + assert.ok( + submissions[0].jobParameters.path.startsWith( + manifest.storage.path + "/uploads/", + ), + ); + assert.deepEqual( + Object.entries(process.env).filter(([key]) => + key.startsWith("DATABRICKS_VOLUME_"), + ), + before, + ); + manifest.storage.path = "/Volumes/main/apps/files/designer_apps/app2"; + assert.equal((await app.get(path, { headers })).status, 404); + assert.equal( + ( + await app.post("/api/designer/run", { + headers, + body: { params: { path: receipt.reference } }, + }) + ).status, + 400, + ); +}); + +test("stream errors and invalid file formats do not leave a completed upload", async (t) => { + const { app, headers, upload, files } = await harness(t); + assert.equal((await upload("a".repeat(1025))).status, 413); + assert.equal(files.size, 0); + assert.equal( + ( + await app.post("/api/designer/uploads/path", { + body: "data", + headers: { + ...headers, + "content-type": "application/octet-stream", + "x-file-name": "report.xlsx", + }, + }) + ).status, + 400, + ); + assert.equal(files.size, 0); +}); diff --git a/packages/appkit/src/plugins/designer/tests/uploads.test.mjs b/packages/appkit/src/plugins/designer/tests/uploads.test.mjs new file mode 100644 index 000000000..657f5f062 --- /dev/null +++ b/packages/appkit/src/plugins/designer/tests/uploads.test.mjs @@ -0,0 +1,730 @@ +import assert from "node:assert/strict"; + +import { test } from "vitest"; + +import * as uploads from "../server/fileUploads"; +import * as parameters from "../server/runParameters"; +const storage = { + volume: "main.default.designer_app1", + path: "/Volumes/main/default/designer_app1/designer_apps/app1", + maxUploadFileSizeBytes: 5 * 1024 * 1024 * 1024, +}; + +const fileParameter = { + name: "path", + type: "file", + label: "Data", + defaultValue: "/Volumes/private/author.csv", +}; + +const manifest = { + version: 8, + appName: "Uploads", + storage, + parameters: [fileParameter], + blocks: [ + { + type: "output", + id: "data", + nodeId: "source", + port: "data", + label: "Data", + }, + ], +}; + +function memoryStore() { + const files = new Map(); + return { + files, + mkdir: async () => {}, + put: async (path, bytes) => { + assert.equal(files.has(path), false); + files.set(path, Buffer.from(bytes)); + }, + putStream: async (path, stream) => { + assert.equal(files.has(path), false); + files.set(path, Buffer.from(await new Response(stream).arrayBuffer())); + }, + read: async (path) => { + if (!files.has(path)) throw new uploads.UploadError(404, "missing"); + return JSON.parse(files.get(path).toString()); + }, + size: async (path) => files.get(path)?.length, + delete: async (path) => { + files.delete(path); + }, + }; +} + +test("a volume switch requires a fresh upload while leaving previous files unchanged", async () => { + const store = memoryStore(); + const viewer = uploads.viewerKey("alice", "100"); + const previous = await uploads.saveUpload( + store, + storage, + viewer, + "path", + "old.csv", + Buffer.from("original"), + ); + const before = new Map(store.files); + const next = { + ...storage, + volume: "main.default.other", + path: "/Volumes/main/default/other/designer_apps/app1", + }; + const refused = await parameters.resolveRunParameters( + { ...manifest, storage: next }, + { path: previous.reference }, + viewer, + store, + ); + assert.equal(refused.ok, false); + assert.equal(refused.code, "UPLOAD_UNAVAILABLE"); + assert.match(refused.error, /Upload the file again/); + assert.deepEqual(store.files, before); + const fresh = await uploads.saveUpload( + store, + next, + viewer, + "path", + "fresh.csv", + Buffer.from("fresh"), + ); + const accepted = await parameters.resolveRunParameters( + { ...manifest, storage: next }, + { path: fresh.reference }, + viewer, + store, + ); + assert.equal(accepted.ok, true); + assert.ok(accepted.params.path.startsWith(next.path)); + for (const [path, bytes] of before) + assert.deepEqual(store.files.get(path), bytes); + // The simple switch does not introduce permanent expiry across a missed A-to-B-to-A cycle. + assert.equal( + ( + await parameters.resolveRunParameters( + manifest, + { path: previous.reference }, + viewer, + store, + ) + ).ok, + true, + ); +}); + +test("temporary storage failures keep the completed upload available for an explicit retry", async () => { + const store = memoryStore(); + const viewer = uploads.viewerKey("alice", "100"); + const upload = await uploads.saveUpload( + store, + storage, + viewer, + "path", + "data.csv", + Buffer.from("value\n1"), + ); + const unavailable = { + ...store, + read: async () => { + throw new uploads.UploadError(502, "Could not access upload storage."); + }, + }; + const refused = await parameters.resolveRunParameters( + manifest, + { path: upload.reference }, + viewer, + unavailable, + ); + assert.equal(refused.ok, false); + assert.equal(refused.code, undefined); + assert.equal(refused.error, "Could not access upload storage."); + const retried = await parameters.resolveRunParameters( + manifest, + { path: upload.reference }, + viewer, + store, + ); + assert.equal(retried.ok, true); + assert.ok(retried.params.path.endsWith("/data.csv")); +}); + +test("saves immutable bytes, resolves an upload after service restart, and isolates viewer/parameter/app", async () => { + const store = memoryStore(); + const alice = uploads.viewerKey("alice", "100"); + const saved = await uploads.saveUpload( + store, + storage, + alice, + "path", + "sales.csv", + Buffer.from("a,b\n1,2\n"), + ); + const resolved = await uploads.resolveUpload( + { ...store }, + storage, + alice, + "path", + saved.reference, + ); + assert.ok(resolved.path.startsWith(`${storage.path}/uploads/${alice}/`)); + assert.equal( + resolved.path.slice(storage.path.length + 1).split("/").length, + 5, + ); + assert.equal(store.files.get(resolved.path).toString(), "a,b\n1,2\n"); + for (const [owner, name, root] of [ + [uploads.viewerKey("bob", "100"), "path", storage], + [alice, "other", storage], + [uploads.viewerKey("alice", "200"), "path", storage], + [alice, "path", { ...storage, path: storage.path + "2" }], + ]) { + await assert.rejects( + uploads.resolveUpload(store, root, owner, name, saved.reference), + /missing/, + ); + } +}); + +test("substitutes only completed owned uploads and stamps ownership for run parameters", async () => { + const store = memoryStore(); + const owner = uploads.viewerKey("alice", "100"); + const saved = await uploads.saveUpload( + store, + storage, + owner, + "path", + "data.csv", + Buffer.from("a\n1\n"), + ); + const resolved = await parameters.resolveRunParameters( + manifest, + { + path: saved.reference, + ld_display_outputs_for: "evil", + _lb_collect_row_counts: "false", + _lb_app_viewer: "bob", + target_node: "evil", + ignored: "value", + }, + owner, + store, + ); + assert.equal(resolved.ok, true); + assert.deepEqual(resolved.params, { + _lb_file_outputs: "{}", + _lb_file_output_volume: "", + path: ( + await uploads.resolveUpload( + store, + storage, + owner, + "path", + saved.reference, + ) + ).path, + ld_display_outputs_for: "source", + _lb_collect_row_counts: "true", + _lb_app_viewer: owner, + _lb_app_parameters: JSON.stringify({ + path: { label: "Data", type: "file", value: "data.csv" }, + }), + }); + for (const value of [ + "", + "/Volumes/main/default/uploads/author.csv", + "../../file", + "upload:bad-id", + ]) { + assert.equal( + ( + await parameters.resolveRunParameters( + manifest, + { path: value }, + owner, + store, + ) + ).ok, + false, + ); + } + assert.equal( + ( + await parameters.resolveRunParameters( + manifest, + { path: saved.reference }, + undefined, + store, + ) + ).ok, + false, + ); + assert.equal( + ( + await parameters.resolveRunParameters( + manifest, + { path: saved.reference }, + uploads.viewerKey("bob", "100"), + store, + ) + ).ok, + false, + ); + assert.equal( + ( + await parameters.resolveRunParameters( + { ...manifest, storage: undefined }, + { path: saved.reference }, + owner, + store, + ) + ).ok, + false, + ); +}); + +test("never makes incomplete or changed uploads available", async () => { + const store = memoryStore(); + const owner = uploads.viewerKey("alice", "100"); + const failing = { + ...store, + put: async (path, bytes) => { + if (path.endsWith(".json")) throw new Error("metadata unavailable"); + await store.put(path, bytes); + }, + }; + await assert.rejects( + uploads.saveUpload( + failing, + storage, + owner, + "path", + "data.csv", + Buffer.from("a\n1\n"), + ), + /metadata unavailable/, + ); + assert.equal(store.files.size, 0); + const saved = await uploads.saveUpload( + store, + storage, + owner, + "path", + "data.csv", + Buffer.from("a\n1\n"), + ); + const { path } = await uploads.resolveUpload( + store, + storage, + owner, + "path", + saved.reference, + ); + store.files.set(path, Buffer.from("changed")); + await assert.rejects( + uploads.resolveUpload(store, storage, owner, "path", saved.reference), + /has changed/, + ); +}); + +test("streams uploads while enforcing actual byte limits, empty files and interrupted requests", async () => { + const store = memoryStore(); + const stream = (...chunks) => + new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(Buffer.from(chunk)); + controller.close(); + }, + }); + const saved = await uploads.saveUploadStream( + store, + { ...storage, maxUploadFileSizeBytes: 4 }, + "viewer", + "path", + "data.csv", + stream("ab", "cd"), + 4, + ); + assert.equal( + ( + await uploads.resolveUpload( + store, + { ...storage, maxUploadFileSizeBytes: 4 }, + "viewer", + "path", + saved.reference, + ) + ).upload.size, + 4, + ); + + await assert.rejects( + uploads.saveUploadStream( + store, + { ...storage, maxUploadFileSizeBytes: 4 }, + "viewer", + "path", + "large.csv", + stream("ab", "cde"), + ), + /5 GB/, + ); + await assert.rejects( + uploads.saveUploadStream( + store, + storage, + "viewer", + "path", + "empty.csv", + stream(), + ), + /non-empty/, + ); + await assert.rejects( + uploads.saveUploadStream( + store, + storage, + "viewer", + "path", + "interrupted.csv", + new ReadableStream({ + start(controller) { + controller.enqueue(Buffer.from("a")); + controller.error(new Error("disconnected")); + }, + }), + ), + /disconnected/, + ); + assert.equal( + [...store.files.keys()].filter((path) => + /large|empty|interrupted/.test(path), + ).length, + 0, + ); +}); + +test("rejects unsafe filenames and declared sizes", async () => { + const store = memoryStore(); + for (const filename of [ + "", + " ", + ".", + "..", + "../data.csv", + "a\\b.csv", + "a\n.csv", + "é".repeat(128), + ]) { + await assert.rejects( + uploads.saveUpload( + store, + storage, + "viewer", + "path", + filename, + Buffer.from("a"), + ), + /valid filename/, + ); + } + await assert.rejects( + uploads.saveUpload( + store, + { ...storage, maxUploadFileSizeBytes: 1 }, + "viewer", + "path", + "data.csv", + Buffer.from("aa"), + ), + /at most/, + ); + assert.equal(store.files.size, 0); +}); + +for (const filename of [ + "data.csv", + "workbook.xlsx", + "workbook.xls", + "data.json", + "data.parquet", + "data.csv.gz", + "データ.xlsx", + "data", +]) { + test(`preserves ${filename}, its bytes and its reference through job parameter resolution`, async () => { + const store = memoryStore(); + const viewer = uploads.viewerKey("alice", "100"); + const bytes = Buffer.from([0, 1, 127, 128, 255]); + const saved = await uploads.saveUpload( + store, + storage, + viewer, + "path", + filename, + bytes, + ); + const resolved = await uploads.resolveUpload( + { ...store }, + storage, + viewer, + "path", + saved.reference, + ); + assert.ok( + resolved.path.endsWith( + `/${saved.reference.slice("upload:".length)}/${filename}`, + ), + ); + assert.deepEqual(store.files.get(resolved.path), bytes); + assert.deepEqual( + await parameters.resolveRunParameters( + manifest, + { path: saved.reference }, + viewer, + store, + ), + { + ok: true, + params: { + _lb_file_outputs: "{}", + _lb_file_output_volume: "", + path: resolved.path, + ld_display_outputs_for: "source", + _lb_collect_row_counts: "true", + _lb_app_viewer: viewer, + _lb_app_parameters: JSON.stringify({ + path: { label: "Data", type: "file", value: filename }, + }), + }, + }, + ); + assert.equal(store.files.size, 2); + }); +} + +test("refuses an upload whose completion record contains an invalid filename", async () => { + const store = memoryStore(); + const viewer = uploads.viewerKey("alice", "100"); + const saved = await uploads.saveUpload( + store, + storage, + viewer, + "path", + "data.json", + Buffer.from("{}"), + ); + const id = saved.reference.slice("upload:".length); + const sidecar = [...store.files.keys()].find((path) => + path.endsWith(`/${id}.json`), + ); + store.files.set( + sidecar, + Buffer.from(JSON.stringify({ ...saved, filename: "../other.json" })), + ); + await assert.rejects( + uploads.resolveUpload(store, storage, viewer, "path", saved.reference), + /incomplete or unreadable/, + ); +}); + +test("run ownership protects history, results and cancellation even after upload inputs are removed", () => { + const alice = uploads.viewerKey("alice", "100"); + const bob = uploads.viewerKey("bob", "100"); + const privateRun = { + job_parameters: [{ name: "_lb_app_viewer", value: alice }], + }; + assert.equal(uploads.canAccessRun(privateRun, alice), true); + assert.equal(uploads.canAccessRun(privateRun, bob), false); + assert.equal(uploads.canAccessRun(privateRun, undefined), false); + assert.equal(uploads.canAccessRun({}, alice), false); + assert.equal(uploads.viewerKey(undefined, "100"), undefined); + assert.equal(uploads.viewerKey(" ", "100"), undefined); +}); + +test("ordinary parameters retain defaults and dropdown validation with authenticated ownership", async () => { + const regular = { + parameters: [ + { + name: "choice", + label: "Choice", + type: "dropdown", + defaultValue: "A", + choices: ["A", "B"], + }, + ], + }; + assert.deepEqual( + await parameters.resolveRunParameters(regular, {}, "alice", memoryStore()), + { + ok: true, + params: { + choice: "A", + _lb_file_outputs: "{}", + _lb_file_output_volume: "", + _lb_app_viewer: "alice", + _lb_app_parameters: JSON.stringify({ + choice: { label: "Choice", type: "dropdown", value: "A" }, + }), + }, + }, + ); + assert.equal( + ( + await parameters.resolveRunParameters( + regular, + { choice: "C" }, + "alice", + memoryStore(), + ) + ).ok, + false, + ); + assert.equal( + ( + await parameters.resolveRunParameters( + regular, + {}, + undefined, + memoryStore(), + ) + ).ok, + false, + ); +}); + +test("preview-only and empty selections explicitly clear inherited file-output Job policies", async () => { + const stalePolicy = JSON.stringify({ + output_0: { volumes: ["main.apps.files"] }, + }); + for (const blocks of [undefined, [], [manifest.blocks[0]]]) { + const resolved = await parameters.resolveRunParameters( + { parameters: [], blocks }, + { _lb_file_outputs: stalePolicy }, + "alice", + memoryStore(), + ); + assert.equal(resolved.ok, true); + assert.equal(resolved.params._lb_file_outputs, "{}"); + assert.equal(resolved.params._lb_app_viewer, "alice"); + // Jobs may merge request overrides with defaults from a newer, partially published runner. + const effectiveParams = { + _lb_file_outputs: stalePolicy, + ...resolved.params, + }; + assert.deepEqual(JSON.parse(effectiveParams._lb_file_outputs), {}); + if (blocks?.length) + assert.equal(resolved.params._lb_collect_row_counts, "true"); + } +}); + +test("ordinary parameters accept scalar values and reject structured values before submission", async () => { + const regular = { + parameters: [ + { name: "value", label: "Value", type: "text", defaultValue: "default" }, + ], + }; + for (const [value, expected] of [ + ["hello", "hello"], + [42, "42"], + [false, "false"], + [null, "default"], + ]) { + const resolved = await parameters.resolveRunParameters( + regular, + { value }, + "alice", + memoryStore(), + ); + assert.equal(resolved.ok, true); + assert.equal(resolved.params.value, expected); + } + for (const value of [{ nested: "value" }, ["a", "b"]]) { + const resolved = await parameters.resolveRunParameters( + regular, + { value }, + "alice", + memoryStore(), + ); + assert.deepEqual(resolved, { + ok: false, + error: '"Value" must be a string, number or boolean.', + }); + } +}); + +test("each submitted file-output run gets a fresh server-owned namespace, never a consumer value", async () => { + const submittedNamespace = "00000000-0000-4000-8000-000000000000"; + const fileManifest = { + parameters: [ + { + name: "_lb_output_namespace", + label: "Reserved", + type: "text", + defaultValue: submittedNamespace, + }, + ], + blocks: [ + { + type: "output", + nodeId: "output_0", + fileOutput: { volumes: ["main.apps.files"] }, + }, + ], + }; + assert.equal(parameters.isReservedParameter("_lb_output_namespace"), true); + const namespaces = []; + for (const viewer of ["alice", "alice", "bob"]) { + const resolved = await parameters.resolveRunParameters( + fileManifest, + { + _lb_output_namespace: submittedNamespace, + }, + viewer, + memoryStore(), + ); + assert.equal(resolved.ok, true); + const namespace = resolved.params._lb_output_namespace; + assert.match( + namespace, + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + assert.notEqual(namespace, submittedNamespace); + namespaces.push(namespace); + } + assert.equal(new Set(namespaces).size, namespaces.length); + const previewOnly = await parameters.resolveRunParameters( + { ...fileManifest, blocks: [] }, + { + _lb_output_namespace: submittedNamespace, + }, + "alice", + memoryStore(), + ); + assert.equal(previewOnly.ok, true); + assert.equal(previewOnly.params._lb_output_namespace, undefined); +}); + +test("rechecks retained uploads against the current publication before starting a run", async () => { + const store = memoryStore(); + const owner = uploads.viewerKey("alice", "100"); + const saved = await uploads.saveUpload( + store, + storage, + owner, + "path", + "data.csv", + Buffer.from("a\n1\n"), + ); + const result = await parameters.resolveRunParameters( + { ...manifest, parameters: [{ ...fileParameter, fileFormats: ["excel"] }] }, + { path: saved.reference }, + owner, + store, + ); + assert.equal(result.ok, false); + assert.match(result.error, /expects Excel/); +}); diff --git a/packages/appkit/tsdown.config.ts b/packages/appkit/tsdown.config.ts index bb17c29ba..cf6ca7aab 100644 --- a/packages/appkit/tsdown.config.ts +++ b/packages/appkit/tsdown.config.ts @@ -11,6 +11,7 @@ export default defineConfig([ entry: [ "src/index.ts", "src/beta.ts", + "src/designer.ts", "src/testing/index.ts", "src/tsdown/index.ts", "src/type-generator/index.ts", diff --git a/template/appkit.plugins.json b/template/appkit.plugins.json index db53b96b7..6e2827f24 100644 --- a/template/appkit.plugins.json +++ b/template/appkit.plugins.json @@ -196,6 +196,41 @@ } } }, + "designer": { + "name": "designer", + "displayName": "Lakeflow Designer", + "description": "Serve published Lakeflow Designer Apps with viewer-scoped runs, results, and file transfers.", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [ + { + "type": "job", + "alias": "Designer runner", + "resourceKey": "job", + "description": "Publisher-owned Designer runner Job", + "permission": "CAN_MANAGE", + "fields": { + "id": { + "env": "DATABRICKS_JOB_ID", + "description": "Job bound by the Designer publisher", + "origin": "user" + } + }, + "binding": { + "yamlKey": "job", + "varFields": [ + [ + "id", + "id" + ] + ] + } + } + ] + }, + "stability": "beta" + }, "files": { "name": "files", "displayName": "Files Plugin",