diff --git a/.github/workflows/agent-task-contracts.yml b/.github/workflows/agent-task-contracts.yml index 44b4f5446..d37425f40 100644 --- a/.github/workflows/agent-task-contracts.yml +++ b/.github/workflows/agent-task-contracts.yml @@ -2,109 +2,21 @@ name: Agent Task Contracts on: pull_request: - paths: - - ".github/scripts/run-agent-task/**" - - ".github/workflows/agent-task-contracts.yml" - - ".github/workflows/build-php-wasm-mysqli-poll-overlay.yml" - - ".github/workflows/run-agent-task.yml" - - "contracts/agent-task-workflow-request.fixture.json" - - "contracts/run-agent-task-reusable-workflow-interface.v1.json" - - "docs/agent-task-reusable-workflow.md" - - "fixtures/agent-task-reusable-workflow-consumer*.yml" - - "fixtures/agent-task-runtime-sources-run-29299109269.json" - - "fixtures/agent-task-runtime-paths-run-29305012941.json" - - "npm-shrinkwrap.json" - - "package.json" - - "scripts/process-marker.mjs" - - "packages/runtime-playground/src/**" - - "tests/browser-*.test.ts" - - "tests/editor-*.test.ts" - - "tests/agent-task-*.test.ts" - - "tests/runtime-sources-materialization.test.ts" - - "tests/runtime-sources-playground-integration.test.ts" - - "tests/execute-native-agent-task-playground-e2e.test.ts" - - "tests/execute-native-agent-task-interruption.test.mjs" - - "tests/trusted-apply-artifact-channel.integration.test.ts" - - "tests/redaction.test.ts" - - "tests/production-boundary-enforcement.test.ts" - - "tests/runtime-tool-policy.test.ts" - - "tests/browser-canonical-preview-origin.test.ts" - - "tests/browser-adaptive-exploration.test.ts" - - "packages/runtime-core/src/browser-adaptive-exploration.ts" - - "packages/runtime-playground/src/browser-actions-runner.ts" - - "packages/runtime-playground/src/browser-adaptive-explorer.ts" - - "packages/runtime-playground/src/browser-artifacts.ts" - - "packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts" - - "packages/runtime-playground/src/browser-preview-routing.ts" - - "packages/runtime-playground/src/browser-probe-runner.ts" - - "packages/runtime-playground/src/editor-command-runners.ts" - - "packages/runtime-playground/src/preview-server.ts" - - "packages/runtime-playground/src/phpunit-command-handlers.ts" - - "packages/runtime-playground/src/playground-cli-runner.ts" - - "packages/runtime-playground/src/playground-wordpress-archive-cache.ts" - - "tests/playground-custom-archive-cache*.test.ts" - - "tests/fixtures/playground-cache-lease-child.ts" - - "tests/playground-phpunit-readonly-cache.integration.test.ts" - - "tests/fixtures/phpunit-playground-harness/**" - - "packages/cli/src/bounded-recipe-plan.ts" - - "packages/runtime-core/src/bounded-runtime-plan.ts" - - "tests/bounded-*.test.ts" - - "tests/disposable-mysql-mysqli.integration.test.ts" - - "tests/mysqli-poll.integration.test.ts" push: - paths: - - ".github/scripts/run-agent-task/**" - - ".github/workflows/agent-task-contracts.yml" - - ".github/workflows/build-php-wasm-mysqli-poll-overlay.yml" - - ".github/workflows/run-agent-task.yml" - - "contracts/agent-task-workflow-request.fixture.json" - - "contracts/run-agent-task-reusable-workflow-interface.v1.json" - - "docs/agent-task-reusable-workflow.md" - - "fixtures/agent-task-reusable-workflow-consumer*.yml" - - "fixtures/agent-task-runtime-sources-run-29299109269.json" - - "fixtures/agent-task-runtime-paths-run-29305012941.json" - - "npm-shrinkwrap.json" - - "package.json" - - "scripts/process-marker.mjs" - - "packages/runtime-playground/src/**" - - "tests/browser-*.test.ts" - - "tests/editor-*.test.ts" - - "tests/agent-task-*.test.ts" - - "tests/runtime-sources-materialization.test.ts" - - "tests/runtime-sources-playground-integration.test.ts" - - "tests/execute-native-agent-task-playground-e2e.test.ts" - - "tests/execute-native-agent-task-interruption.test.mjs" - - "tests/trusted-apply-artifact-channel.integration.test.ts" - - "tests/redaction.test.ts" - - "tests/production-boundary-enforcement.test.ts" - - "tests/runtime-tool-policy.test.ts" - - "tests/browser-canonical-preview-origin.test.ts" - - "tests/browser-adaptive-exploration.test.ts" - - "packages/runtime-core/src/browser-adaptive-exploration.ts" - - "packages/runtime-playground/src/browser-actions-runner.ts" - - "packages/runtime-playground/src/browser-adaptive-explorer.ts" - - "packages/runtime-playground/src/browser-artifacts.ts" - - "packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts" - - "packages/runtime-playground/src/browser-preview-routing.ts" - - "packages/runtime-playground/src/browser-probe-runner.ts" - - "packages/runtime-playground/src/editor-command-runners.ts" - - "packages/runtime-playground/src/preview-server.ts" - - "packages/runtime-playground/src/phpunit-command-handlers.ts" - - "packages/runtime-playground/src/playground-cli-runner.ts" - - "packages/runtime-playground/src/playground-wordpress-archive-cache.ts" - - "tests/playground-custom-archive-cache*.test.ts" - - "tests/fixtures/playground-cache-lease-child.ts" - - "tests/playground-phpunit-readonly-cache.integration.test.ts" - - "tests/fixtures/phpunit-playground-harness/**" - - "packages/cli/src/bounded-recipe-plan.ts" - - "packages/runtime-core/src/bounded-runtime-plan.ts" - - "tests/bounded-*.test.ts" - - "tests/disposable-mysql-mysqli.integration.test.ts" - - "tests/mysqli-poll.integration.test.ts" + branches: + - main + +permissions: + contents: read + +concurrency: + group: integration-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - contracts: + integration: runs-on: ubuntu-latest + timeout-minutes: 45 steps: - uses: actions/checkout@v4 with: @@ -118,42 +30,7 @@ jobs: php-version: '8.2' tools: composer:v2 - run: npm ci - - name: Cache Playwright Chromium - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('npm-shrinkwrap.json') }} - - name: Install Playwright Chromium - run: npx playwright install --with-deps chromium - - run: npm run build - - run: npm run test:browser-accessibility-oracles - - run: npm run test:browser-visual-compare-url-capture - - run: npm run test:browser-visual-compare-animated-media - - run: npm run test:browser-visual-compare-capture-reliability - - run: npm run test:browser-visual-compare-layout-drift - - run: npm run test:agent-task-contracts - - run: npm run test:browser-canonical-preview-origin - - run: npm run test:bounded-runtime-plan - - run: npm run test:bounded-recipe-plan - - run: npm run test:bounded-recipe-plan-integration - - run: npm run test:recipe-step-continuation - - run: npm run test:phpunit-runtime-rejection - - run: npm run test:php-wasm-runtime-rejection-any-command - - run: npm run test:disposable-mysql-mysqli-e2e - - run: npm run test:mysqli-poll-integration - - run: npm run test:runtime-sources-playground-integration - - run: npm run test:playground-phpunit-readonly-cache-integration - - run: npm run test:playground-custom-archive-cache - - run: npm run test:native-agent-task-playground-e2e - - run: npm run test:native-agent-task-interruption - - run: npm run test:trusted-apply-artifact-channel - - run: npm run test:runtime-command-artifact-bounds - - run: npm run test:recipe-declared-artifacts - - run: npm run test:redaction - - run: npm run test:browser-preview-routing - - run: npm run test:browser-routed-command-security - - run: npm run test:production-boundary-enforcement - - run: npm run test:runtime-tool-policy + - run: npm run test:integration workflow-lint: runs-on: ubuntu-latest @@ -161,4 +38,4 @@ jobs: - uses: actions/checkout@v4 - uses: docker://rhysd/actionlint:1.7.7 with: - args: .github/workflows/agent-task-contracts.yml .github/workflows/build-php-wasm-mysqli-poll-overlay.yml .github/workflows/run-agent-task.yml + args: .github/workflows/agent-task-contracts.yml .github/workflows/browser.yml .github/workflows/build-php-wasm-mysqli-poll-overlay.yml .github/workflows/run-agent-task.yml diff --git a/.github/workflows/browser.yml b/.github/workflows/browser.yml new file mode 100644 index 000000000..1a400441b --- /dev/null +++ b/.github/workflows/browser.yml @@ -0,0 +1,40 @@ +name: Browser + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +concurrency: + group: browser-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + browser: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + tools: composer:v2 + - run: npm ci + - name: Cache Playwright Chromium + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('npm-shrinkwrap.json') }} + - name: Install Playwright Chromium + run: npx playwright install --with-deps chromium + - run: npm run test:browser diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 59399e013..052a98836 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,9 +1,7 @@ name: Check -# Deliberately unfiltered. The smoke aggregate is the canonical test surface, so -# every pull request runs all of it. A hand-curated `paths:` list is what let -# packages/wordpress-plugin and packages/runtime-cloudflare go uncovered, and it -# rots silently when a glob stops matching. +# Deliberately unfiltered. This fast lane owns build, type, and contract coverage; +# integration and browser workflows own tests that provision external runtimes. on: pull_request: push: @@ -19,9 +17,9 @@ concurrency: jobs: check: - name: smoke aggregate + name: fast contracts runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: @@ -35,12 +33,5 @@ jobs: php-version: '8.2' tools: composer:v2 - run: npm ci - - name: Cache Playwright Chromium - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('npm-shrinkwrap.json') }} - - name: Install Playwright Chromium - run: npx playwright install --with-deps chromium - name: npm run check run: npm run check diff --git a/README.md b/README.md index b560ad3aa..1a6c0e66d 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,10 @@ npm run build npm run check ``` -`npm run check` runs the full smoke manifest aggregate. Use `npm run smoke -- --list` to see focused smoke groups and commands. +`npm run check` runs the fast build and contract lane. `npm run test:integration` +owns Playground and Docker coverage, `npm run test:browser` owns serial Chromium +coverage, and `npm run test:all` composes every root-package lane for release gates. +Use `npm run smoke -- --list` to see focused smoke groups and commands. ## Distribution Artifacts diff --git a/docs/README.md b/docs/README.md index 5a3a37f0c..5fbdc1132 100644 --- a/docs/README.md +++ b/docs/README.md @@ -91,9 +91,11 @@ unless this index says otherwise. `packages/runtime-core/src/runtime-boundary-contracts.ts` and `packages/runtime-core/src/runtime-profile-compiler.ts`. - JSON Schema factory: `packages/runtime-core/src/recipe-schema.ts`. -- Default check coverage: `npm run check` includes - `npm run test:generic-primitives` through the smoke manifest `core` group. -- Disposable MySQL integration coverage runs through - `npm run test:disposable-mysql-mysqli-e2e`. The test detects Docker with +- Default check coverage: `npm run check` includes build, type, and generic + primitive contracts through the smoke manifest `declared` group. +- Playground and disposable MySQL integration coverage runs through + `npm run test:integration`. The MySQL test detects Docker with `docker info`; Docker-capable CI/Lab runs the public recipe path and PHP-WASM `mysqli` assertion, while hosts without a Docker daemon report an explicit skip. +- Serial Chromium coverage runs through `npm run test:browser`. Release gates + compose all root-package lanes with `npm run test:all`. diff --git a/homeboy.json b/homeboy.json index 7f9a5ca30..378991711 100644 --- a/homeboy.json +++ b/homeboy.json @@ -84,7 +84,7 @@ "npm run build" ], "test": [ - "npm run check", + "npm run test:all", "npm run test:release-target", "npm run test:sharp-release-runtime", "npm run test:prepare-declaration-rebuild", diff --git a/package.json b/package.json index 0ca14ca9c..3c72d3790 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "test:native-agent-task-playground-e2e": "tsx tests/execute-native-agent-task-playground-e2e.test.ts", "test:bench-command-step-behavior": "tsx tests/bench-command-step-behavior.test.ts", "test:external-http-load-integration": "npm run build && tsx tests/external-http-load.integration.test.ts", - "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:browser-canonical-preview-origin && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", + "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", "test:temp-runtime-cleanup": "tsx tests/temp-runtime-cleanup.test.ts", "smoke:cli-version": "tsx scripts/cli-version-smoke.ts", "test:php-runner-workspace-tools": "php scripts/php-runner-workspace-tools-smoke.php", @@ -170,11 +170,13 @@ "test:mysqli-poll-integration": "tsx tests/mysqli-poll.integration.test.ts", "test:disposable-mysql-mysqli-e2e": "tsx tests/disposable-mysql-mysqli.integration.test.ts", "test:wordpress-runtime-discovery-coverage-plan": "tsx tests/wordpress-runtime-discovery-coverage-plan.test.ts", - "test:docs-boundary-language": "tsx tests/docs-boundary-language.test.ts", "test:production-boundary-enforcement": "tsx tests/production-boundary-enforcement.test.ts", "test:browser-viewport-replay": "node ./node_modules/typescript/bin/tsc -b packages/runtime-core --force && node ./node_modules/typescript/bin/tsc -b packages/runtime-playground packages/cli --force && node scripts/ensure-cli-bin-executable.mjs && tsx tests/browser-viewport-replay.test.ts", + "test:integration": "npm run build && npm run smoke -- --group=integration --concurrency=2", + "test:browser": "npm run build && npm run smoke -- --group=browser", + "test:all": "npm run smoke -- --all", "check": "npm run smoke -- --group=check", - "check:all": "npm run check && npm run cloudflare:check" + "check:all": "npm run test:all && npm run cloudflare:check" }, "workspaces": [ "packages/cli", diff --git a/scripts/agent-runtime-task-ability-smoke.ts b/scripts/agent-runtime-task-ability-smoke.ts deleted file mode 100644 index 141dd0860..000000000 --- a/scripts/agent-runtime-task-ability-smoke.ts +++ /dev/null @@ -1,147 +0,0 @@ -import assert from "node:assert/strict" -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { chdir, cwd } from "node:process" -import { buildAgentTaskRecipe } from "../packages/runtime-core/src/agent-task-recipe.js" -import { normalizeTaskInput } from "../packages/runtime-core/src/task-input.js" -import { runRecipeRunCommand } from "../packages/cli/src/commands/recipe-run.js" - -const root = mkdtempSync(join(tmpdir(), "wp-codebox-runtime-task-ability-smoke-")) -const originalCwd = cwd() - -try { - chdir(root) - const componentPath = writeRuntimeTaskComponent(root) - const available = await runRuntimeTaskRecipe(root, componentPath, "example/runtime-task") - const availableRuntime = sandboxAgentRuntime(available) - assert.equal(available.success, true, JSON.stringify(availableRuntime, null, 2)) - assert.equal(availableRuntime.success, true, JSON.stringify(availableRuntime, null, 2)) - assert.deepEqual(availableRuntime.result, { - schema: "example/runtime-task-result/v1", - success: true, - concept_packet: { title: "Runtime task ability available" }, - }) - - const runtimePackage = await runRuntimeTaskRecipe(root, componentPath, "wp-codebox/run-runtime-package") - const runtimePackageRuntime = sandboxAgentRuntime(runtimePackage) - assert.equal(runtimePackage.success, true, JSON.stringify(runtimePackageRuntime, null, 2)) - assert.equal(runtimePackageRuntime.success, true, JSON.stringify(runtimePackageRuntime, null, 2)) - assert.equal(runtimePackageRuntime.result?.schema, "example/runtime-package-result/v1") - assert.equal(runtimePackageRuntime.result?.success, true) - assert.equal(runtimePackageRuntime.result?.concept_packet?.title, "Runtime task ability available") - - const missing = await runRuntimeTaskRecipe(root, componentPath, "example/missing-runtime-task") - const missingRuntime = sandboxAgentRuntime(missing) - assert.equal(missingRuntime.success, false) - assert.equal(missingRuntime.error?.code, "runtime_task_ability_missing_preflight") - assert.equal(missingRuntime.error?.data?.preflight?.schema, "wp-codebox/runtime-task-ability-preflight/v1") - assert.equal(missingRuntime.error?.data?.preflight?.ability, "example/missing-runtime-task") - assert.equal(missingRuntime.error?.data?.preflight?.available, false) - assert.ok(missingRuntime.error?.data?.preflight?.registered_ability_ids.includes("example/runtime-task")) - - console.log("agent-runtime-task-ability-smoke: ok") -} finally { - chdir(originalCwd) - rmSync(root, { recursive: true, force: true }) -} - -function writeRuntimeTaskComponent(rootPath: string): string { - const pluginPath = join(rootPath, "runtime-task-component") - mkdirSync(pluginPath, { recursive: true }) - writeFileSync(join(pluginPath, "runtime-task-component.php"), ` 'Example Runtime Task', - 'description' => 'Synthetic runtime task ability for sandbox materialization tests.', - 'category' => 'wp-codebox', - 'execute_callback' => static function ( array $input ): array { - return array( - 'schema' => 'example/runtime-task-result/v1', - 'success' => true, - 'concept_packet' => array( 'title' => (string) ( $input['title'] ?? '' ) ), - ); - }, - 'permission_callback' => '__return_true', - 'input_schema' => array( 'type' => 'object' ), - 'output_schema' => array( 'type' => 'object' ), - ) - ); - - wp_register_ability( - 'wp-codebox/run-runtime-package', - array( - 'label' => 'Example Runtime Package Adapter', - 'description' => 'Synthetic runtime package adapter for sandbox ability tests.', - 'category' => 'wp-codebox', - 'execute_callback' => static function ( array $input ): array { - return array( - 'schema' => 'example/runtime-package-result/v1', - 'success' => true, - 'concept_packet' => array( 'title' => (string) ( $input['title'] ?? '' ) ), - ); - }, - 'permission_callback' => '__return_true', - 'input_schema' => array( 'type' => 'object' ), - 'output_schema' => array( 'type' => 'object' ), - ) - ); -} ); -`) - return pluginPath -} - -async function runRuntimeTaskRecipe(rootPath: string, componentPath: string, ability: string): Promise<{ success?: boolean; executions?: Array<{ recipeCommand?: string; stdout?: string; parsed?: unknown }> }> { - const recipe = buildAgentTaskRecipe({ - goal: `verify runtime task ability ${ability}`, - component_contracts: [ - { slug: "runtime-task-component", path: componentPath, loadAs: "mu-plugin", activate: false }, - ], - runtime_task: { - ability, - input: { title: "Runtime task ability available" }, - }, - }, normalizeTaskInput({ goal: `verify runtime task ability ${ability}` }), "latest") - const recipePath = join(rootPath, `${ability.replace(/[^a-z0-9_-]+/gi, "-")}.json`) - writeFileSync(recipePath, JSON.stringify(recipe, null, 2)) - - const output = await captureStdout(async () => await runRecipeRunCommand(["--recipe", recipePath, "--json"])) - return JSON.parse(output) as { success?: boolean; executions?: Array<{ recipeCommand?: string; stdout?: string; parsed?: unknown }> } -} - -function sandboxAgentRuntime(runOutput: { executions?: Array<{ recipeCommand?: string; stdout?: string; parsed?: unknown }> }): { success?: boolean; result?: any; error?: { code?: string; data?: { preflight?: { schema?: string; ability?: string; resolved_ability?: string; available?: boolean; registered_ability_ids?: string[] } } } } { - const execution = runOutput.executions?.find((item) => item.recipeCommand === "wp-codebox.agent-sandbox-run") - assert.ok(execution, "agent sandbox execution should be present") - const parsed = typeof execution.parsed === "object" && execution.parsed !== null - ? execution.parsed as Record - : JSON.parse(String(execution.stdout ?? "{}")) as Record - const output = typeof parsed.output === "string" ? JSON.parse(parsed.output) as Record : parsed - return output.agent_runtime as { success?: boolean; result?: any; error?: { code?: string; data?: { preflight?: { schema?: string; ability?: string; resolved_ability?: string; available?: boolean; registered_ability_ids?: string[] } } } } -} - -async function captureStdout(callback: () => Promise): Promise { - const originalWrite = process.stdout.write.bind(process.stdout) - let stdout = "" - ;(process.stdout.write as typeof process.stdout.write) = ((chunk: string | Uint8Array, encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void), callback?: (error?: Error | null) => void) => { - stdout += typeof chunk === "string" ? chunk : chunk.toString() - if (typeof encodingOrCallback === "function") { - encodingOrCallback() - } else if (callback) { - callback() - } - return true - }) as typeof process.stdout.write - try { - await callback() - return stdout - } finally { - process.stdout.write = originalWrite - } -} diff --git a/scripts/php-browser-contained-site-contract-smoke.php b/scripts/php-browser-contained-site-contract-smoke.php index 78a182d6e..5f990aee9 100644 --- a/scripts/php-browser-contained-site-contract-smoke.php +++ b/scripts/php-browser-contained-site-contract-smoke.php @@ -6,6 +6,8 @@ $GLOBALS['wp_codebox_test_transient'] = false; $GLOBALS['wp_codebox_test_transients'] = array(); +$GLOBALS['wp_codebox_test_logged_in'] = false; +$GLOBALS['wp_codebox_test_manage_options'] = false; final class WP_Error { /** @param array $data */ @@ -30,12 +32,11 @@ function sanitize_key( string $key ): string { } function is_user_logged_in(): bool { - return false; + return $GLOBALS['wp_codebox_test_logged_in']; } function current_user_can( string $capability ): bool { - unset( $capability ); - return false; + return 'manage_options' === $capability && $GLOBALS['wp_codebox_test_manage_options']; } final class WP_Codebox_Test_Request { @@ -112,6 +113,12 @@ function expect( bool $condition, string $message ): void { expect( WP_Codebox_Abilities::can_hydrate_browser_blueprint_ref( new WP_Codebox_Test_Request( array( 'ref' => 'prepared:studio-native-preview:' . $source_digest ) ) ), 'Expected public prepared blueprint refs to be hydratable.' ); expect( WP_Codebox_Abilities::can_hydrate_browser_blueprint_ref( new WP_Codebox_Test_Request( array( 'cache_key' => 'studio-native-preview', 'input_hash' => $source_digest ) ) ), 'Expected public prepared blueprint cache key and input hash to be hydratable.' ); expect( ! WP_Codebox_Abilities::can_hydrate_browser_blueprint_ref( new WP_Codebox_Test_Request( array( 'ref' => 'prepared:studio-native-preview:not-a-hash' ) ) ), 'Expected malformed public prepared blueprint refs to remain forbidden.' ); +$GLOBALS['wp_codebox_test_logged_in'] = true; +expect( WP_Codebox_Abilities::can_hydrate_browser_blueprint_ref(), 'Expected authenticated users to hydrate private blueprint refs.' ); +$GLOBALS['wp_codebox_test_logged_in'] = false; +$GLOBALS['wp_codebox_test_manage_options'] = true; +expect( WP_Codebox_Abilities::can_hydrate_browser_blueprint_ref(), 'Expected administrators to hydrate private blueprint refs.' ); +$GLOBALS['wp_codebox_test_manage_options'] = false; $miss = WP_Codebox_Abilities::get_browser_contained_site_status( array( diff --git a/scripts/run-smoke.ts b/scripts/run-smoke.ts index 90179ac69..0de473450 100644 --- a/scripts/run-smoke.ts +++ b/scripts/run-smoke.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process" import { fileURLToPath } from "node:url" import { smokeGroups, smokeManifest, type SmokeCommand } from "./smoke-manifest.ts" -import { discoveredParallelCommands, discoveredSerialCommands } from "./smoke-discovery.ts" +import { discoveredParallelCommands, discoveredSerialCommands, type SmokeLane } from "./smoke-discovery.ts" const repositoryRoot = fileURLToPath(new URL("..", import.meta.url)) const DEFAULT_CONCURRENCY = 8 @@ -15,13 +15,14 @@ function usage(): string { return [ "Usage: npm run smoke -- [--group= | --command= | --all | --list]", "", - "Groups:", + "Declared groups:", ...Object.entries(smokeGroups).map(([name, group]) => ` ${name.padEnd(10)} ${group.description}`), "", - "Aggregate groups:", - ...Object.entries(smokeManifest.aggregateGroups).map( - ([name, groups]) => ` ${name.padEnd(10)} ${groups.join(", ")}`, - ), + "Aggregate lanes:", + " check build, type, and fast contracts", + " integration Playground, Docker, and runtime integration contracts", + " browser serial Chromium contracts", + " all every root-package lane", ].join("\n") } @@ -176,6 +177,33 @@ async function runInParallel(commands: SmokeCommand[], concurrency: number): Pro return failures } +async function runLane(name: string, lane: SmokeLane, declared: SmokeCommand[], concurrency: number): Promise { + const parallel = discoveredParallelCommands(repositoryRoot, lane) + const serial = discoveredSerialCommands(repositoryRoot, lane) + + console.log(`[smoke] ${name}: ${declared.length} declared, ${parallel.length} parallel (concurrency ${concurrency}), ${serial.length} serial`) + for (const command of declared) await runCommand(command) + + console.log(`\n[smoke] discovered phase: ${parallel.length} files at concurrency ${concurrency}`) + const failures = await runInParallel(parallel, concurrency) + + console.log(`\n[smoke] serial phase: ${serial.length} files`) + for (const command of serial) { + const outcome = await runCapturedCommand(command) + if (outcome.error) { + failures.push(outcome.error) + console.log(`\n[smoke] FAIL ${command.name}`) + console.log(outcome.output.trimEnd()) + } + } + + if (failures.length > 0) { + throw new Error(`${failures.length} smoke command(s) failed:\n` + failures.map((failure) => ` - ${failure.message}`).join("\n")) + } + + console.log(`\n[smoke] ${name} passed: ${declared.length + parallel.length + serial.length} command(s)`) +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)) @@ -195,45 +223,26 @@ async function main(): Promise { return } - const name = options.all ? "check" : options.group ?? "check" + const name = options.all ? "all" : options.group ?? "check" + const aggregateLanes = new Set(["check", "integration", "browser", "all"]) - if (name !== "check") { + if (!aggregateLanes.has(name)) { const group = resolveGroup(name) console.log(`[smoke] Running ${group.commands.length} command(s) from ${group.name}`) for (const command of group.commands) await runCommand(command) return } - // The aggregate: declared commands first (they build artifacts the discovered - // files rely on), then discovered files in parallel, then the serial tail. - const declared = resolveGroup("check") - const parallel = discoveredParallelCommands(repositoryRoot) - const serial = discoveredSerialCommands(repositoryRoot) - - console.log( - `[smoke] check: ${declared.commands.length} declared, ${parallel.length} discovered (concurrency ${options.concurrency}), ${serial.length} serial`, - ) - - for (const command of declared.commands) await runCommand(command) - - console.log(`\n[smoke] discovered phase: ${parallel.length} files at concurrency ${options.concurrency}`) - const failures = await runInParallel(parallel, options.concurrency) - - console.log(`\n[smoke] serial phase: ${serial.length} files`) - for (const command of serial) { - const outcome = await runCapturedCommand(command) - if (outcome.error) { - failures.push(outcome.error) - console.log(`\n[smoke] FAIL ${command.name}`) - console.log(outcome.output.trimEnd()) - } - } - - if (failures.length > 0) { - throw new Error(`${failures.length} smoke command(s) failed:\n` + failures.map((f) => ` - ${f.message}`).join("\n")) + if (name === "all") { + await runLane("check", "fast", resolveGroup("check").commands, options.concurrency) + await runLane("integration", "integration", [], Math.min(options.concurrency, 2)) + await runLane("browser", "browser", [], 1) + console.log("\n[smoke] all root-package lanes passed") + return } - console.log(`\n[smoke] check passed: ${declared.commands.length + parallel.length + serial.length} command(s)`) + const lane: SmokeLane = name === "check" ? "fast" : name as SmokeLane + await runLane(name, lane, name === "check" ? resolveGroup("check").commands : [], options.concurrency) } main().catch((error: unknown) => { diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts index 1f79b0efd..cbf990862 100644 --- a/scripts/smoke-discovery.ts +++ b/scripts/smoke-discovery.ts @@ -6,8 +6,9 @@ import type { SmokeCommand } from "./smoke-manifest.js" /* * Test files are discovered by convention rather than registered by hand. * Adding tests/.test.ts or scripts/-smoke.ts is enough to make it - * run. Anything that must not run has to be listed below with a reason, so the - * exclusions stay short, visible, and reviewable. + * run. Files are classified into fast, integration, and browser lanes below. + * Anything that must not run has to be listed with a reason, so exclusions stay + * short, visible, and reviewable. */ export const DISCOVERY_PATTERNS = { @@ -16,13 +17,12 @@ export const DISCOVERY_PATTERNS = { } as const type Exclusion = { file: string; reason: string } +export type SmokeLane = "fast" | "integration" | "browser" | "all" export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ { file: "scripts/run-smoke.ts", reason: "the runner itself; discovering it would recurse" }, // Require an environment the aggregate does not provision. - { file: "tests/mysqli-poll.integration.test.ts", reason: "requires Docker; runs in the agent-task-contracts workflow" }, - { file: "tests/runtime-sources-playground-integration.test.ts", reason: "exceeds the per-file budget; runs in the agent-task-contracts workflow" }, { file: "tests/release-package-coverage.test.ts", reason: "needs the 427 MB plugin zip from package:wordpress-plugin; runs in the Homeboy gate" }, { file: "tests/derive-cloudflare-core-contract-atomic.test.mjs", reason: "covers the Cloudflare-owned derivation writer; runs with package boundaries in the Cloudflare Check workflow" }, { file: "tests/runtime-package-boundaries.test.mjs", reason: "requires the independent Cloudflare package install; runs in the Cloudflare Check workflow" }, @@ -31,38 +31,43 @@ export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ reason: "destructive to shared state: deletes packages/runtime-core/dist and runs npm install at the repository root, which breaks every concurrent import of @automattic/wp-codebox-core; runs in the Homeboy gate", }, - - // Known-failing and unmaintained. Tracked for triage; see the discovery audit - // in issue #2402. These were added in June 2026, never wired to a gate, and - // have not been touched since. - { file: "tests/artifact-reference-dtos.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "tests/browser-blueprint-ref-permission.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "tests/command-diagnostics.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "tests/docs-boundary-language.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "tests/temp-runtime-cleanup.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "tests/wordpress-runtime-discovery-coverage-plan.test.ts", reason: "failing and unmaintained; pending triage" }, - { file: "scripts/agent-runtime-task-ability-smoke.ts", reason: "failing and unmaintained; pending triage" }, ] -/* - * These contend on the shared Playground WordPress archive cache, or boot a - * full browser and WordPress runtime and time out when starved of CPU. They are - * correct in isolation, so they run in a serial phase after the parallel one - * rather than being excluded. - */ -export const DISCOVERY_SERIAL: readonly string[] = [ +const INTEGRATION_FILES: readonly string[] = [ "scripts/doctor-command-smoke.ts", + "tests/execute-native-agent-task-playground-e2e.test.ts", + "tests/playground-custom-archive-cache-process.test.ts", + "tests/playground-readonly-mounts.test.ts", + "tests/runtime-sources-playground-integration.test.ts", +] + +const BROWSER_FILES: readonly string[] = [ + "tests/browser-accessibility-oracles.test.ts", + "tests/browser-action-corpus.test.ts", + "tests/browser-adaptive-exploration.test.ts", + "tests/browser-canonical-preview-origin.test.ts", + "tests/browser-multi-actor-scenario.test.ts", + "tests/browser-recipe-file-payloads.integration.test.ts", + "tests/browser-routed-command-security.test.ts", + "tests/browser-visual-compare-animated-media.test.ts", + "tests/browser-visual-compare-capture-reliability.test.ts", + "tests/browser-visual-compare-dom-snapshots.test.ts", + "tests/browser-visual-compare-url-capture.test.ts", + "tests/browser-viewport-replay.test.ts", + "tests/editor-actions-save.integration.test.ts", + "tests/playground-mapped-domain-multisite.integration.test.ts", + "tests/playground-staged-upload-preview.integration.test.ts", + "tests/runtime-backed-multisite-workload.integration.test.ts", +] + +/* Files that share process handlers or Playground caches remain serial within their lane. */ +const DISCOVERY_SERIAL: readonly string[] = [ "tests/bounded-recipe-plan.integration.test.ts", "tests/phpunit-runtime-rejection.test.ts", + "tests/playground-readonly-mounts-integration.test.ts", "tests/playground-readonly-mounts.test.ts", "tests/playground-phpunit-bootstrap-failure.integration.test.ts", - "tests/browser-actions-navigation-capture.browser.test.ts", - "tests/editor-actions-save.integration.test.ts", - // Boots a runtime and asserts step counts, so contention flips it rather - // than slowing it. Passed 3 of 3 standalone after failing once under load. "tests/recipe-step-continuation.integration.test.ts", - // Asserts cancellation timing, so it fails when starved rather than slowed. - "tests/browser-adaptive-exploration.test.ts", ] /* @@ -74,7 +79,6 @@ export const CHAIN_OWNED_FILES: readonly string[] = [ "tests/artifact-path-primitives.test.ts", "tests/bench-command-step-behavior.test.ts", "tests/browser-callback-materialization-contracts.test.ts", - "tests/browser-canonical-preview-origin.test.ts", "tests/external-mysql-runtime-service.test.ts", "tests/generic-ability-runtime-run.test.ts", "tests/native-mariadb-runtime-service.test.ts", @@ -84,6 +88,8 @@ export const CHAIN_OWNED_FILES: readonly string[] = [ const excluded = new Set([...DISCOVERY_EXCLUSIONS.map((entry) => entry.file), ...CHAIN_OWNED_FILES]) const serial = new Set(DISCOVERY_SERIAL) +const integrations = new Set(INTEGRATION_FILES) +const browsers = new Set(BROWSER_FILES) function listFiles(root: string, directory: string, pattern: RegExp): string[] { return readdirSync(join(root, directory)) @@ -100,6 +106,17 @@ export function discoverSmokeFiles(root = process.cwd()): string[] { return files.filter((file) => !excluded.has(file)).sort() } +function laneFor(file: string): Exclude { + if (file.endsWith(".browser.test.ts") || browsers.has(file)) return "browser" + if (/[.-]integration\.test\./.test(file) || integrations.has(file)) return "integration" + return "fast" +} + +function filesForLane(root: string, lane: SmokeLane): string[] { + const files = discoverSmokeFiles(root) + return lane === "all" ? files : files.filter((file) => laneFor(file) === lane) +} + function toCommand(file: string, disableTsxCache = false): SmokeCommand { return { name: file, @@ -108,17 +125,24 @@ function toCommand(file: string, disableTsxCache = false): SmokeCommand { } } -export function discoveredCommands(root = process.cwd()): SmokeCommand[] { - return discoverSmokeFiles(root).map(toCommand) +export function discoveredCommands(root = process.cwd(), lane: SmokeLane = "all"): SmokeCommand[] { + return filesForLane(root, lane).map((file) => toCommand(file)) } /** Files safe to run concurrently. */ -export function discoveredParallelCommands(root = process.cwd()): SmokeCommand[] { - return discoverSmokeFiles(root).filter((file) => !serial.has(file)).map((file) => toCommand(file, true)) +export function discoveredParallelCommands(root = process.cwd(), lane: SmokeLane = "all"): SmokeCommand[] { + return filesForLane(root, lane) + .filter((file) => laneFor(file) !== "browser" && !serial.has(file)) + .map((file) => toCommand(file, true)) } /** Files that must run one at a time, after the parallel phase. */ -export function discoveredSerialCommands(root = process.cwd()): SmokeCommand[] { - const found = new Set(discoverSmokeFiles(root)) - return DISCOVERY_SERIAL.filter((file) => found.has(file)).map(toCommand) +export function discoveredSerialCommands(root = process.cwd(), lane: SmokeLane = "all"): SmokeCommand[] { + const found = new Set(filesForLane(root, lane)) + const ordered = lane === "browser" + ? [...found] + : lane === "all" + ? [...found].filter((file) => laneFor(file) === "browser" || serial.has(file)) + : DISCOVERY_SERIAL.filter((file) => found.has(file)) + return ordered.map((file) => toCommand(file)) } diff --git a/scripts/smoke-manifest.ts b/scripts/smoke-manifest.ts index ac225d5d3..e17c69cd7 100644 --- a/scripts/smoke-manifest.ts +++ b/scripts/smoke-manifest.ts @@ -20,7 +20,7 @@ function npmScript(name: string): SmokeCommand { /* * Test files are not registered here. `scripts/smoke-discovery.ts` finds * tests/*.test.{ts,mjs} and scripts/*-smoke.{ts,php} by convention, and - * `npm run check` runs them after the commands below. + * each smoke lane runs its matching discovered files after any declared work. * * This group is only for work that is not a single test file: compilation and * typechecking. Everything else belongs in a discovered file. diff --git a/tests/artifact-reference-dtos.test.ts b/tests/artifact-reference-dtos.test.ts index 2c51a9713..42261d2ee 100644 --- a/tests/artifact-reference-dtos.test.ts +++ b/tests/artifact-reference-dtos.test.ts @@ -18,9 +18,9 @@ import { } from "../packages/runtime-core/src/index.js" const ref = normalizePublicArtifactRefDTO({ - artifact_id: "bundle-1", - directory: "artifacts/run-1", - contentDigest: "abc123", + id: "bundle-1", + path: "artifacts/run-1", + sha256: "abc123", }) assert.equal(ref?.schema, PUBLIC_ARTIFACT_REF_DTO_SCHEMA) @@ -30,18 +30,18 @@ assert.equal(ref?.path, "artifacts/run-1") assert.equal(ref?.sha256, "abc123") assert.deepEqual(ref?.digest, { algorithm: "sha256", value: "abc123" }) -const browserAliasRef = normalizePublicArtifactRefDTO({ - artifact_type: "browser-screenshot", - artifactId: "screenshot-1", - artifactsPath: "files/browser/screenshot.png", - content_digest: "def456", +const browserRef = normalizePublicArtifactRefDTO({ + kind: "browser-screenshot", + id: "screenshot-1", + path: "files/browser/screenshot.png", + sha256: "def456", }) -assert.equal(browserAliasRef?.kind, "browser-screenshot") -assert.equal(browserAliasRef?.id, "screenshot-1") -assert.equal(browserAliasRef?.path, "files/browser/screenshot.png") -assert.equal(browserAliasRef?.sha256, "def456") -assert.deepEqual(browserAliasRef?.digest, { algorithm: "sha256", value: "def456" }) +assert.equal(browserRef?.kind, "browser-screenshot") +assert.equal(browserRef?.id, "screenshot-1") +assert.equal(browserRef?.path, "files/browser/screenshot.png") +assert.equal(browserRef?.sha256, "def456") +assert.deepEqual(browserRef?.digest, { algorithm: "sha256", value: "def456" }) const runResult = { artifacts: { diff --git a/tests/browser-blueprint-ref-permission.test.ts b/tests/browser-blueprint-ref-permission.test.ts deleted file mode 100644 index 941657ac9..000000000 --- a/tests/browser-blueprint-ref-permission.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import assert from "node:assert/strict" -import { readFileSync } from "node:fs" -import { join } from "node:path" - -import { repoRoot } from "../scripts/test-kit.js" - -const source = readFileSync(join(repoRoot, "packages/wordpress-plugin/src/class-wp-codebox-abilities.php"), "utf8") - -assert.match(source, /'permission_callback'\s*=>\s*array\(\s*self::class,\s*'can_hydrate_browser_blueprint_ref'\s*\)/) -assert.match(source, /public static function can_hydrate_browser_blueprint_ref\(\): bool/) -assert.match(source, /is_user_logged_in\(\)\s*\|\|\s*current_user_can\(\s*'manage_options'\s*\)/) - -console.log("browser blueprint ref permission ok") diff --git a/tests/command-diagnostics.test.ts b/tests/command-diagnostics.test.ts index 6cdd09105..04359bc64 100644 --- a/tests/command-diagnostics.test.ts +++ b/tests/command-diagnostics.test.ts @@ -31,13 +31,23 @@ const stepProperties = ((schema.$defs as Record).step as { prop assert.ok(stepProperties.diagnostics, "recipe step schema exposes diagnostics capture") const noCapture = await recipeExecutionSpec({ command: "wordpress.run-php", args: ["code=echo 'ok';"] }, process.cwd()) -assert.deepEqual(noCapture, { command: "wordpress.run-php", args: ["code=echo 'ok';"], diagnostics: undefined }) +assert.deepEqual(noCapture, { + command: "wordpress.run-php", + args: ["code=echo 'ok';"], + diagnostics: undefined, + originalCommand: "wordpress.run-php", + originalArgs: ["code=echo 'ok';"], + resolvedArgs: ["code=echo 'ok';"], +}) const withCapture = await recipeExecutionSpec({ command: "wordpress.run-php", args: ["code=echo 'ok';"], diagnostics: { capture: ["wpdb-queries"], maxItems: 1, maxBytes: 128 } }, process.cwd()) assert.deepEqual(withCapture, { command: "wordpress.run-php", args: ["code=echo 'ok';", "capture-diagnostics=wpdb-queries", "diagnostics-max-items=1", "diagnostics-max-bytes=128"], diagnostics: { capture: ["wpdb-queries"], maxItems: 1, maxBytes: 128 }, + originalCommand: "wordpress.run-php", + originalArgs: ["code=echo 'ok';"], + resolvedArgs: ["code=echo 'ok';", "capture-diagnostics=wpdb-queries", "diagnostics-max-items=1", "diagnostics-max-bytes=128"], }) const workloadJson = JSON.stringify({ @@ -53,13 +63,20 @@ assert.deepEqual(workloadSpec, { "expected-result-schema=\"wp-codebox/wordpress-workload-run-result/v1\"", ], diagnostics: undefined, + originalCommand: "wordpress.run-workload", + originalArgs: [`workload-json=${workloadJson}`], + resolvedArgs: [ + "name=wp-codebox/run-wordpress-workload", + `input=${workloadJson}`, + "expected-result-schema=\"wp-codebox/wordpress-workload-run-result/v1\"", + ], }) const workloadDirectory = await mkdtemp(join(tmpdir(), "wp-codebox-workload-json-")) try { await writeFile(join(workloadDirectory, "workload.json"), workloadJson) const workloadFileSpec = await recipeExecutionSpec({ command: "wordpress.run-workload", args: ["workload-json=workload.json"] }, workloadDirectory) - assert.deepEqual(workloadFileSpec, workloadSpec) + assert.deepEqual(workloadFileSpec, { ...workloadSpec, originalArgs: ["workload-json=workload.json"] }) } finally { await rm(workloadDirectory, { recursive: true, force: true }) } diff --git a/tests/docs-boundary-language.test.ts b/tests/docs-boundary-language.test.ts deleted file mode 100644 index 7a2f156b3..000000000 --- a/tests/docs-boundary-language.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import assert from "node:assert/strict" -import { readdir, readFile } from "node:fs/promises" -import { relative } from "node:path" -import { runtimeContractManifest } from "../packages/runtime-core/src/public.js" - -const root = new URL("..", import.meta.url) -const publicConsumerRoots = ["README.md", "docs", "examples", "packages/wordpress-plugin/README.md", "packages/cli/README.md"] -const internalReferenceDocs = new Set([ - "docs/browser-runtime-dependency-audit.md", - "docs/portable-wp-codebox.md", - "docs/transfer-namespace-plan.md", - "docs/transfer-readiness-checklist.md", -]) - -const publicDocsText = await readPublicText(publicConsumerRoots) -const manifest = runtimeContractManifest() -const publicAbilityIds = flattenStringValues(manifest.abilities) - -for (const abilityId of publicAbilityIds) { - assert.match( - publicDocsText, - new RegExp(escapeRegExp(abilityId)), - `public docs should document Codebox public ability ${abilityId}`, - ) -} - -for (const rawPublicPath of [ - /agents\/[a-z0-9._/-]+/i, - /wp-codebox\.agent-sandbox-run/, - /from ["']@automattic\/wp-codebox-playground/, - /@wp-playground\//, -]) { - assert.doesNotMatch(publicDocsText, rawPublicPath, `public docs must not teach ${rawPublicPath}`) -} - -for (const forbiddenConsumerGuidance of [ - /studio\s+wp\s+datamachine/i, - /wp\s+datamachine/i, - /agents-api\/[a-z0-9._/-]+/i, - /datamachine\/[a-z0-9._/-]+/i, - /data-machine-code\/[a-z0-9._/-]+/i, - /call\s+(?:the\s+)?(?:Data Machine|Agents API|Data Machine Code)\b/i, - /use\s+(?:the\s+)?(?:Data Machine|Agents API|Data Machine Code)\s+(?:ability|api|endpoint)\b/i, -]) { - assert.doesNotMatch(publicDocsText, forbiddenConsumerGuidance, `public docs must not teach consumers to call ${forbiddenConsumerGuidance}`) -} - -const publicExamplesText = await readPublicText(["examples"]) -assert.doesNotMatch(publicExamplesText, /wp-codebox\.agent-sandbox-run|agents\/[a-z0-9._/-]+/i) - -const publicApiContract = await readFile(new URL("docs/public-api-contract.md", root), "utf8") -assert.match(publicApiContract, /External integrations should compose the Codebox core facades,\s+WordPress abilities, CLI, or browser SDK/) -assert.match(publicApiContract, /Product consumers should use the Codebox-owned public surfaces/) -assert.match(publicApiContract, /manifest intentionally excludes backend handler bindings/) -assert.match(publicApiContract, /Internal\/default substrate adapters are implementation\s+details/) -assert.match(publicApiContract, /They are not consumer\s+API names/) -assert.match(publicApiContract, /advanced adapter surface/) -assert.match(publicApiContract, /`wp-codebox\/run-plan-progress\/v1`/) -assert.match(publicApiContract, /Hosts may stream or persist those snapshots in\s+their own job system/) -assert.match(publicApiContract, /host UIs own the button, policy, and durable cancellation request transport/) - -const cookbookReadme = await readFile(new URL("examples/recipes/cookbook/README.md", root), "utf8") -assert.doesNotMatch(cookbookReadme, /legacy-compatibility-recipes/) -assert.doesNotMatch(cookbookReadme, /codex-agent-smoke\.json|claude-code-agent-smoke\.json|headless-browser-agent-task\.json/) - -console.log("docs boundary language ok") - -async function readPublicText(paths: string[]): Promise { - const files = (await Promise.all(paths.map(collectTextFiles))).flat() - const chunks: string[] = [] - - for (const file of files) { - if (internalReferenceDocs.has(file)) continue - chunks.push(await readFile(new URL(file, root), "utf8")) - } - - return chunks.join("\n") -} - -async function collectTextFiles(path: string): Promise { - if (/\.(md|json|ts|js|ya?ml)$/.test(path)) { - return [path] - } - - if (/\.[^.]+$/.test(path)) { - return [] - } - - const entries = await readdir(new URL(`${path}/`, root), { withFileTypes: true }) - const files = await Promise.all(entries.map((entry) => { - const child = `${path}/${entry.name}` - return entry.isDirectory() ? collectTextFiles(child) : collectTextFiles(child) - })) - - return files.flat().map((file) => relative(root.pathname, new URL(file, root).pathname)) -} - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") -} - -function flattenStringValues(value: unknown): string[] { - if (typeof value === "string") return [value] - if (!value || typeof value !== "object") return [] - return Object.values(value).flatMap(flattenStringValues) -} diff --git a/tests/production-boundary-enforcement.test.ts b/tests/production-boundary-enforcement.test.ts index 05249122c..5e7fc1b7b 100644 --- a/tests/production-boundary-enforcement.test.ts +++ b/tests/production-boundary-enforcement.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import { readdir, readFile } from "node:fs/promises" import { join, relative } from "node:path" +import { runtimeContractManifest } from "../packages/runtime-core/src/public.js" const root = new URL("..", import.meta.url) const packagesDir = new URL("../packages/", import.meta.url) @@ -9,6 +10,7 @@ const runtimeCoreSrcDir = new URL("../packages/runtime-core/src/", import.meta.u const publicDocPaths = [ "README.md", "docs/README.md", + "docs/public-api-contract.md", "docs/portable-wp-codebox.md", "docs/runner-workspace-backend-contract.md", "packages/cli/README.md", @@ -39,6 +41,11 @@ const forbiddenPublicExportTargets = [ const forbiddenPublicImportSpecifiers = [ /@wp-playground\//, ] +const forbiddenRawPublicPaths = [ + /agents\/[a-z0-9._/-]+/i, + /wp-codebox\.agent-sandbox-run/, + /from ["']@automattic\/wp-codebox-playground/, +] const runtimeBackendPackages = new Set([ "@automattic/wp-codebox-playground", "@automattic/wp-codebox-runtime-cloudflare", @@ -64,9 +71,11 @@ const forbiddenPublicContractVocabulary = [ ] const violations: string[] = [] +const publicDocs = new Map() for (const rel of publicDocPaths) { const source = await readFile(new URL(`../${rel}`, import.meta.url), "utf8") + publicDocs.set(rel, source) for (const term of forbiddenPublicSurfaceTerms) { if (term.test(source)) { violations.push(`${rel} exposes ${term} in public documentation`) @@ -77,6 +86,42 @@ for (const rel of publicDocPaths) { violations.push(`${rel} tells public consumers to call internal substrate ${guidance}`) } } + for (const rawPath of forbiddenRawPublicPaths) { + if (rawPath.test(source)) { + violations.push(`${rel} teaches backend path ${rawPath}`) + } + } +} + +const publicDocsText = [...publicDocs.values()].join("\n") +for (const abilityId of flattenStringValues(runtimeContractManifest().abilities)) { + if (!publicDocsText.includes(abilityId)) { + violations.push(`public documentation omits Codebox ability ${abilityId}`) + } +} + +const publicApiContract = publicDocs.get("docs/public-api-contract.md") ?? "" +for (const expectedContract of [ + /External integrations should compose the Codebox core facades,\s+WordPress abilities, CLI, or browser SDK/, + /Product consumers should use the Codebox-owned public surfaces/, + /manifest intentionally excludes backend handler bindings/, + /Internal\/default substrate adapters are implementation\s+details/, + /They are not consumer\s+API names/, + /advanced adapter surface/, + /`wp-codebox\/run-plan-progress\/v1`/, + /Hosts may stream or persist those snapshots in\s+their own job system/, + /host UIs own the button, policy, and durable cancellation request transport/, +]) { + if (!expectedContract.test(publicApiContract)) { + violations.push(`docs/public-api-contract.md omits ${expectedContract}`) + } +} + +const cookbookReadme = publicDocs.get("examples/recipes/cookbook/README.md") ?? "" +for (const obsoleteRecipe of [/legacy-compatibility-recipes/, /codex-agent-smoke\.json|claude-code-agent-smoke\.json|headless-browser-agent-task\.json/]) { + if (obsoleteRecipe.test(cookbookReadme)) { + violations.push(`examples/recipes/cookbook/README.md references obsolete recipe ${obsoleteRecipe}`) + } } for (const manifest of await packageManifests(packagesDir)) { @@ -205,3 +250,9 @@ function exportedContractNames(source: string): string[] { return value ? [value] : [] }) } + +function flattenStringValues(value: unknown): string[] { + if (typeof value === "string") return [value] + if (!value || typeof value !== "object") return [] + return Object.values(value).flatMap(flattenStringValues) +} diff --git a/tests/release-package-coverage.test.ts b/tests/release-package-coverage.test.ts index f124069ae..849a981e1 100644 --- a/tests/release-package-coverage.test.ts +++ b/tests/release-package-coverage.test.ts @@ -25,7 +25,7 @@ assert.deepEqual(homeboy.release?.package_coverage, [{ assert.deepEqual(homeboy.extensions?.nodejs, { settings: { release_package_script: "release:package" } }, "the Node release provider must receive the project artifact manifest setting") assert.deepEqual(homeboy.scripts?.build, ["npm run package:wordpress-plugin"], "component builds retain singular WordPress deploy-artifact ownership") assert.deepEqual(homeboy.scripts?.test, [ - "npm run check", + "npm run test:all", "npm run test:release-target", "npm run test:sharp-release-runtime", "npm run test:prepare-declaration-rebuild", diff --git a/tests/runtime-sources-playground-integration.test.ts b/tests/runtime-sources-playground-integration.test.ts index fa33833a7..e251c3f1e 100644 --- a/tests/runtime-sources-playground-integration.test.ts +++ b/tests/runtime-sources-playground-integration.test.ts @@ -159,12 +159,20 @@ echo wp_json_encode( array( 'imported_slug' => $imports[0]['agent_slug'] ?? '', } } - const runtimePackageOutput = await runRuntimePackage() - const runtimeExecution = runtimePackageOutput.executions?.find((execution: { stdout?: string }) => execution.stdout?.includes("agent_runtime")) - const runtime = JSON.parse(JSON.parse(runtimeExecution?.stdout ?? "{}").output ?? "{}") - assert.equal(runtime.agent_runtime?.success, true, JSON.stringify(runtimePackageOutput.executions?.map((execution: { command?: string, stdout?: string }) => ({ command: execution.command, stdout: execution.stdout })))) - assert.equal(runtime.agent_runtime.result.package.slug, "flat-runtime-agent", "the generated runtime must execute the imported agent identity") - const requests = JSON.parse(runtimePackageOutput.executions?.filter((execution: { command?: string }) => execution.command === "wordpress.run-php").at(-1)?.stdout ?? "[]") + const runtimePackageOutput = await runRuntimePackage() + const runtime = agentRuntimeOutput(runtimePackageOutput) + assert.equal(runtime.agent_runtime?.success, true, JSON.stringify(runtimePackageOutput.executions?.map((execution: { command?: string, stdout?: string }) => ({ command: execution.command, stdout: execution.stdout })))) + assert.equal(runtime.agent_runtime.result.package.slug, "flat-runtime-agent", "the generated runtime must execute the imported agent identity") + const missingAbilityOutput = await runRuntimePackage((task) => { task.ability = "example/missing-runtime-task" }) + const missingAbilityRuntime = agentRuntimeOutput(missingAbilityOutput) + assert.equal(missingAbilityRuntime.agent_runtime?.success, false) + assert.equal(missingAbilityRuntime.agent_runtime?.error?.code, "runtime_task_ability_missing_preflight") + assert.equal(missingAbilityRuntime.agent_runtime?.error?.data?.preflight?.schema, "wp-codebox/runtime-task-ability-preflight/v1") + assert.equal(missingAbilityRuntime.agent_runtime?.error?.data?.preflight?.ability, "example/missing-runtime-task") + assert.equal(missingAbilityRuntime.agent_runtime?.error?.data?.preflight?.available, false) + assert.ok(missingAbilityRuntime.agent_runtime?.error?.data?.preflight?.registered_ability_ids.includes("wp-codebox/run-runtime-package")) + + const requests = JSON.parse(runtimePackageOutput.executions?.filter((execution: { command?: string }) => execution.command === "wordpress.run-php").at(-1)?.stdout ?? "[]") const providerTurns = requests.filter((request: { url: string }) => request.url.endsWith("/responses")) assert.equal(providerTurns.length, 3, "the intercepted provider must receive read, edit, and terminal turns") const providerTurn = providerTurns[0] @@ -219,3 +227,8 @@ echo wp_json_encode( array( 'imported_slug' => $imports[0]['agent_slug'] ?? '', } finally { await rm(root, { recursive: true, force: true }) } + +function agentRuntimeOutput(output: { executions?: Array<{ stdout?: string }> }): any { + const execution = output.executions?.find((candidate) => candidate.stdout?.includes("agent_runtime")) + return JSON.parse(JSON.parse(execution?.stdout ?? "{}").output ?? "{}") +} diff --git a/tests/smoke-discovery.test.ts b/tests/smoke-discovery.test.ts index 7a918375b..6bc5001ae 100644 --- a/tests/smoke-discovery.test.ts +++ b/tests/smoke-discovery.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" -import { discoveredParallelCommands, discoveredSerialCommands } from "../scripts/smoke-discovery.js" +import { discoveredCommands, discoveredParallelCommands, discoveredSerialCommands } from "../scripts/smoke-discovery.js" const root = await mkdtemp(join(tmpdir(), "wp-codebox-smoke-discovery-")) @@ -12,15 +12,28 @@ try { await mkdir(join(root, "scripts")) await writeFile(join(root, "tests", "parallel.test.ts"), "") await writeFile(join(root, "tests", "recipe-step-continuation.integration.test.ts"), "") + await writeFile(join(root, "tests", "playground-readonly-mounts-integration.test.ts"), "") + await writeFile(join(root, "tests", "browser-fixture.browser.test.ts"), "") await writeFile(join(root, "scripts", "native-smoke.php"), "") - assert.deepEqual(discoveredParallelCommands(root), [ + assert.deepEqual(discoveredParallelCommands(root, "fast"), [ { name: "scripts/native-smoke.php", command: "php", args: ["scripts/native-smoke.php"] }, { name: "tests/parallel.test.ts", command: "tsx", args: ["--no-cache", "tests/parallel.test.ts"] }, ]) - assert.deepEqual(discoveredSerialCommands(root), [ + assert.deepEqual(discoveredParallelCommands(root, "integration"), []) + assert.deepEqual(discoveredSerialCommands(root, "integration"), [ + { name: "tests/playground-readonly-mounts-integration.test.ts", command: "tsx", args: ["tests/playground-readonly-mounts-integration.test.ts"] }, { name: "tests/recipe-step-continuation.integration.test.ts", command: "tsx", args: ["tests/recipe-step-continuation.integration.test.ts"] }, ]) + assert.deepEqual(discoveredParallelCommands(root, "browser"), []) + assert.deepEqual(discoveredSerialCommands(root, "browser"), [ + { name: "tests/browser-fixture.browser.test.ts", command: "tsx", args: ["tests/browser-fixture.browser.test.ts"] }, + ]) + assert.deepEqual( + [...discoveredParallelCommands(root, "all"), ...discoveredSerialCommands(root, "all")].map((command) => command.name).sort(), + discoveredCommands(root).map((command) => command.name).sort(), + "the exhaustive lane composes every discovered file exactly once", + ) } finally { await rm(root, { recursive: true, force: true }) } diff --git a/tests/temp-runtime-cleanup.test.ts b/tests/temp-runtime-cleanup.test.ts index 6d0bf1c43..4ea3e77c7 100644 --- a/tests/temp-runtime-cleanup.test.ts +++ b/tests/temp-runtime-cleanup.test.ts @@ -1,12 +1,12 @@ import assert from "node:assert/strict" -import { access, lutimes, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, utimes, writeFile } from "node:fs/promises" +import { access, lutimes, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, symlink, utimes, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, join } from "node:path" import { inventoryTempRuntimeDirectories, type ProcessEvidence } from "../packages/cli/src/commands/temp-runtime-cleanup.js" import { cleanupRecipePreparedSources, prepareRecipeWorkspaces } from "../packages/cli/src/recipe-sources.js" import type { WorkspaceRecipe } from "../packages/runtime-core/src/index.js" -const customTmp = await mkdtemp(join(tmpdir(), "wp-codebox-cleanup-test-root-")) +const customTmp = await realpath(await mkdtemp(join(tmpdir(), "wp-codebox-cleanup-test-root-"))) const originalTmp = process.env.TMPDIR process.env.TMPDIR = customTmp const complete = (rows: ProcessEvidence["rows"] = []): ProcessEvidence => ({ available: true, complete: true, rows, blockers: [] }) diff --git a/tests/wordpress-runtime-discovery-coverage-plan.test.ts b/tests/wordpress-runtime-discovery-coverage-plan.test.ts index 3431cbf32..26bc9924d 100644 --- a/tests/wordpress-runtime-discovery-coverage-plan.test.ts +++ b/tests/wordpress-runtime-discovery-coverage-plan.test.ts @@ -57,7 +57,7 @@ assert.equal(plan.discovered.some((item) => item.id === "frontend-url-root"), tr assert.equal(plan.discovered.some((item) => item.id === "db-inspect-posts"), true) assert.equal(plan.discovered.some((item) => item.id === "block-core-paragraph-server-render-sample-attributes"), true) assert.equal(plan.discovered.some((item) => item.target?.entrypoint === "wordpress.block-render"), true) -assert.equal(plan.discovered.some((item) => item.target?.entrypoint === "wordpress.block-exercise"), true) +assert.equal(plan.discovered.some((item) => item.target?.entrypoint === "wordpress.editor-actions"), true) assert.equal(plan.discovered.some((item) => item.id === "crud-list-post"), true) assert.equal(plan.discovered.some((item) => item.id === "crud-read-post"), true) assert.equal(plan.discovered.some((item) => item.id === "crud-list-term-category"), true) @@ -82,7 +82,6 @@ assert.equal(plan.skipped.some((item) => item.reason?.code === "admin_page_capab assert.equal(plan.skipped.some((item) => item.reason?.code === "external_table_not_fuzzed"), true) assert.equal(plan.untested.some((item) => item.reason?.code === "mutating_rest_method_requires_explicit_opt_in"), true) assert.equal(plan.untested.some((item) => item.reason?.code === "block_inserter_unsupported"), true) -assert.equal(plan.untested.some((item) => item.reason?.code === "block_editor_insert_save_runtime_unsupported"), true) assert.equal(plan.untested.some((item) => item.reason?.code === "crud_resource_identifier_required"), true) assert.equal(plan.executable.every((item) => item.input !== undefined && !item.reason), true) assert.equal(plan.discovered.every((item) => Boolean(item.metadata?.requiredRunnerCapabilities)), true)