diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee256f..336a7d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,8 @@ jobs: with: node-version: "24" - run: bun install --frozen-lockfile + # Screenshot e2e tests drive a real browser + - run: bunx playwright install --with-deps chromium - run: bun run check - run: bun prettier --check . - run: bun test tests/unit/ tests/e2e/ @@ -38,6 +40,11 @@ jobs: npm init -y > /dev/null npm install "$RUNNER_TEMP"/srcpack-*.tgz > /dev/null echo 'export const x = 1;' > src/index.ts - printf 'import { defineConfig } from "srcpack";\nexport default defineConfig({ bundles: { app: "src/**/*" } });\n' > srcpack.config.mts + printf 'import { defineConfig } from "srcpack";\nexport default defineConfig({ bundles: { app: "src/**/*", home: { screenshot: "http://localhost:9/", onDemand: true } } });\n' > srcpack.config.mts ./node_modules/.bin/srcpack --dry-run | tee out.txt grep -q 'src/index.ts' out.txt + # Playwright is an optional peer, so it is absent here: a preview + # needs no browser, and a capture explains how to install one + ./node_modules/.bin/srcpack home --dry-run | grep -q 'home-NN.png' + if ./node_modules/.bin/srcpack home 2> err.txt; then exit 1; fi + grep -q 'screenshots need Playwright' err.txt diff --git a/README.md b/README.md index 1853ab2..d1e0942 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Zero-config CLI for bundling code into LLM-optimized context files. ```bash npx srcpack init # Create config interactively -npx srcpack # Bundle all +npx srcpack # Bundle all except on-demand ``` ## Why @@ -57,7 +57,7 @@ Or add to `package.json`: | `bundles` | — | Named bundle definitions | | `upload` | — | Upload destination(s) | -\*Only the default `.srcpack` is emptied automatically — it's srcpack's directory by convention. Any other `outDir` needs an explicit `emptyOutDir: true`, so `outDir: "src"` can't quietly delete your sources. Emptying also happens only on a full run, so `npx srcpack web` leaves other bundles in place. +\*Only the default `.srcpack` is emptied automatically — it's srcpack's directory by convention. Any other `outDir` needs an explicit `emptyOutDir: true`, so `outDir: "src"` can't quietly delete your sources. Named runs (`npx srcpack web`) never empty it, so other bundles stay in place. ### Bundle Config @@ -78,9 +78,11 @@ Or add to `package.json`: { include: "src/**/*", linear: { team: "ENG" }, // Linear issues as virtual files + screenshot: "http://localhost:5173", // rendered page as numbered PNGs outfile: "~/Downloads/bundle.txt", // custom output path index: true, // include index header (default) - prompt: "./prompts/review.md" // prepend from file (or inline text) + prompt: "./prompts/review.md", // prepend from file (or inline text) + onDemand: true // build only when named } ``` @@ -88,6 +90,8 @@ Patterns follow glob syntax. Prefix with `!` to exclude, `+` to force-include (b A pattern can also name a set of changed files: `git:staged`, `git:unstaged`, `git:untracked`, `git:dirty`, or `git:` (e.g. `git:main`, `git:HEAD~3`). Deleted files are skipped, and `git:` compares against the merge base so a stale branch still reports only your own changes. See [Git sources](https://kriasoft.com/srcpack/configuration#git-sources-git-prefix). +A full run skips bundles marked `onDemand: true`; name them explicitly to build them. Emptying `outDir` still removes their previous output there. See [On-Demand Bundles](https://kriasoft.com/srcpack/configuration#on-demand-bundles). + ### Linear Issues A bundle can include [Linear](https://linear.app) issues next to your code. Each issue becomes a virtual file at `linear/issues/ENG-123.md`, so it gets its own index entry and line range — letting you ask whether `[4] src/board.ts` actually implements `[2] ENG-123`. @@ -104,6 +108,18 @@ bundles: { Authentication reads `LINEAR_API_KEY` from the environment (Linear → Settings → Security & access → Personal API keys), never from the config file. `team` is required, completed/canceled/duplicate issues are excluded by default, and issues obey `!` exclusions like any other entry. See [Linear issues](https://kriasoft.com/srcpack/configuration#linear-issues). +### Screenshots + +A bundle can capture a rendered page as PNGs, with overlapping detail slices for tall pages. Srcpack scrolls to load lazy content before capture and hides Astro and Nuxt dev toolbars. + +```typescript +bundles: { + home: { screenshot: "http://localhost:5173/", onDemand: true }, +} +``` + +`npx srcpack home` writes PNGs to `.srcpack`: `home-00.png` is the whole page when available; tall pages also get `home-01.png`, `home-02.png`, … detail slices. Attach them in filename order. Images stay local, even with upload configured. For a one-off, no config is needed: `npx srcpack --screenshot localhost:5173 --viewport mobile`. See [Screenshots](https://kriasoft.com/srcpack/configuration#screenshots) for Playwright and browser setup. + ### Google Drive Upload To upload bundles to Google Drive, add OAuth credentials to your config: @@ -154,14 +170,15 @@ import { utils } from "./utils"; ## CLI ```bash -npx srcpack # Bundle all, upload if configured +npx srcpack # Bundle all except on-demand, upload if configured npx srcpack web api # Bundle specific bundles only npx srcpack --staged # Bundle staged changes (no config needed) npx srcpack --dirty # Bundle staged + unstaged + untracked npx srcpack --since main # Bundle changes since main +npx srcpack --screenshot localhost:5173 # Capture a page as PNGs npx srcpack --dry-run # Preview without writing files npx srcpack --emptyOutDir # Empty output directory before writing -npx srcpack --no-emptyOutDir # Keep existing files in output directory +npx srcpack --no-emptyOutDir # Skip clearing the output directory npx srcpack --no-upload # Bundle only, skip upload npx srcpack init # Interactive config setup npx srcpack login # Authenticate with Google Drive diff --git a/bun.lock b/bun.lock index 7140073..e9ca1b1 100644 --- a/bun.lock +++ b/bun.lock @@ -19,11 +19,18 @@ "devDependencies": { "@types/bun": "^1.3.14", "@types/picomatch": "^4.0.3", + "playwright": "1.63.0", "prettier": "^3.9.6", "typescript": "^6.0.3", "vitepress": "^2.0.0-alpha.19", "vitepress-plugin-llms": "^1.13.5", }, + "peerDependencies": { + "playwright": "*", + }, + "optionalPeers": [ + "playwright", + ], }, }, "packages": { @@ -523,6 +530,10 @@ "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], + "playwright": ["playwright@1.63.0", "", { "dependencies": { "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" } }, "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg=="], + + "playwright-core": ["playwright-core@1.63.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg=="], + "postcss": ["postcss@8.5.26", "", { "dependencies": { "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ=="], "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], @@ -657,8 +668,6 @@ "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], - "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], - "yargs": ["yargs@17.7.3", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g=="], "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], diff --git a/docs/adr/005-on-demand-bundles.md b/docs/adr/005-on-demand-bundles.md new file mode 100644 index 0000000..b64393e --- /dev/null +++ b/docs/adr/005-on-demand-bundles.md @@ -0,0 +1,24 @@ +# ADR 005: On-demand bundles + +**Status:** Accepted — 2026-09-13 + +## Context + +Some bundles are not worth rebuilding on every `srcpack`: a Linear bundle costs a network round trip and a token, and a screenshot bundle needs a running dev server. Without a way to opt out, they either slow every run or fail it, and users drop them from the config — losing the "define once, rerun by name" workflow. + +## Decision + +A bundle may set `onDemand: true`. A full run (`srcpack`) skips it and prints `On demand: `; naming it (`srcpack home`) builds it. `--dry-run` previews the same selection as the command it accompanies. If every bundle is on demand, a full run prints that line and exits 0. + +The rule is the same for every source. Screenshot bundles are not on demand implicitly: a default that depends on source type is one more rule to learn. + +Emptying does not change. A full run that empties `outDir` — by default only `.srcpack` — removes on-demand output inside it; the workflow is `srcpack`, then `srcpack home`. Where emptying is off, or an `outfile` sits outside `outDir`, that output survives and goes stale, exactly as a renamed bundle's does. + +## Alternatives + +- **`enabled: false`** — "disabled, unless named" contradicts itself. +- **Preserving on-demand output across full runs** — turns `emptyDirectory` into a growing list of exceptions, and custom outfiles make those exceptions paths rather than names. If it proves essential, an ownership manifest is the cleaner answer. + +## Consequences + +A full run no longer means "every configured bundle". Under the default `.srcpack`, output of an on-demand bundle does not survive a full run, which is visible (the file is gone) rather than stale. diff --git a/docs/adr/006-screenshot-bundles.md b/docs/adr/006-screenshot-bundles.md new file mode 100644 index 0000000..2b34900 --- /dev/null +++ b/docs/adr/006-screenshot-bundles.md @@ -0,0 +1,59 @@ +# ADR 006: Screenshots as a bundle source + +**Status:** Accepted — 2026-09-13 + +## Context + +Reviewing a web app with a model means handing it the rendered page, not only the code. `page.screenshot()` is not enough on its own: + +- **Tall pages become unreadable.** Vision models downscale every image to a fixed pixel budget, so a 1440×9000 page reaches them as a thumbnail. +- **The page isn't finished when `load` fires.** A full-page screenshot doesn't move the viewport, so lazy images and IntersectionObserver content stay blank. +- **Dev chrome pollutes the image.** Framework toolbars and cookie banners get reviewed as if they were the design. + +And the workflow should be srcpack's: define once, rerun by name. A bundle stops meaning "sources → one text file" and becomes a named context packet — text, images, or both. + +## Decision + +**`screenshot` is a bundle source key**, the same shape as `linear`: + +```ts +bundles: { + home: { screenshot: "http://localhost:5173/", onDemand: true }, + pricing: { + include: "src/pages/pricing/**/*", + screenshot: { url: "http://localhost:5173/pricing", viewport: "mobile" }, + }, +} +``` + +A top-level bundle `url` would drag `viewport` and `hide` onto every bundle, where on a text bundle they would do nothing. `prompt`, `index` and `outfile` describe the text file, so on a screenshot-only bundle they are config errors — which is why `index` is no longer defaulted in the schema. + +**Not generalizing to `sources: [{ provider }]`.** [ADR 003](./003-linear-issues-as-virtual-files.md) predicted that shape once a third source arrived. This is the third source, but `include` and `linear` produce entries in one text file while `screenshot` produces sibling binary files. A common provider abstraction would hide that difference rather than simplify it. + +**Output is a flat numbered family, `/-NN.png`.** `00` is the whole page and `01…` are detail slices top to bottom, so lexical order is review order. Each image carries a logical index: an overview omitted for size leaves `01…` in place rather than renumbering a detail slice into `00`. There is no `outfile`, so a family always lives in `outDir` and removing stale images never reaches a directory srcpack doesn't own. + +**A family is a claim on a directory plus a name prefix.** Its directory resolves fully — it is an ancestor of every PNG, unlike a text output's last component, which `rename` replaces (ADR 004). Families collide when their folded names match (`Web`/`web`); a text `outfile` collides with a family when it resolves into the family's directory under one of its numbered names, the same bundle's family included. Stale-image cleanup matches exactly, never folded, since folding could only widen a deletion. + +**Configured and active writers are checked separately.** Configured bundles are checked against each other on every run, so a config error doesn't depend on what was asked for. The bundles a run writes — ad-hoc included — are checked against configured bundles of a different name, so `--screenshot` shadows a configured bundle named `screenshot` but cannot overwrite another bundle's file. + +**Capture settles the page before it slices it.** It scrolls one viewport at a time, re-reading the page height so sections that load taller than their placeholders are followed, capped at 50 steps and 15 seconds with a warning. At each apparent bottom it waits for the network to go quiet and scrolls on if a response appended content, since growth measured but never visited is sliced with its lazy content blank. The wait uses its own in-flight request tracker: Playwright's `networkidle` load state resolves immediately once it has fired, which misses exactly the requests scrolling started. Slices are clipped captures of at most 2,200 device px: the fewest that cover the page with at least 160 px overlap, shrunk to share it evenly so a page just over the limit doesn't become two near-identical images; the overview is best-effort. + +**Playwright is an optional peer**, resolved from the project first (`playwright`, then `@playwright/test`), then srcpack's own install. Under `npx`, srcpack runs from the npm cache where a bare import never sees the project's copy. The peer range is `*`: a project that never takes a screenshot shouldn't have its install judged against its Playwright version. Compatibility (1.41 up to 2) is checked at load instead, which also covers a copy found in the project, where a peer range constrains nothing. Without Playwright's Chromium, the system Chrome is used. + +**Images stay local.** Drive uploads find files by name and update in place, so a page that shrank would leave old slices there with nothing to remove them. A mixed bundle's text file still uploads. + +**`--dry-run` launches no browser.** Unlike Linear, there is no cheap way to answer "what would this produce" short of the capture itself, and a plain `fetch` would be a second code path that can pass where the capture then fails. The preview lists URL, viewport and destination; config and collision errors still surface. + +## Alternatives + +- **A top-level `url` on bundles** — rejected above. +- **`sources: [{ provider }]`** — hides the text-versus-files difference it claims to unify. +- **Stitching one tall image, or an image library for slicing** — clipped Playwright captures need no dependency, and a stitched image is exactly what downscaling ruins. +- **Several URLs per bundle** — numbering across pages hides which image is which page; `srcpack home pricing` already composes. +- **Uploading images** — needs Drive-side cleanup of shrunk pages first. + +## Consequences + +srcpack can now require a browser, but only for bundles that ask for one; nobody else downloads anything. Screenshot bundles need a running server, which is what `onDemand` ([ADR 005](./005-on-demand-bundles.md)) is for, and the unreachable-URL error in a full run says so. + +Captured images are held in memory until every bundle resolves, so a failure anywhere leaves the previous run's files in place. As with text, each file is replaced atomically but a run is not. Authenticated pages, custom viewport sizes and dark mode are deferred. diff --git a/docs/adr/index.md b/docs/adr/index.md index 211b4f4..ec1e189 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -10,3 +10,5 @@ A record is never rewritten once accepted. When a decision is reversed, a new re | [002](./002-minimum-node-version.md) | Node 22.18 as the minimum runtime | Accepted 2026-08-15 | | [003](./003-linear-issues-as-virtual-files.md) | Linear issues as virtual files, under a `linear` bundle key | Accepted 2026-08-15 | | [004](./004-path-boundaries.md) | What srcpack may read, delete and overwrite | Accepted 2026-08-15 | +| [005](./005-on-demand-bundles.md) | `onDemand` bundles: skipped by a full run, built when named | Accepted 2026-09-13 | +| [006](./006-screenshot-bundles.md) | Screenshots as a `screenshot` bundle source, written as numbered PNGs | Accepted 2026-09-13 | diff --git a/docs/cli.md b/docs/cli.md index 2abe897..e565659 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -50,7 +50,7 @@ yarn add -D srcpack ### `srcpack` (default) -Bundle all configured bundles and upload if configured. +Build configured bundles except those marked [`onDemand`](./configuration.md#on-demand-bundles), and upload their text files if configured. ::: code-group @@ -94,6 +94,8 @@ yarn dlx srcpack web api ::: +Naming a bundle builds it even when it is marked [`onDemand`](./configuration.md#on-demand-bundles); a full run skips those and lists them after the summary. + **Changed files only:** ```sh @@ -102,12 +104,23 @@ npx srcpack --dirty # staged + unstaged + untracked npx srcpack --since main # everything you changed since main ``` -These build a one-off bundle from the current change set and need no config file — handy for handing a work-in-progress to an LLM. The bundle is named after the flag (`.srcpack/staged.txt`), other bundles in `outDir` are left alone, and nothing is written when there are no changes. +These build a one-off bundle from the current change set and need no config file — handy for handing a work-in-progress to an LLM. The bundle is named after the flag (`.srcpack/staged.txt`), other bundles in `outDir` are left alone unless you pass `--emptyOutDir`, and nothing is written when there are no changes. Ad-hoc bundles stay local: they are never uploaded, even with Google Drive configured. Declare a named bundle to publish changes. +One-off flags use the discovered config's `root` and `outDir`, but replace the source definition of any configured bundle with the same name. For example, `--screenshot` replaces the images of a configured bundle named `screenshot`. Output collisions with differently named bundles are rejected. + For a permanent version with review instructions attached, put a [git source](./configuration.md#git-sources-git-prefix) in your config instead. +**A page, as images:** + +```sh +npx srcpack --screenshot localhost:5173/pricing +npx srcpack --screenshot localhost:5173 --viewport mobile +``` + +Captures the page as numbered PNGs in `outDir` (by default `.srcpack/screenshot-00.png`, `screenshot-01.png`, …), with detail slices for tall pages. No config file is required. A URL without a scheme gets `http://`. Like the change-set flags, it leaves other files in `outDir` alone unless you pass `--emptyOutDir`, and never uploads. It needs Playwright — see [Screenshots](./configuration.md#screenshots), which also covers declaring a page you capture repeatedly. + ### `srcpack init` Create a `srcpack.config.ts` interactively. @@ -162,19 +175,25 @@ Opens a browser to authorize access. Tokens are stored in `~/.config/srcpack/cre ## Options -| Option | Description | -| ------------------ | ---------------------------------------------- | -| `--staged` | Bundle staged changes only | -| `--dirty` | Bundle staged, unstaged, and untracked changes | -| `--since ` | Bundle changes since `` | -| `--dry-run` | Preview bundles without writing files | -| `--emptyOutDir` | Empty the output directory before writing | -| `--no-emptyOutDir` | Keep what is already in the output directory | -| `--no-upload` | Bundle only, skip upload | -| `-h`, `--help` | Show help | -| `-v`, `--version` | Show version | +| Option | Description | +| -------------------- | ---------------------------------------------------- | +| `--staged` | Bundle staged changes only | +| `--dirty` | Bundle staged, unstaged, and untracked changes | +| `--since ` | Bundle changes since `` | +| `--screenshot ` | Capture a page as numbered PNGs | +| `--viewport ` | `desktop` (default) or `mobile`, with `--screenshot` | +| `--dry-run` | Preview bundles without writing files | +| `--emptyOutDir` | Empty the output directory before writing | +| `--no-emptyOutDir` | Skip clearing the output directory | +| `--no-upload` | Bundle only, skip upload | +| `-h`, `--help` | Show help | +| `-v`, `--version` | Show version | + +`--no-emptyOutDir` disables directory-wide clearing; bundles still replace their outputs and remove stale files from their own previous output. See [output cleanup](./configuration.md#outdir). + +The one-off flags (`--staged`, `--dirty`, `--since`, `--screenshot`) cannot be combined with each other or with bundle names. `--viewport` applies only to `--screenshot`; set the viewport in config for a named bundle. -An unrecognized option is an error, not a no-op — `--no-uplaod` would otherwise upload, and `--dry-rnu` would write. +An unrecognized option is an error, not a no-op — `--no-uplaod` would otherwise upload, and `--dry-rnu` would write. Values may follow a space or an `=`: `--since main` and `--since=main` are the same. ## Examples @@ -213,7 +232,7 @@ Output: Dry run: 2 bundles, 4 files, 938 lines ``` -Each bundle lists the files it would contain, so you can check the shape of a pattern before anything is written; `outDir` is left alone too. A bundle that declares [`linear`](./configuration.md#linear-issues) still calls the API — the counts are what it would produce right now, which it can't know offline. +Each bundle lists the files it would contain, so you can check the shape of a pattern before anything is written; `outDir` is left alone too. A bundle that declares [`linear`](./configuration.md#linear-issues) still calls the API — the counts are what it would produce right now, which it can't know offline. A [`screenshot`](./configuration.md#screenshots) source is listed with its URL, viewport and destination without launching a browser. A mixed bundle still resolves its text sources. ### Bundle without upload diff --git a/docs/configuration.md b/docs/configuration.md index db22c95..3aa0ce5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -83,7 +83,9 @@ Ownership is decided by physical path. A `.srcpack` that turns out to be a symli Emptying waits until every bundle has resolved, immediately before the new files are written. A run that fails while resolving — an unreadable `.gitignore`, an expired `LINEAR_API_KEY` — leaves the previous output intact. -Emptying only happens on a full run. `srcpack web` leaves the bundles it isn't building in place, since it has no way to tell which of them are stale. +Named runs (`srcpack web`) never empty the directory. One-off runs (`--staged`, `--dirty`, `--since`, `--screenshot`) empty it only when you pass `--emptyOutDir`, regardless of the config setting. + +`emptyOutDir: false` and `--no-emptyOutDir` disable directory-wide clearing. Selected bundles still replace their outputs, remove stale numbered images, and remove empty text outputs inside `outDir`. ## Bundle Definitions @@ -125,14 +127,35 @@ bundles: { | --- | --- | --- | --- | | `include` | `string \| string[]` | — | Glob pattern(s) | | `linear` | `string \| object` | — | Linear issues (see below) | +| `screenshot` | `string \| object` | — | Page captured as PNGs (see below) | | `outfile` | `string` | `{outDir}/{name}.txt` | Custom output path | | `index` | `boolean` | `true` | Include index header | | `prompt` | `string` | — | Text or file path (`./`, `~/`) to prepend | +| `onDemand` | `boolean` | `false` | Build only when named (see below) | -A bundle needs at least one source: `include`, `linear`, or both. +A bundle needs at least one source: `include`, `linear`, `screenshot`, or any combination. `outfile`, `index` and `prompt` apply only to text and require `include` or `linear`. Images always go to `outDir`. Two bundles may not write to the same file. Names that differ only by case, or only in Unicode normalisation, count as the same file everywhere: on a case-insensitive filesystem — the default on macOS and Windows — `Web.txt` and `web.txt` are one directory entry, and APFS treats the two spellings of `Café` the same way, so one bundle would silently overwrite the other. +### On-Demand Bundles + +A bundle that is slow, remote, or only occasionally useful can opt out of full runs: + +```ts +bundles: { + code: "src/**/*", + backlog: { linear: "ENG", onDemand: true }, +} +``` + +| Command | Builds | +| ----------------- | ----------------------------------------- | +| `srcpack` | every bundle without `onDemand: true` | +| `srcpack backlog` | `backlog`, on demand or not | +| `--dry-run` | the same selection as the run it previews | + +A full run lists what it skipped (`On demand: backlog`). If every bundle is on demand, it writes no bundles and exits successfully; the same emptying rules still apply. A full run that empties `outDir` (by default, only `.srcpack` is emptied) removes an on-demand bundle's previous output there — run `srcpack`, then `srcpack backlog`. + ## Pattern Syntax Patterns follow standard glob syntax with special prefixes: @@ -282,6 +305,112 @@ Notes: - **`--dry-run` still calls the API**, because it has to in order to answer "what would this produce right now". There is no cache; a run without network access fails. Requests time out after 30 seconds. - **`linear/issues/` is reserved** once a bundle pulls issues. A file on disk at that path is an error: two entries would share one name, and an `!` exclusion drops both rather than choosing. Rename the file, or narrow the include patterns so it isn't matched. +## Screenshots + +A bundle can capture a rendered page as PNG images a vision model can actually read — on its own, or next to the code that renders it: + +```ts +bundles: { + home: { screenshot: "http://localhost:5173/", onDemand: true }, + + "home-mobile": { + screenshot: { + url: "http://localhost:5173/", + viewport: "mobile", + hide: ["#cookie-banner", ".intercom-launcher"], + }, + onDemand: true, + }, + + // Implementation and rendered result under one name + pricing: { + include: "src/pages/pricing/**/*", + screenshot: "http://localhost:5173/pricing", + prompt: "Review this implementation against the attached screenshots.", + onDemand: true, + }, +} +``` + +```console +$ srcpack home home-mobile + home 4 images page 1440×6,210 → .srcpack/home-00.png … home-03.png + home-mobile 11 images page 412×9,480 → .srcpack/home-mobile-00.png … home-mobile-10.png + +Bundled: 2 bundles, 15 images +``` + +Then drag `.srcpack/home-*.png` into ChatGPT, in filename order. A mixed bundle like `pricing` writes its text file and its images, and prints a line for each. + +`onDemand: true` keeps a full `srcpack` from failing whenever the dev server isn't running — see [On-Demand Bundles](#on-demand-bundles). + +### Setup + +Screenshots use [Playwright](https://playwright.dev), which srcpack doesn't install for you — most projects never need a browser: + +::: code-group + +```sh [npm] +npm install -D playwright && npx playwright install chromium +``` + +```sh [bun] +bun add -d playwright && bunx playwright install chromium +``` + +```sh [pnpm] +pnpm add -D playwright && pnpm exec playwright install chromium +``` + +```sh [yarn] +yarn add -D playwright && yarn playwright install chromium +``` + +::: + +A project that already uses Playwright Test needs nothing new. Without Playwright's Chromium, srcpack uses your installed Google Chrome. Playwright 1.41 or a later 1.x is required. + +### Filenames + +| File | Contains | +| ----------------------------- | ---------------------------------------- | +| `-00.png` | The whole page, for layout | +| `-01.png`, `-02.png`, … | Overlapping detail slices, top to bottom | + +Vision models scale every image down to a fixed pixel budget, so a tall page captured whole arrives as a thumbnail. Detail slices stay readable after that: they are at most 2,200 device pixels tall and overlap by about 160, so text cut at one edge is whole in the next; a page just over one slice becomes two shorter slices rather than two near-copies. A page that fits in one slice produces only `-00.png`. If the highest image index exceeds 99, all filenames use wider zero-padding (`-000.png`, `-001.png`, …) to preserve filename order. + +The overview is best-effort. If a very tall page can't be captured whole, or its image would exceed ChatGPT's 20 MB per-image limit, srcpack warns and writes the detail slices alone, starting at `-01`. Those still cover the whole page. + +### Options + +| Option | Type | Default | Description | +| --- | --- | --- | --- | +| `url` | `string` | — | Required. An `http://` or `https://` URL | +| `viewport` | `"desktop" \| "mobile"` | `"desktop"` | Viewport preset (below) | +| `hide` | `string[]` | — | CSS selectors hidden during capture | + +The string form `screenshot: "http://…"` is shorthand for `{ url: "http://…" }`. + +| Viewport | CSS size | Pixel ratio | Emulation | +| --- | --- | --- | --- | +| `desktop` | 1440×900 | 1 | — | +| `mobile` | 412×839 | 2 | Pixel 7 user agent, touch, mobile layout | + +### Notes + +- **The page is scrolled before capture**, so lazy images and content that appears on scroll are rendered. Sections that grow as they load are followed down, and at the bottom srcpack waits for 500 ms of network quiet, capped at 5 seconds per wait, and keeps scrolling if requests added content. It stops after 50 viewports or 15 seconds and warns that content further down may not have loaded. +- **Framework dev toolbars are hidden** automatically (Astro, Nuxt). Next.js is left alone: its `nextjs-portal` also shows build and runtime errors, which a review should see. Use `hide` to suppress `"nextjs-portal"` or selectors for cookie banners and chat widgets. Elements are hidden during capture with `visibility: hidden`, preserving their layout space. +- **A page that doesn't load fails the run**: an unreachable URL, a non-2xx status, or navigation that does not finish loading within 30 seconds. A screenshot of a 404 page would look like success. The previous run's images are left untouched. +- **Stale images are removed.** When a page shrinks from six images to four, `-04` and `-05` are deleted, so an old slice is never attached with the new set. +- **The URL needs a scheme** in config: `"localhost:5173"` is an error. (`--screenshot` on the command line adds `http://` for you.) +- **Images stay local.** A configured upload skips them and says so; a mixed bundle's text file still uploads. +- **`--dry-run` doesn't open the page.** It lists the URL, viewport and destination; how many images a page produces is only known after rendering it. +- **A page without ``** lays out 980 CSS pixels wide under `mobile`, as it would on a real phone. If a mobile capture looks like the desktop site, that's why. +- **Output collisions are rejected**, including screenshot bundle names that differ only by case and a text `outfile` that lands in `outDir` under any bundle's numbered image name, even its own. Checks cover all configured bundles, including those skipped by this run. +- **Virtualized lists and content that disappears once scrolled past** may be missing: the page is scrolled to load content, then captured from the top. +- **A page that scrolls inside its own container** (the window never scrolls) is captured as a single viewport. +- **Pages behind a login aren't supported yet.** + ## Automatic Exclusions Srcpack skips: @@ -289,7 +418,7 @@ Srcpack skips: - Files matching `.gitignore` — including `node_modules/`, build output, and secrets, since those are already ignored in any normal project. Nested `.gitignore` files count too, resolved the way git resolves them: the rule in the deepest directory wins, and nothing under an ignored directory is re-included. A monorepo's `packages/app/.gitignore` hides its `.env` here exactly as it does for git. - Binary files (images, fonts, compiled assets), detected by content - Symlinks, so a link can't pull in a file from outside the project — including symlinked directories, which are not walked into -- Its own output — `outDir` and every configured `outfile`. Otherwise a rerun would bundle the previous run's file, nesting it again each time. +- Its own output — every configured or active text output, plus `outDir` unless it contains the project root. In that case, excluding the whole directory would also exclude the sources; individual text outputs remain excluded and PNGs are skipped as binary. Everything else matched by a pattern is included, so exclude what you don't want explicitly: `["src/**/*", "!bun.lock"]`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 1b6af72..c8677d0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -201,7 +201,7 @@ Each issue becomes its own indexed entry (`linear/issues/ENG-123.md`). Set `LINE ::: code-group ```sh [npm] -npx srcpack # Bundle all +npx srcpack # Bundle all except on-demand npx srcpack web api # Bundle specific bundles only npx srcpack --staged # Bundle staged changes (no config needed) npx srcpack --dry-run # Preview without writing @@ -209,7 +209,7 @@ npx srcpack --no-upload # Skip upload even if configured ``` ```sh [bun] -bunx srcpack # Bundle all +bunx srcpack # Bundle all except on-demand bunx srcpack web api # Bundle specific bundles only bunx srcpack --staged # Bundle staged changes (no config needed) bunx srcpack --dry-run # Preview without writing @@ -217,7 +217,7 @@ bunx srcpack --no-upload # Skip upload even if configured ``` ```sh [pnpm] -pnpm dlx srcpack # Bundle all +pnpm dlx srcpack # Bundle all except on-demand pnpm dlx srcpack web api # Bundle specific bundles only pnpm dlx srcpack --staged # Bundle staged changes (no config needed) pnpm dlx srcpack --dry-run # Preview without writing @@ -225,7 +225,7 @@ pnpm dlx srcpack --no-upload # Skip upload even if configured ``` ```sh [yarn] -yarn dlx srcpack # Bundle all +yarn dlx srcpack # Bundle all except on-demand yarn dlx srcpack web api # Bundle specific bundles only yarn dlx srcpack --staged # Bundle staged changes (no config needed) yarn dlx srcpack --dry-run # Preview without writing diff --git a/docs/upload.md b/docs/upload.md index 163a94f..3496df2 100644 --- a/docs/upload.md +++ b/docs/upload.md @@ -1,6 +1,6 @@ # Google Drive Upload -Srcpack can automatically upload bundles to Google Drive, making them accessible from any device or shareable with your team. +Srcpack can automatically upload bundle text files to Google Drive, making them accessible from any device or shareable with your team. [Screenshots](./configuration.md#screenshots) stay local; a mixed bundle uploads only its text file. ## Setup diff --git a/package.json b/package.json index 6e2d5a3..96c57f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srcpack", - "version": "0.3.0", + "version": "1.0.0", "description": "Zero-config CLI for bundling code into LLM-optimized context files", "keywords": [ "llm", @@ -87,9 +87,18 @@ "picomatch": "^4.0.5", "zod": "^4.4.3" }, + "peerDependencies": { + "playwright": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + } + }, "devDependencies": { "@types/bun": "^1.3.14", "@types/picomatch": "^4.0.3", + "playwright": "1.63.0", "prettier": "^3.9.6", "typescript": "^6.0.3", "vitepress": "^2.0.0-alpha.19", diff --git a/src/args.ts b/src/args.ts new file mode 100644 index 0000000..3884206 --- /dev/null +++ b/src/args.ts @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: MIT + +import { parseArgs } from "node:util"; +import type { BundleConfig } from "./config.ts"; + +/** A mistake on the command line: reported without a stack trace. */ +export class UsageError extends Error { + constructor(message: string) { + super(message); + this.name = "UsageError"; + } +} + +/** + * One-off bundle from a `git:` source or a URL; no config file required. + */ +export interface AdHocBundle { + name: string; + source: BundleConfig; +} + +export interface CliArgs { + /** Bundle names in input order. Empty for full and ad-hoc runs. */ + bundles: string[]; + adHoc: AdHocBundle | null; + dryRun: boolean; + /** Explicit CLI override; `undefined` lets the CLI apply config and run defaults. */ + emptyOutDir: boolean | undefined; + upload: boolean; +} + +/** + * Every option the CLI accepts; `strict` rejects the rest. A typo that gets + * quietly dropped is the dangerous kind: `--no-uplaod` uploads, `--dry-rnu` + * writes, `--no-emptyOutdir` empties. Same rule as the config — a token that + * changes what a run destroys or publishes is never a silent no-op. + * + * Negatives are declared literally rather than with `allowNegative`, which + * negates every boolean and would make `--upload` valid by defining `upload`. + */ +const OPTIONS = { + staged: { type: "boolean" }, + dirty: { type: "boolean" }, + // Valued options are `multiple` so a repeat is seen and rejected rather + // than last-one-wins + since: { type: "string", multiple: true }, + screenshot: { type: "string", multiple: true }, + viewport: { type: "string", multiple: true }, + "dry-run": { type: "boolean" }, + emptyOutDir: { type: "boolean" }, + "no-emptyOutDir": { type: "boolean" }, + "no-upload": { type: "boolean" }, +} as const; + +const AD_HOC_FLAGS = ["staged", "dirty", "since", "screenshot"] as const; + +const MISSING_VALUE = { + since: "Missing revision: --since (e.g. --since main)", + screenshot: + "Missing URL: --screenshot (e.g. --screenshot localhost:5173)", + viewport: "Missing viewport: --viewport ", +}; + +const VIEWPORTS = ["desktop", "mobile"] as const; + +/** + * Parse everything after `srcpack` except `--help`, `--version` and the + * `init`/`login` subcommands, which the CLI handles before this. + */ +export function parseCliArgs(argv: string[]): CliArgs { + // parseArgs accepts `--` as a terminator even when strict, which would turn + // a stray `srcpack --` into a full run that empties and uploads + if (argv.includes("--")) throw unknownOption("--"); + + let parsed; + try { + parsed = parseArgs({ + args: argv, + options: OPTIONS, + strict: true, + allowPositionals: true, + }); + } catch (error) { + throw toUsageError(error as NodeJS.ErrnoException); + } + const { values, positionals } = parsed; + + // Both flags set both values, so parseArgs can't reject this on its own + if (values.emptyOutDir && values["no-emptyOutDir"]) { + throw new UsageError("Cannot combine --emptyOutDir with --no-emptyOutDir."); + } + + const adHocFlags = AD_HOC_FLAGS.filter((flag) => values[flag] !== undefined); + if (adHocFlags.length > 1) { + throw new UsageError( + `Cannot combine ${adHocFlags.map((flag) => `--${flag}`).join(" and ")}.`, + ); + } + for (const flag of Object.keys( + MISSING_VALUE, + ) as (keyof typeof MISSING_VALUE)[]) { + const given = values[flag] ?? []; + if (given.length > 1) { + throw new UsageError( + `--${flag} takes one value; got ${given.map((v) => `"${v}"`).join(" and ")}.`, + ); + } + } + const [viewport] = values.viewport ?? []; + if (viewport !== undefined && !values.screenshot) { + throw new UsageError("--viewport applies to --screenshot ."); + } + const adHoc = toAdHocBundle(adHocFlags[0], { + since: values.since?.[0], + screenshot: values.screenshot?.[0], + viewport, + }); + if (adHoc && positionals.length) { + throw new UsageError(`Cannot combine --${adHoc.name} with named bundles.`); + } + + return { + bundles: positionals, + adHoc, + dryRun: values["dry-run"] ?? false, + emptyOutDir: values.emptyOutDir + ? true + : values["no-emptyOutDir"] + ? false + : undefined, + upload: !values["no-upload"], + }; +} + +function toAdHocBundle( + flag: (typeof AD_HOC_FLAGS)[number] | undefined, + value: { since?: string; screenshot?: string; viewport?: string }, +): AdHocBundle | null { + switch (flag) { + case "staged": + return { name: "staged", source: ["git:staged"] }; + case "dirty": + return { name: "dirty", source: ["git:dirty"] }; + case "since": { + const rev = value.since; + // `--since=` parses as an empty value rather than a missing one + if (!rev) throw new UsageError(MISSING_VALUE.since); + // A range pins both endpoints, so it would silently drop the uncommitted + // work --since promises. Ranges belong in a config `git:` source. + if (rev.includes("..")) { + throw new UsageError( + `--since takes a revision, not a range: "${rev}". Use a git: source in your config for ranges.`, + ); + } + // `git diff` can't see untracked files, but a new file written on this + // branch is part of "what changed since " + return { name: "since", source: [`git:${rev}`, "git:untracked"] }; + } + case "screenshot": { + if (!value.screenshot) throw new UsageError(MISSING_VALUE.screenshot); + // Typed at a prompt, `localhost:5173` means http. Config URLs must carry + // the scheme, since there they are written once and read by others. + // Only a leading scheme counts: `localhost:5173/?next=https://x` has none + const url = /^[a-z][a-z\d+.-]*:\/\//i.test(value.screenshot) + ? value.screenshot + : `http://${value.screenshot}`; + if (!/^https?:\/\//i.test(url) || !URL.canParse(url)) { + throw new UsageError( + `--screenshot takes an http(s) URL, got "${value.screenshot}".`, + ); + } + const viewport = value.viewport ?? "desktop"; + if (!(VIEWPORTS as readonly string[]).includes(viewport)) { + throw new UsageError( + `--viewport must be "desktop" or "mobile", got "${viewport}".`, + ); + } + return { + name: "screenshot", + source: { + screenshot: { + url, + viewport: viewport as (typeof VIEWPORTS)[number], + }, + }, + }; + } + default: + return null; + } +} + +function unknownOption(option: string): UsageError { + return new UsageError( + `Unknown option: ${option}\nRun \`srcpack --help\` to see the available options.`, + ); +} + +/** + * Translate unknown options and missing values into srcpack usage errors. + * Avoid parseArgs suggestions to use `--`, which srcpack rejects. + */ +function toUsageError(error: NodeJS.ErrnoException): Error { + switch (error.code) { + case "ERR_PARSE_ARGS_UNKNOWN_OPTION": + return unknownOption( + /'([^']+)'/.exec(error.message)?.[1] ?? error.message, + ); + case "ERR_PARSE_ARGS_INVALID_OPTION_VALUE": { + // Missing (`--since`) or swallowed by the next flag (`--since -x`) + const flag = /^Option '--([\w-]+)/.exec(error.message)?.[1]; + return new UsageError( + flag && Object.hasOwn(MISSING_VALUE, flag) + ? MISSING_VALUE[flag as keyof typeof MISSING_VALUE] + : error.message, + ); + } + default: + return error; + } +} diff --git a/src/bundle.ts b/src/bundle.ts index 860f8b6..db8b9a9 100644 --- a/src/bundle.ts +++ b/src/bundle.ts @@ -11,6 +11,7 @@ import { type BundleConfigInput, type LinearSourceInput, } from "./config.ts"; +import { pathKey } from "./fs.ts"; import { isGitSource, resolveGitSource } from "./git.ts"; import { resolveLinearSource } from "./linear.ts"; @@ -330,30 +331,6 @@ function isExternalPattern(pattern: string): boolean { return normalized.startsWith("../"); } -/** - * The key two paths are compared by: canonical spelling, then case folded, on - * every platform. A case-insensitive filesystem — the default on macOS and - * Windows — treats `Context.txt` and `context.txt` as one directory entry, and - * APFS additionally folds Unicode normalisation, so `Café.txt` written as - * precomposed U+00E9 and as `e` plus U+0301 is also one entry. Normalising - * before folding is what makes the comparison sound: equal inputs stay equal - * afterwards whether or not case folding preserves normalisation. `realpath` - * resolves - * an existing component to its on-disk spelling, but that doesn't cover these: - * an output not yet written has no on-disk spelling, and the destination entry - * is deliberately left unresolved so `rename` replaces a symlink rather than - * following it. A config that works in Linux CI and loses a bundle on the - * author's laptop is worse than one rejected everywhere, so the rule is the - * same on every platform rather than keyed to the filesystem under it. - * - * Comparison only. Paths used for I/O keep their original spelling, and - * ownership stays an exact match: folding there could only widen what srcpack - * deletes, which is the one direction that must never be widened by a guess. - */ -export function pathKey(path: string): string { - return path.normalize("NFC").toLowerCase(); -} - /** * Whether a path is one srcpack writes. `outputs` holds absolute paths of * files or directories; a directory covers everything beneath it. diff --git a/src/cli.ts b/src/cli.ts index fe6005b..93675fe 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,33 +1,18 @@ #!/usr/bin/env node // SPDX-License-Identifier: MIT -import { - mkdir, - readdir, - readFile, - realpath, - rename, - rm, - writeFile, -} from "node:fs/promises"; -import { - basename, - dirname, - isAbsolute, - join, - relative, - resolve, - sep, -} from "node:path"; +import { mkdir, readdir, readFile, rm } from "node:fs/promises"; +import { basename, dirname, join, relative, resolve } from "node:path"; import ora from "ora"; -import { bundleOne, pathKey, type BundleResult } from "./bundle.ts"; +import { parseCliArgs, UsageError } from "./args.ts"; +import { bundleOne, type BundleResult } from "./bundle.ts"; import { ConfigError, loadConfig, parseConfig, - type BundleConfig, type UploadConfig, } from "./config.ts"; +import { isInside, physicalPath, writeFileAtomic } from "./fs.ts"; import { ensureAuthenticated, login, @@ -38,12 +23,14 @@ import { import { GitError } from "./git.ts"; import { runInit } from "./init.ts"; import { LinearError } from "./linear.ts"; - -interface BundleOutput { - name: string; - outfile: string; - result: BundleResult; -} +import { planOutputs, selectBundles, type ResolvedBundle } from "./plan.ts"; +import { + createCapturer, + imageFileName, + isImageOf, + ScreenshotError, + type CapturedPage, +} from "./screenshot.ts"; function sumLines(result: BundleResult): number { return result.index.reduce((sum, entry) => sum + entry.lines, 0); @@ -57,70 +44,35 @@ function plural(n: number, singular: string, pluralForm?: string): string { return n === 1 ? singular : (pluralForm ?? singular + "s"); } -/** The directory srcpack owns by convention, and the only one it clears unasked. */ -const DEFAULT_OUT_DIR = ".srcpack"; - /** - * Where a path physically is, with symlinks resolved. Destructive decisions are - * made on this rather than the lexical path: `.srcpack -> ../shared` is inside - * the project by name and somewhere else in fact, and it is the somewhere else - * whose contents `rm` would take. - * - * Resolves as much of the path as exists, however deep that is. Stopping at the - * immediate parent would call `.srcpack/nested/x.txt` and `alias/nested/x.txt` - * different files until `mkdir -p` runs, which is one step too late to still be - * a check: aliasing is a property of the ancestors, not of when they were made. + * "3 bundles, 42 files, 3,120 lines, 15 images". Files and lines appear only + * when some bundle wrote text; images aren't counted in a dry run, which never + * renders them. */ -async function physicalPath(path: string): Promise { - try { - return await realpath(path); - } catch { - const parent = dirname(path); - // dirname("/") === "/": nothing above the filesystem root left to resolve - if (parent === path) return path; - return join(await physicalPath(parent), basename(path)); +function formatCounts(outputs: ResolvedBundle[]): string { + const texts = outputs.flatMap((o) => (o.text ? [o.text] : [])); + const files = texts.reduce((sum, t) => sum + t.index.length, 0); + const lines = texts.reduce((sum, t) => sum + sumLines(t), 0); + const images = outputs.reduce( + (sum, o) => sum + (o.images?.images.length ?? 0), + 0, + ); + const counts = [`${outputs.length} ${plural(outputs.length, "bundle")}`]; + if (texts.length) { + counts.push( + `${formatNumber(files)} ${plural(files, "file")}`, + `${formatNumber(lines)} ${plural(lines, "line")}`, + ); } + if (images) counts.push(`${formatNumber(images)} ${plural(images, "image")}`); + return counts.join(", "); } -/** - * Where `rename` puts a directory entry: ancestors resolved, the entry itself - * left alone. Writing replaces the entry instead of following it, so a bundle - * whose output is a symlink is identified as the link rather than its target — - * two bundles writing over one link's target are still two separate files. - */ -async function entryPath(path: string): Promise { - return join(await physicalPath(dirname(path)), basename(path)); -} - -function isInside(path: string, dir: string): boolean { - const rel = relative(dir, path); - // Compare against ".." as a whole segment — "..cache/x" is a child, not an escape - return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel); -} - -/** - * Write a bundle by replacing the directory entry rather than the file behind - * it. Writing in place follows a symlink sitting at the output path, so - * `.srcpack/web.txt -> ~/.ssh/config` would be written through; rename replaces - * the link itself. It also makes each file appear whole or not at all. - * - * The temp name carries the pid so two runs can't rename each other's file. - */ -async function writeBundle(path: string, content: string): Promise { - const temp = `${path}.${process.pid}.tmp`; - try { - await writeFile(temp, content); - await rename(temp, path); - } finally { - // A failed write or rename would otherwise leave a partial file behind: - // stale inside outDir, and bundled by the next run beside a custom outfile. - await rm(temp, { force: true }); - } -} +/** The directory srcpack owns by convention, and the only one it clears unasked. */ +const DEFAULT_OUT_DIR = ".srcpack"; /** * Empty a directory while preserving specified entries (e.g., `.git`). - * Uses `force: true` to handle read-only or in-use files. */ async function emptyDirectory(dir: string, skip: string[] = []): Promise { let entries: string[]; @@ -144,83 +96,30 @@ async function emptyDirectory(dir: string, skip: string[] = []): Promise { } /** - * One-off bundle from a `git:` source instead of a configured one. Needs no - * config file — reviewing what you just wrote is throwaway, not worth committing. + * Write a bundle's images, then remove the stale ones: a page that shrank from + * six images to four would otherwise leave `-04` and `-05` to be attached with + * the new set. Stale files match exactly, never folded — folding could only + * widen what gets deleted (ADR 004). Returns the paths written, in order. */ -interface AdHocBundle { - name: string; - patterns: string[]; -} - -const AD_HOC_FLAGS = ["--staged", "--dirty", "--since"] as const; - -/** - * Every option the CLI accepts. Anything else is a typo, and a typo that gets - * quietly dropped is the dangerous kind: `--no-uplaod` uploads, `--dry-rnu` - * writes, `--no-emptyOutdir` empties. Same rule as the config — a token that - * changes what a run destroys or publishes is never a silent no-op. - */ -const KNOWN_FLAGS = new Set([ - ...AD_HOC_FLAGS, - "--dry-run", - "--emptyOutDir", - "--no-emptyOutDir", - "--no-upload", - "--help", - "-h", - "--version", - "-v", -]); - -function assertKnownFlags(args: string[]): void { - const unknown = args.find( - (arg) => arg.startsWith("-") && !KNOWN_FLAGS.has(arg), - ); - if (unknown) { - console.error(`Unknown option: ${unknown}`); - console.error("Run `srcpack --help` to see the available options."); - process.exit(1); - } - if (args.includes("--emptyOutDir") && args.includes("--no-emptyOutDir")) { - console.error("Cannot combine --emptyOutDir with --no-emptyOutDir."); - process.exit(1); - } -} - -function parseAdHocBundle(args: string[]): AdHocBundle | null { - const flags = AD_HOC_FLAGS.filter((flag) => args.includes(flag)); - - if (flags.length > 1) { - console.error(`Cannot combine ${flags.join(" and ")}.`); - process.exit(1); - } - - switch (flags[0]) { - case "--staged": - return { name: "staged", patterns: ["git:staged"] }; - case "--dirty": - return { name: "dirty", patterns: ["git:dirty"] }; - case "--since": { - const rev = args[args.indexOf("--since") + 1]; - if (!rev || rev.startsWith("-")) { - console.error("Missing revision: --since (e.g. --since main)"); - process.exit(1); - } - // A range pins both endpoints, so it would silently drop the uncommitted - // work --since promises. Ranges belong in a config `git:` source. - if (rev.includes("..")) { - console.error( - `--since takes a revision, not a range: "${rev}". Use a git: source in your config for ranges.`, - ); - process.exit(1); - } - // `git diff` can't see untracked files, but a new file written on this - // branch is part of "what changed since " - return { name: "since", patterns: [`git:${rev}`, "git:untracked"] }; - } - default: - return null; +async function writeImages( + name: string, + dir: string, + captured: CapturedPage, +): Promise { + await mkdir(dir, { recursive: true }); + const highest = Math.max(...captured.images.map((image) => image.index)); + const written: string[] = []; + for (const { index, data } of captured.images) { + const path = join(dir, imageFileName(name, index, highest)); + await writeFileAtomic(path, data); + written.push(path); } + const current = new Set(written.map((path) => basename(path))); + const stale = (await readdir(dir)).filter( + (file) => isImageOf(name, file) && !current.has(file), + ); + await Promise.all(stale.map((file) => rm(join(dir, file), { force: true }))); + return written; } /** Resolves to the package root from both `src/cli.ts` and `dist/cli.js`. */ @@ -245,24 +144,28 @@ async function main() { srcpack - Bundle and upload tool Usage: - npx srcpack Bundle all, upload if configured - npx srcpack web api Bundle specific bundles only - npx srcpack --staged Bundle staged changes (no config needed) - npx srcpack --dry-run Preview bundles without writing files - npx srcpack --no-upload Bundle only, skip upload - npx srcpack init Interactive config setup - npx srcpack login Authenticate with Google Drive + npx srcpack Bundle all except on-demand, upload if configured + npx srcpack web api Bundle specific bundles only + npx srcpack --staged Bundle staged changes (no config needed) + npx srcpack --screenshot localhost:5173 + Capture a page as PNGs (no config needed) + npx srcpack --dry-run Preview bundles without writing files + npx srcpack --no-upload Bundle only, skip upload + npx srcpack init Interactive config setup + npx srcpack login Authenticate with Google Drive Options: - --staged Bundle staged changes only - --dirty Bundle staged, unstaged, and untracked changes - --since Bundle changes since (e.g. --since main) - --dry-run Preview bundles without writing files - --emptyOutDir Empty output directory before writing - --no-emptyOutDir Keep existing files in output directory - --no-upload Skip uploading to cloud storage - -h, --help Show this help message - -v, --version Show version + --staged Bundle staged changes only + --dirty Bundle staged, unstaged, and untracked changes + --since Bundle changes since (e.g. --since main) + --screenshot Capture a page as numbered PNGs + --viewport desktop (default) or mobile, with --screenshot + --dry-run Preview bundles without writing files + --emptyOutDir Empty output directory before writing + --no-emptyOutDir Skip clearing the output directory + --no-upload Skip uploading to cloud storage + -h, --help Show this help message + -v, --version Show version `); return; } @@ -280,27 +183,13 @@ Options: return; } - assertKnownFlags(args); - - const dryRun = args.includes("--dry-run"); - const noUpload = args.includes("--no-upload"); - // CLI flags: --emptyOutDir forces true, --no-emptyOutDir forces false - const emptyOutDirFlag = args.includes("--emptyOutDir") - ? true - : args.includes("--no-emptyOutDir") - ? false - : undefined; - const adHoc = parseAdHocBundle(args); - const sinceIndex = args.indexOf("--since"); - const sinceValueIndex = sinceIndex === -1 ? -1 : sinceIndex + 1; - const requestedBundles = args.filter( - (arg, i) => !arg.startsWith("-") && i !== sinceValueIndex, - ); - - if (adHoc && requestedBundles.length) { - console.error(`Cannot combine --${adHoc.name} with named bundles.`); - process.exit(1); - } + const { + bundles: requestedBundles, + adHoc, + dryRun, + emptyOutDir: emptyOutDirFlag, + upload, + } = parseCliArgs(args); let config = await loadConfig(); @@ -315,23 +204,18 @@ Options: config = parseConfig({ bundles: {} }); } - const bundles = adHoc ? { [adHoc.name]: adHoc.patterns } : config.bundles; + const bundles = adHoc ? { [adHoc.name]: adHoc.source } : config.bundles; - // Determine which bundles to process - const bundleNames = requestedBundles.length - ? requestedBundles - : Object.keys(bundles); - - // Validate requested bundle names exist - for (const name of bundleNames) { - // hasOwn, not `in`: `srcpack toString` would otherwise find Object.prototype - if (!Object.hasOwn(bundles, name)) { - console.error(`Unknown bundle: ${name}`); - process.exit(1); - } - } + const { names: bundleNames, skipped } = selectBundles( + bundles, + requestedBundles, + ); + const onDemandNote = skipped.length + ? `On demand: ${skipped.join(", ")}` + : undefined; - if (bundleNames.length === 0) { + // Every bundle on demand is still a full run: it goes on to empty outDir + if (bundleNames.length === 0 && !onDemandNote) { console.log("No bundles configured."); return; } @@ -382,164 +266,190 @@ Options: ); } - // srcpack never bundles what srcpack writes. Every configured outfile is - // named explicitly; outDir covers stale bundles from renamed config entries - // too, but not when it holds the root — that would exclude the whole project. - // - // Both spellings of every output are recorded. A glob rooted at a symlink - // yields lexical paths, one rooted at the real directory yields physical - // ones, and either can name a file the previous run wrote. - const ownOutputs = new Set(); - const writers = new Map(); - for (const [name, bundleConfig] of Object.entries(config.bundles)) { - const outfile = resolve( - root, - getOutfile(bundleConfig, name, config.outDir), - ); - // Two bundles sharing one file is silent loss: the second write replaces the - // first, and the upload step then sends the survivor twice under two names. - // Keyed by destination entry — `.srcpack/a.txt` and `alias/a.txt` are two - // spellings of one file as soon as `alias` links to `.srcpack`, and so are - // `Web.txt` and `web.txt` wherever the filesystem folds case. - const entry = await entryPath(outfile); - const key = pathKey(entry); - const first = writers.get(key); - if (first) { - throw new ConfigError( - `Bundles "${first}" and "${name}" both write to "${relative(root, outfile) || outfile}". ` + - "Give one of them its own outfile.", - ); - } - writers.set(key, name); - ownOutputs.add(outfile); - ownOutputs.add(entry); - } - if (!outDirHoldsRoot) { - ownOutputs.add(outDirPath); - ownOutputs.add(outDirPhysical); - } - const outputPaths = [...ownOutputs]; + // srcpack never bundles what srcpack writes. Every output is named + // explicitly; outDir covers stale bundles from renamed config entries too, + // but not when it holds the root — that would exclude the whole project. + const { bundles: plans, ownOutputs } = await planOutputs( + root, + config.outDir, + config.bundles, + bundleNames.map((name) => [name, bundles[name]!]), + ); + if (!outDirHoldsRoot) ownOutputs.push(outDirPath, outDirPhysical); - const outputs: BundleOutput[] = []; + const outputs: ResolvedBundle[] = []; + // Printed once the spinner stops, which would otherwise draw over them + const warnings: string[] = []; + // Images stay in memory until every bundle has resolved, so a failure in + // any capture leaves the previous run's files in place (ADR 003 run order). + // A dry run previews without a browser: counting images needs a render. + const capturer = createCapturer(root); // Process all bundles with progress const bundleSpinner = ora({ - text: `Bundling ${bundleNames[0]}...`, + text: "Bundling...", color: "cyan", }).start(); try { - for (let i = 0; i < bundleNames.length; i++) { - const name = bundleNames[i]!; - bundleSpinner.text = `Bundling ${name}... (${i + 1}/${bundleNames.length})`; - const bundleConfig = bundles[name]!; - let result: BundleResult; + for (let i = 0; i < plans.length; i++) { + const plan = plans[i]!; + const progress = `${plan.name}... (${i + 1}/${plans.length})`; + const output: ResolvedBundle = { plan }; try { - result = await bundleOne(bundleConfig, root, outputPaths); + if (plan.text) { + bundleSpinner.text = `Bundling ${progress}`; + output.text = await bundleOne(plan.source, root, ownOutputs); + } + if (plan.images && !dryRun) { + bundleSpinner.text = `Capturing ${progress}`; + output.images = await capturer.capture( + plan.images.target, + (message) => warnings.push(`Bundle "${plan.name}": ${message}`), + ); + } } catch (error) { + // A dev server that isn't running fails every full run; say how to + // keep the bundle without that + if ( + error instanceof ScreenshotError && + error.unreachable && + requestedBundles.length === 0 && + !adHoc + ) { + error.message += " Set onDemand: true to capture it only when named."; + } // A config can declare many bundles; the underlying message says what // broke but not which bundle asked for it. if ( error instanceof ConfigError || error instanceof GitError || - error instanceof LinearError + error instanceof LinearError || + error instanceof ScreenshotError ) { - error.message = `Bundle "${name}": ${error.message}`; + error.message = `Bundle "${plan.name}": ${error.message}`; } throw error; } - const outfile = getOutfile(bundleConfig, name, config.outDir); - outputs.push({ name, outfile, result }); + outputs.push(output); } } finally { bundleSpinner.stop(); + await capturer.close(); + // Also when a later bundle fails: a page-growth warning may explain it + for (const warning of warnings) console.warn(warning); } - // Empty outDir only once every bundle has resolved, and only for a full run: - // a named subset can't tell what is stale, so `srcpack web` must not delete - // api.txt. Emptying earlier would destroy a good previous run whenever a - // later bundle fails — routine once a source is remote, since an expired - // token or a rate limit aborts the run after outDir is already gone. - // Resolution doesn't need the files removed first: `ownOutputs` already keeps - // srcpack's own output from being bundled. + // Clear only after all sources resolve, preserving prior output on failure. + // Named runs keep other bundles; ad-hoc runs reach this only with an explicit + // --emptyOutDir. `ownOutputs` excludes prior output during resolution. if (emptyOutDir && !dryRun && requestedBundles.length === 0) { await emptyDirectory(outDirPath, [".git"]); } + if (outputs.length === 0) { + console.log(onDemandNote); + return; + } + + const textResults = outputs.flatMap((o) => (o.text ? [o.text] : [])); + // Calculate column widths for aligned output - const maxNameLen = Math.max(...outputs.map((o) => o.name.length)); + const maxNameLen = Math.max(...outputs.map((o) => o.plan.name.length)); const maxFilesLen = Math.max( - ...outputs.map((o) => formatNumber(o.result.index.length).length), + 0, + ...textResults.map((text) => formatNumber(text.index.length).length), ); const maxLinesLen = Math.max( - ...outputs.map((o) => formatNumber(sumLines(o.result)).length), + 0, + ...textResults.map((text) => formatNumber(sumLines(text)).length), ); - // Print each bundle + // Print each bundle. A mixed bundle prints its text line, then its images. console.log(); - for (const { name, outfile, result } of outputs) { - const fileCount = result.index.length; - const lineCount = sumLines(result); - const outPath = resolve(root, outfile); - - const nameCol = name.padEnd(maxNameLen); - const filesCol = formatNumber(fileCount).padStart(maxFilesLen); - const linesCol = formatNumber(lineCount).padStart(maxLinesLen); - - if (dryRun) { - console.log( - ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")}`, - ); - for (const entry of result.index) { - console.log(` ${entry.path}`); + for (const { plan, text: result, images } of outputs) { + const nameCol = plan.name.padEnd(maxNameLen); + + if (plan.text && result) { + const fileCount = result.index.length; + const lineCount = sumLines(result); + const outPath = plan.text.outfile; + const filesCol = formatNumber(fileCount).padStart(maxFilesLen); + const linesCol = formatNumber(lineCount).padStart(maxLinesLen); + + if (dryRun) { + console.log( + ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")}`, + ); + for (const entry of result.index) { + console.log(` ${entry.path}`); + } + } else if (fileCount === 0) { + // Remove stale text only inside outDir; custom outfiles outside it survive. + if (isInside(outPath, outDirPath)) { + await rm(outPath, { force: true }); + } + console.log( + ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")} → skipped`, + ); + } else { + await mkdir(dirname(outPath), { recursive: true }); + await writeFileAtomic(outPath, result.content); + const displayPath = relative(process.cwd(), outPath); + console.log( + ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")} → ${displayPath}`, + ); } - } else if (fileCount === 0) { - // Drop a previous run's file so the bundle never goes stale, but only - // inside outDir — a custom outfile points at a location srcpack doesn't own - if (isInside(outPath, outDirPath)) { - await rm(outPath, { force: true }); + } + + if (plan.images) { + const { target, dir } = plan.images; + if (!images) { + const pattern = join(dir, `${plan.name}-NN.png`); + console.log( + ` ${nameCol} screenshot ${target.url} ${target.viewport} → ${relative(process.cwd(), pattern)}`, + ); + } else { + const written = await writeImages(plan.name, dir, images); + const first = relative(process.cwd(), written[0]!); + const range = + written.length === 1 + ? first + : `${first} … ${basename(written.at(-1)!)}`; + console.log( + ` ${nameCol} ${formatNumber(written.length)} ${plural(written.length, "image")} page ${images.width}×${formatNumber(images.height)} → ${range}`, + ); } - console.log( - ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")} → skipped`, - ); - } else { - await mkdir(dirname(outPath), { recursive: true }); - await writeBundle(outPath, result.content); - const displayPath = relative(process.cwd(), outPath); - console.log( - ` ${nameCol} ${filesCol} ${plural(fileCount, "file")} ${linesCol} ${plural(lineCount, "line")} → ${displayPath}`, - ); } } - // Print summary - const totalFiles = outputs.reduce((sum, o) => sum + o.result.index.length, 0); - const totalLines = outputs.reduce((sum, o) => sum + sumLines(o.result), 0); - const bundleWord = plural(outputs.length, "bundle"); - const fileWord = plural(totalFiles, "file"); - const lineWord = plural(totalLines, "line"); - console.log(); if (dryRun) { - console.log( - `Dry run: ${outputs.length} ${bundleWord}, ${formatNumber(totalFiles)} ${fileWord}, ${formatNumber(totalLines)} ${lineWord}`, - ); + console.log(`Dry run: ${formatCounts(outputs)}`); + if (onDemandNote) console.log(onDemandNote); } else { - console.log( - `Bundled: ${outputs.length} ${bundleWord}, ${formatNumber(totalFiles)} ${fileWord}, ${formatNumber(totalLines)} ${lineWord}`, - ); + console.log(`Bundled: ${formatCounts(outputs)}`); + if (onDemandNote) console.log(onDemandNote); // Ad-hoc bundles stay local: uploading work-in-progress to Drive is not // what --staged asks for, and `upload.exclude` can't name a bundle the // config doesn't declare. Configure a named bundle to publish changes. - if (config.upload && !noUpload && !adHoc) { + if (config.upload && upload && !adHoc) { const uploads = Array.isArray(config.upload) ? config.upload : [config.upload]; + // Drive finds files by name and updates them in place, so a page that + // shrank would leave its old slices there with nothing to remove them. + // A mixed bundle's text file still uploads. + const local = outputs.filter((o) => o.images).map((o) => o.plan.name); + if (local.length && uploads.some(isGdriveConfigured)) { + console.log(`Images stay local: ${local.join(", ")}`); + } + for (const uploadConfig of uploads) { if (isGdriveConfigured(uploadConfig)) { - await handleGdriveUpload(uploadConfig, outputs, root); + await handleGdriveUpload(uploadConfig, outputs); } } } @@ -619,13 +529,14 @@ function printUploadConfigHelp(): void { async function handleGdriveUpload( uploadConfig: UploadConfig, - outputs: BundleOutput[], - root: string, + outputs: ResolvedBundle[], ): Promise { // Filter out excluded bundles and empty ones (never written to disk) const excludeSet = new Set(uploadConfig.exclude ?? []); - const toUpload = outputs.filter( - (o) => !excludeSet.has(o.name) && o.result.index.length > 0, + const toUpload = outputs.flatMap(({ plan, text }) => + plan.text && text && text.index.length > 0 && !excludeSet.has(plan.name) + ? [{ name: plan.name, outfile: plan.text.outfile }] + : [], ); if (toUpload.length === 0) { @@ -645,10 +556,9 @@ async function handleGdriveUpload( try { for (let i = 0; i < toUpload.length; i++) { - const output = toUpload[i]!; - const filePath = resolve(root, output.outfile); - uploadSpinner.text = `Uploading ${output.name}... (${i + 1}/${toUpload.length})`; - const result = await uploadFile(filePath, uploadConfig); + const { name, outfile } = toUpload[i]!; + uploadSpinner.text = `Uploading ${name}... (${i + 1}/${toUpload.length})`; + const result = await uploadFile(outfile, uploadConfig); results.push(result); } } finally { @@ -681,27 +591,15 @@ async function handleGdriveUpload( } } -function getOutfile( - bundleConfig: BundleConfig, - name: string, - outDir: string, -): string { - if ( - typeof bundleConfig === "object" && - !Array.isArray(bundleConfig) && - bundleConfig.outfile - ) { - return bundleConfig.outfile; - } - return join(outDir, `${name}.txt`); -} - main().catch((err) => { - // Config, git and Linear failures are user-facing; a stack trace adds noise + // Usage, config, git, Linear and screenshot failures are user-facing; a stack + // trace adds noise console.error( - err instanceof ConfigError || + err instanceof UsageError || + err instanceof ConfigError || err instanceof GitError || - err instanceof LinearError + err instanceof LinearError || + err instanceof ScreenshotError ? err.message : err, ); diff --git a/src/config.ts b/src/config.ts index 36b856f..08a4e66 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,6 +77,45 @@ const LinearSourceSchema = z.union([ }), ]); +/** + * An absolute http(s) URL. A scheme-less `localhost:5173` is called out by + * name: `URL` would otherwise parse it as protocol `localhost:`. + */ +const HttpUrlSchema = z.string().superRefine((value, ctx) => { + if (!value.includes("://")) { + ctx.addIssue({ + code: "custom", + message: 'URL needs a scheme, e.g. "http://localhost:5173/"', + }); + } else if (!/^https?:\/\//i.test(value) || !URL.canParse(value)) { + ctx.addIssue({ code: "custom", message: "Expected an http(s) URL" }); + } +}); + +/** + * A rendered page as numbered PNGs in `outDir`, independent of the text + * output path. See ADR 006 for why it is a separate source key. + * + * @example + * ```ts + * bundles: { + * home: { screenshot: "http://localhost:5173/", onDemand: true }, + * phone: { screenshot: { url: "http://localhost:5173/", viewport: "mobile" } }, + * } + * ``` + */ +const ScreenshotSourceSchema = z.union([ + /** Shorthand for `{ url: "" }`. */ + HttpUrlSchema, + z.strictObject({ + url: HttpUrlSchema, + /** Defaults to "desktop". */ + viewport: z.enum(["desktop", "mobile"]).optional(), + /** CSS selectors hidden during capture: cookie banners, chat widgets. */ + hide: z.array(z.string().min(1)).optional(), + }), +]); + /** * Bundle configuration. Accepts a string pattern, array of patterns, or object. * Patterns prefixed with `!` are exclusions. Patterns prefixed with `+` force @@ -86,7 +125,8 @@ const LinearSourceSchema = z.union([ * `git:unstaged`, `git:untracked`, `git:dirty`, or `git:` (e.g. * `git:main`, `git:HEAD~3`). * - * The object form takes files (`include`), Linear issues (`linear`), or both. + * The object form takes files (`include`), Linear issues (`linear`), page + * screenshots (`screenshot`), or any combination. * * @example * ```ts @@ -105,15 +145,46 @@ const BundleConfigSchema = z.union([ include: PatternsSchema.optional(), /** Linear issues to include in the bundle. */ linear: LinearSourceSchema.optional(), + /** Page to capture as numbered PNGs in `outDir`. */ + screenshot: ScreenshotSourceSchema.optional(), /** Custom output file path. Defaults to `/.txt`. */ outfile: z.string().min(1).optional(), - /** Include file index header in output. Defaults to true. */ - index: z.boolean().default(true), + /** + * Include file index header in output. Defaults to true — read as + * `index ?? true`, so screenshot-only bundles can reject an explicit value. + */ + index: z.boolean().optional(), /** Text to prepend to bundle (e.g., review instructions for LLMs). */ prompt: z.string().optional(), + /** + * Skipped by a full run, built when named: `srcpack `. For bundles + * too slow, remote or situational to rebuild every time. A full run that + * empties `outDir` still removes their output there — preserving it + * would make emptying a growing list of exceptions (ADR 005). + */ + onDemand: z.boolean().optional(), }) - .refine((bundle) => bundle.include || bundle.linear, { - message: 'Bundle needs a source: "include" patterns, "linear", or both', + .superRefine((bundle, ctx) => { + if (bundle.include || bundle.linear) return; + if (!bundle.screenshot) { + ctx.addIssue({ + code: "custom", + message: + 'Bundle needs a source: "include" patterns, "linear", or "screenshot"', + }); + return; + } + // These shape a text file that a screenshot-only bundle never writes, so + // setting one is a misunderstanding rather than a harmless no-op + for (const key of ["prompt", "index", "outfile"] as const) { + if (bundle[key] !== undefined) { + ctx.addIssue({ + code: "custom", + path: [key], + message: `"${key}" applies to the text file, and a screenshot-only bundle writes none. Add "include" or "linear", or remove it`, + }); + } + } }), ]); @@ -164,10 +235,8 @@ const ConfigSchema = z bundles: z.record(BundleNameSchema, BundleConfigSchema), }) .superRefine((config, ctx) => { - // `upload.exclude` is the only thing keeping a bundle off Google Drive, so a - // name that matches nothing uploads the bundle it was meant to hold back — - // the one failure mode where a typo is worse than a missing line. A stale - // entry left over from a deleted bundle is cheap to fix by comparison. + // Reject misspelled exclusions so a bundle intended to stay local cannot + // silently upload. Removed bundles must also be removed from this list. const uploads = config.upload ? Array.isArray(config.upload) ? config.upload @@ -193,6 +262,7 @@ const ConfigSchema = z export type UploadConfig = z.infer; export type LinearSourceInput = z.input; +export type ScreenshotSource = z.infer; export type BundleConfig = z.infer; export type BundleConfigInput = z.input; export type Config = z.infer; diff --git a/src/fs.ts b/src/fs.ts new file mode 100644 index 0000000..8a7b7b7 --- /dev/null +++ b/src/fs.ts @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT + +import { realpath, rename, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, sep } from "node:path"; + +/** + * Compare paths using Unicode NFC followed by lowercase, on every platform. + * Normalizing first gives equivalent spellings the same input to lowercasing. + * This catches case and normalization collisions even for unwritten outputs + * and unresolved final entries, where realpath cannot canonicalize spelling. + * Applying one rule everywhere prevents a config from passing on Linux and + * overwriting a bundle on a case-insensitive filesystem (ADR 004). + * + * Comparison only: I/O retains the original spelling. Ownership and deletion + * use exact matches so folding cannot widen what srcpack removes. + */ +export function pathKey(path: string): string { + return path.normalize("NFC").toLowerCase(); +} + +/** + * Where a path physically is, with symlinks resolved. Destructive decisions are + * made on this rather than the lexical path: `.srcpack -> ../shared` is inside + * the project by name and somewhere else in fact, and it is the somewhere else + * whose contents `rm` would take. + * + * Resolves as much of the path as exists, however deep that is. Stopping at the + * immediate parent would call `.srcpack/nested/x.txt` and `alias/nested/x.txt` + * different files until `mkdir -p` runs, which is one step too late to still be + * a check: aliasing is a property of the ancestors, not of when they were made. + */ +export async function physicalPath(path: string): Promise { + try { + return await realpath(path); + } catch { + const parent = dirname(path); + // dirname("/") === "/": nothing above the filesystem root left to resolve + if (parent === path) return path; + return join(await physicalPath(parent), basename(path)); + } +} + +/** + * Where `rename` puts a directory entry: ancestors resolved, the entry itself + * left alone. Writing replaces the entry instead of following it, so a bundle + * whose output is a symlink is identified as the link rather than its target — + * writing to the link path and to its target produces two separate files. + */ +export async function entryPath(path: string): Promise { + return join(await physicalPath(dirname(path)), basename(path)); +} + +export function isInside(path: string, dir: string): boolean { + const rel = relative(dir, path); + // Compare against ".." as a whole segment — "..cache/x" is a child, not an escape + return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel); +} + +/** + * Write a file by replacing the directory entry rather than the file behind + * it. Writing in place follows a symlink sitting at the output path, so + * `.srcpack/web.txt -> ~/.ssh/config` would be written through; rename replaces + * the link itself. It also makes each file appear whole or not at all. + * + * The temp name carries the pid so two runs can't rename each other's file. + */ +export async function writeFileAtomic( + path: string, + data: string | Uint8Array, +): Promise { + const temp = `${path}.${process.pid}.tmp`; + try { + await writeFile(temp, data); + await rename(temp, path); + } finally { + // A failed write or rename would otherwise leave a partial file behind: + // stale inside outDir, and bundled by the next run beside a custom outfile. + await rm(temp, { force: true }); + } +} diff --git a/src/plan.ts b/src/plan.ts new file mode 100644 index 0000000..62aac1f --- /dev/null +++ b/src/plan.ts @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: MIT + +import { basename, dirname, join, relative, resolve } from "node:path"; +import type { BundleResult } from "./bundle.ts"; +import { ConfigError, type BundleConfig } from "./config.ts"; +import { entryPath, pathKey, physicalPath } from "./fs.ts"; +import { + isImageOf, + toScreenshotTarget, + type CapturedPage, + type ScreenshotTarget, +} from "./screenshot.ts"; + +// Check output destinations before resolving sources or launching a browser. +// Retain each active plan through writing, reporting and upload. + +/** Where a bundle writes. */ +export interface PlannedBundle { + name: string; + source: BundleConfig; + /** Absolute path of the text output. Absent for a screenshot-only bundle. */ + text?: { outfile: string }; + /** Page to capture into `/-NN.png`. */ + images?: { target: ScreenshotTarget; dir: string }; +} + +/** What a bundle produced, ready to write. */ +export interface ResolvedBundle { + plan: PlannedBundle; + text?: BundleResult; + images?: CapturedPage; +} + +export interface BundleSelection { + /** Bundles this run builds, in the order named or configured. */ + names: string[]; + /** On-demand bundles a full run left out. Empty when bundles are named. */ + skipped: string[]; +} + +function isOnDemand(config: BundleConfig): boolean { + return ( + typeof config === "object" && + !Array.isArray(config) && + config.onDemand === true + ); +} + +/** + * Pick the bundles a run builds. Naming a bundle always builds it; a full run + * builds everything not marked `onDemand`. + */ +export function selectBundles( + bundles: Record, + requested: string[], +): BundleSelection { + for (const name of requested) { + // hasOwn, not `in`: `srcpack toString` would otherwise find Object.prototype + if (!Object.hasOwn(bundles, name)) { + throw new ConfigError(`Unknown bundle: ${name}`); + } + } + if (requested.length) { + return { names: [...new Set(requested)], skipped: [] }; + } + + const names: string[] = []; + const skipped: string[] = []; + for (const [name, config] of Object.entries(bundles)) { + (isOnDemand(config) ? skipped : names).push(name); + } + return { names, skipped }; +} + +export interface OutputPlan { + /** The bundles this run builds, in order. */ + bundles: PlannedBundle[]; + /** + * Absolute paths srcpack writes, never to be bundled: every configured text + * output and every active one, each in lexical and entry spelling. A glob + * rooted at a symlink yields lexical paths, one rooted at the real directory + * yields physical ones, and either can name a file the previous run wrote. + * + * Image families need no entry: they always live in outDir, which the CLI + * excludes, and when outDir holds the root a PNG is skipped as binary anyway. + */ + ownOutputs: string[]; +} + +function planBundle( + name: string, + source: BundleConfig, + root: string, + outDir: string, +): PlannedBundle { + const object = + typeof source === "object" && !Array.isArray(source) ? source : undefined; + const plan: PlannedBundle = { name, source }; + // Every bundle writes text unless a screenshot is all it declares + if (!object || object.include || object.linear) { + const outfile = object?.outfile ?? join(outDir, `${name}.txt`); + plan.text = { outfile: resolve(root, outfile) }; + } + // Keep image cleanup within outDir, independent of any text outfile. + if (object?.screenshot) { + plan.images = { + target: toScreenshotTarget(object.screenshot), + dir: resolve(root, outDir), + }; + } + return plan; +} + +/** + * A claim on a destination: a text file, or an image family — a directory + * plus a name prefix. Compared folded (`pathKey`) and physically. + * + * A text file's directory is its entry path's parent, as for any output. A + * family's directory resolves fully: it is an ancestor of every PNG, and + * `rename` replaces only the last component (ADR 004). + */ +interface Claim { + owner: string; + kind: "file" | "family"; + dir: string; + /** Folded file name, or folded bundle name for a family. */ + name: string; + /** Absolute path shown in a collision message. */ + display: string; +} + +function overlaps(a: Claim, b: Claim): boolean { + if (a.dir !== b.dir) return false; + if (a.kind === b.kind) return a.name === b.name; + const [file, family] = a.kind === "file" ? [a, b] : [b, a]; + return isImageOf(family.name, file.name); +} + +/** `entry` is the text output's entry path, resolved once by the caller. */ +async function claimsOf( + plan: PlannedBundle, + entry: string | undefined, +): Promise { + const claims: Claim[] = []; + if (plan.text && entry) { + claims.push({ + owner: plan.name, + kind: "file", + dir: pathKey(dirname(entry)), + name: pathKey(basename(entry)), + display: plan.text.outfile, + }); + } + if (plan.images) { + claims.push({ + owner: plan.name, + kind: "family", + dir: pathKey(await physicalPath(plan.images.dir)), + name: pathKey(plan.name), + display: join(plan.images.dir, `${plan.name}-NN.png`), + }); + } + return claims; +} + +function collision(first: Claim, second: Claim, root: string): ConfigError { + // Name the later bundle's destination, unless only the earlier one is a file: + // a file is what the user can change + const { display } = + first.kind === "file" && second.kind === "family" ? first : second; + const where = relative(root, display) || display; + if (first.owner === second.owner) { + return new ConfigError( + `Bundle "${first.owner}" writes its text and its images to "${where}". Give it another outfile.`, + ); + } + return new ConfigError( + `Bundles "${first.owner}" and "${second.owner}" both write to "${where}". ` + + (first.kind === "family" && second.kind === "family" + ? "Rename one of them." + : "Give one of them its own outfile."), + ); +} + +/** + * Derive where every bundle writes and reject two bundles sharing a file. + * + * Resolve directory aliases and compare destinations with `pathKey` to prevent + * silent overwrites. Text files also collide with numbered images in the same + * directory, including their own bundle's images. + * + * Configured bundles are checked against each other on every run, so a config + * error doesn't depend on what was asked for. Active bundles — the ones `active` + * names, ad-hoc included — are checked against configured bundles of a + * different name: an ad-hoc bundle shadows a configured bundle of its own name + * (`--staged` over a `staged` bundle) but never overwrites another's output. + * Active bundles need no check among themselves: they are either configured, + * and already checked, or a single ad-hoc bundle. + */ +export async function planOutputs( + root: string, + outDir: string, + configured: Record, + active: [name: string, source: BundleConfig][], +): Promise { + const ownOutputs = new Set(); + const claims: Claim[] = []; + + const claim = async (plan: PlannedBundle, shadowsOwnName: boolean) => { + const entry = plan.text && (await entryPath(plan.text.outfile)); + for (const next of await claimsOf(plan, entry)) { + const taken = claims.find( + (prior) => + !(shadowsOwnName && prior.owner === next.owner) && + overlaps(prior, next), + ); + if (taken) throw collision(taken, next, root); + claims.push(next); + } + if (plan.text && entry) { + ownOutputs.add(plan.text.outfile); + ownOutputs.add(entry); + } + }; + + for (const [name, source] of Object.entries(configured)) { + await claim(planBundle(name, source, root, outDir), false); + } + + const bundles: PlannedBundle[] = []; + for (const [name, source] of active) { + const plan = planBundle(name, source, root, outDir); + await claim(plan, true); + bundles.push(plan); + } + + return { bundles, ownOutputs: [...ownOutputs] }; +} diff --git a/src/screenshot.ts b/src/screenshot.ts new file mode 100644 index 0000000..62408d4 --- /dev/null +++ b/src/screenshot.ts @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: MIT + +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import type * as PlaywrightModule from "playwright"; +import type { ScreenshotSource } from "./config.ts"; + +/** A capture failure worth a clean message: no Playwright, no page, no browser. */ +export class ScreenshotError extends Error { + /** Nothing answered at the URL — most often a dev server that isn't running. */ + readonly unreachable: boolean; + + constructor(message: string, options: { unreachable?: boolean } = {}) { + super(message); + this.name = "ScreenshotError"; + this.unreachable = options.unreachable ?? false; + } +} + +export type Viewport = "desktop" | "mobile"; + +/** A screenshot source with its defaults applied. */ +export interface ScreenshotTarget { + url: string; + viewport: Viewport; + hide: string[]; +} + +export function toScreenshotTarget(source: ScreenshotSource): ScreenshotTarget { + return typeof source === "string" + ? { url: source, viewport: "desktop", hide: [] } + : { + url: source.url, + viewport: source.viewport ?? "desktop", + hide: source.hide ?? [], + }; +} + +// Vision models downscale each image to a fixed pixel budget, so a tall page +// captured whole arrives as a thumbnail. Detail slices at most 2,200 device px +// tall stay legible after that downscaling. Overlap preserves text across slice +// boundaries. Device px keep the image height consistent across DPRs: at DPR 2, +// each slice covers half as many CSS pixels. +const SLICE_HEIGHT = 2200; +const SLICE_OVERLAP = 160; + +/** A vertical region of the page, in CSS px. */ +export interface Slice { + y: number; + height: number; +} + +/** + * Split a page into the fewest overlapping slices that cover it top to bottom, + * each overlapping the next by at least `SLICE_OVERLAP` device px. + * + * Slices then shrink to share the page evenly rather than staying at the + * maximum: a page one pixel taller than a slice becomes two half-page slices, + * not two near-identical full ones that spend a model's attention twice. + */ +export function planSlices(cssHeight: number, dpr: number): Slice[] { + const maxHeight = SLICE_HEIGHT / dpr; + const overlap = SLICE_OVERLAP / dpr; + if (cssHeight <= maxHeight) return [{ y: 0, height: cssHeight }]; + + const count = Math.ceil((cssHeight - overlap) / (maxHeight - overlap)); + // Exactly `overlap` between neighbours at this height; rounded up to whole + // CSS px, which only adds overlap, and never past `maxHeight` given `count` + const height = Math.ceil((cssHeight + (count - 1) * overlap) / count); + const step = (cssHeight - height) / (count - 1); + // Starts are floored, which never widens a gap past `height - overlap`. The + // last start is set, not computed: `i * step` can round below the exact + // value and leave the bottom row uncovered. + return Array.from({ length: count }, (_, i) => ({ + y: i === count - 1 ? cssHeight - height : Math.floor(i * step), + height, + })); +} + +/** + * `home-00.png`, `home-01.png`, … Index 0 is the whole page, so filename + * order is review order. Padding grows with the highest index so the order + * holds past 99. + */ +export function imageFileName( + name: string, + index: number, + highestIndex: number, +): string { + const width = Math.max(2, String(highestIndex).length); + return `${name}-${String(index).padStart(width, "0")}.png`; +} + +/** + * Whether `file` is one of bundle `name`'s numbered images. The suffix is + * digits only, so `home-01-02.png` (bundle `home-01`) is never one of `home`'s. + * + * An exact match, because it decides what stale-image cleanup deletes and + * folding could only widen that (ADR 004). To ask whether two spellings + * collide, pass `pathKey`s. + */ +export function isImageOf(name: string, file: string): boolean { + return ( + file.startsWith(`${name}-`) && + /^\d{2,}\.png$/.test(file.slice(name.length + 1)) + ); +} + +export type Playwright = Pick; + +/** Loads a module by specifier, throwing `MODULE_NOT_FOUND` when absent. */ +export type Require = (id: string) => unknown; + +/** + * Where Playwright may be found, most specific first. The project comes before + * srcpack: under `npx srcpack`, srcpack runs from the npm cache, where a bare + * `import("playwright")` never sees the project's copy. `@playwright/test` + * re-exports `chromium` and `devices`, so a project with Playwright Test needs + * nothing new. + */ +function candidates(root: string): [Require, string][] { + const project = createRequire(join(root, "package.json")); + const own = createRequire(import.meta.url); + return [ + [project, "playwright"], + [project, "@playwright/test"], + [own, "playwright"], + ]; +} + +// 1.41 added the screenshot `style` option, which hides elements without +// mutating the page. Checked here rather than in the peer range, which is `*` +// so installs that never capture aren't judged by their Playwright version. +const MIN_MINOR = 41; + +/** + * Load Playwright without bundling it. `require` through `createRequire` keeps + * `bun build` from inlining an optional peer that most users never install. + */ +export function loadPlaywright( + root: string, + from: [Require, string][] = candidates(root), + userAgent = process.env.npm_config_user_agent, +): Playwright { + for (const [load, id] of from) { + let version: string; + try { + ({ version } = load(`${id}/package.json`) as { version: string }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "MODULE_NOT_FOUND") { + continue; + } + throw error; + } + const [major, minor] = version.split(".").map(Number); + if (major !== 1 || minor! < MIN_MINOR) { + throw new ScreenshotError( + major === 1 + ? `Playwright ${version} is too old; srcpack needs 1.${MIN_MINOR} or newer.` + : `Playwright ${version} is not supported; srcpack needs 1.${MIN_MINOR} or a later 1.x.`, + ); + } + return load(id) as Playwright; + } + const pm = packageManager(userAgent); + throw new ScreenshotError( + `screenshots need Playwright. Install it with:\n ${pm.add} playwright && ${pm.exec} playwright install chromium`, + ); +} + +/** + * Launch Playwright's Chromium, falling back to system Chrome only when the + * executable is missing: the browser download is where this flow loses people, + * and most developers already have Chrome. Other launch errors stay visible. + */ +export async function launchBrowser( + playwright: Playwright, + userAgent = process.env.npm_config_user_agent, +): Promise { + try { + return await playwright.chromium.launch(); + } catch (error) { + if (!isMissingBrowser(error)) throw error; + } + try { + return await playwright.chromium.launch({ channel: "chrome" }); + } catch (error) { + // Chrome that exists but fails to start has a real reason worth showing + if (!isMissingBrowser(error)) throw error; + const pm = packageManager(userAgent); + throw new ScreenshotError( + `screenshots need a browser. Install Chromium with:\n ${pm.exec} playwright install chromium`, + ); + } +} + +/** Playwright's wording for a bundled build, then for a system channel. */ +function isMissingBrowser(error: unknown): boolean { + return /Executable doesn't exist|distribution '[^']+' is not found/.test( + (error as Error).message, + ); +} + +/** What a capture produced. Dimensions are the page's, in CSS px. */ +export interface CapturedPage { + width: number; + height: number; + /** + * Index 0 is the whole page, 1… the detail slices top to bottom. An index is + * carried rather than implied by position, so an omitted overview leaves + * `01…` in place instead of renumbering a detail slice into `00`. + */ + images: { index: number; data: Uint8Array }[]; +} + +/** One browser for a whole run, launched when the first capture needs it. */ +export interface Capturer { + capture( + target: ScreenshotTarget, + warn: (message: string) => void, + ): Promise; + /** Close the browser, if one was launched. Never throws. */ + close(): Promise; +} + +// Standalone Playwright has no test deadline to fall back on, so every +// navigation and capture gets an explicit one. +const TIMEOUT = 30_000; + +// Settling scrolls one viewport per step, dwelling so observers fire and +// images start loading at each position, and waits for the network at each +// apparent bottom. The step cap bounds infinite scroll; the deadline bounds +// the whole walk, so a page that keeps appending can't turn every step into a +// network wait. +const SETTLE_STEPS = 50; +const SETTLE_DWELL = 100; +const SETTLE_DEADLINE = 15_000; + +// The network counts as idle after 500 ms without a request in flight — the +// same window as Playwright's `networkidle`. Capped, since uncapped it hangs on +// beacons and long polling. +const NETWORK_QUIET = 500; +const NETWORK_IDLE = 5_000; + +// OpenAI's documented per-image upload limit +const CHATGPT_MAX_IMAGE_BYTES = 20 * 1024 * 1024; + +// Framework dev chrome that would otherwise be reviewed as part of the design. +// Not `nextjs-portal`: it also hosts Next's build and runtime error overlay, +// and a clean page over a broken app is worse evidence than a noisy one. +const DEV_OVERLAYS = ["astro-dev-toolbar", "nuxt-devtools-container"]; + +/** `net::` codes that mean nothing is listening, not that the page is broken. */ +const UNREACHABLE: Record = { + ERR_CONNECTION_REFUSED: "connection refused", + ERR_CONNECTION_RESET: "connection reset", + ERR_CONNECTION_TIMED_OUT: "connection timed out", + ERR_ADDRESS_UNREACHABLE: "address unreachable", + ERR_NAME_NOT_RESOLVED: "host not found", +}; + +export function createCapturer( + root: string, + load: () => Playwright = () => loadPlaywright(root), +): Capturer { + let session: + | Promise<{ playwright: Playwright; browser: PlaywrightModule.Browser }> + | undefined; + + return { + async capture(target, warn) { + session ??= (async () => { + const playwright = load(); + return { playwright, browser: await launchBrowser(playwright) }; + })(); + const { playwright, browser } = await session; + const context = await browser.newContext( + contextOptions(playwright, target.viewport), + ); + try { + return await capturePage(await context.newPage(), target, warn); + } finally { + await context.close(); + } + }, + async close() { + // A launch that failed has nothing to close, and its error is reported + const opened = await session?.catch(() => undefined); + await opened?.browser.close().catch(() => {}); + }, + }; +} + +/** + * Desktop is 1440×900 at DPR 1. Mobile starts from Playwright's `Pixel 7` + * profile — a mobile user agent matters, since server-rendered responsive + * sites send desktop markup to a desktop one — with DPR normalized from 2.625 + * to 2, so detail slices are 824 px wide: a text budget comparable to desktop, + * deliberately not exact emulation. + */ +function contextOptions( + playwright: Playwright, + viewport: Viewport, +): PlaywrightModule.BrowserContextOptions { + return viewport === "mobile" + ? { ...playwright.devices["Pixel 7"], deviceScaleFactor: 2 } + : { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }; +} + +async function capturePage( + page: PlaywrightModule.Page, + target: ScreenshotTarget, + warn: (message: string) => void, +): Promise { + page.setDefaultTimeout(TIMEOUT); + const network = watchNetwork(page); + await open(page, target.url); + await settle(page, network, warn); + + // In-page code is passed as strings: srcpack is typed without DOM globals. + // + // Width is the layout viewport — 980 px for a page without + // `` under mobile emulation, as on a real phone — not + // the scroll width: accidental horizontal overflow would otherwise widen + // every slice past the text budget. Height is what window scrolling reaches, + // the same element `settle` scrolled; an app that scrolls inside its own + // container has nothing further down to slice. + const { width, height, dpr } = await page.evaluate<{ + width: number; + height: number; + dpr: number; + }>(`({ + width: document.documentElement.clientWidth, + height: (document.scrollingElement ?? document.documentElement).scrollHeight, + dpr: devicePixelRatio, + })`); + + // Hidden through the capture's own stylesheet, so the page isn't mutated + const options = { + animations: "disabled", + style: [...DEV_OVERLAYS, ...target.hide] + .map((selector) => `${selector} { visibility: hidden !important; }`) + .join("\n"), + } as const; + + const slices = planSlices(height, dpr); + const images: CapturedPage["images"] = []; + for (const [i, { y, height: sliceHeight }] of slices.entries()) { + const data = await page.screenshot({ + ...options, + fullPage: true, + clip: { x: 0, y, width, height: sliceHeight }, + scale: "device", + }); + // A page that fits in one slice is its own overview + images.push({ index: slices.length === 1 ? 0 : i + 1, data }); + } + + if (slices.length > 1) { + const overview = await captureOverview(page, options, warn); + if (overview) images.unshift({ index: 0, data: overview }); + } + + return { width, height, images }; +} + +/** + * Navigate, failing on anything but a successful page. A screenshot of a 404 + * is wrong output that looks right. No separate `fetch` first: it doubles the + * request and can disagree with the browser about redirects and headers. + */ +async function open(page: PlaywrightModule.Page, url: string): Promise { + let response; + try { + response = await page.goto(url, { waitUntil: "load" }); + } catch (error) { + const { name, message } = error as Error; + const code = /net::(ERR_[A-Z_]+)/.exec(message)?.[1]; + if (code && Object.hasOwn(UNREACHABLE, code)) { + throw new ScreenshotError( + `${url} is not reachable (${UNREACHABLE[code]}). Is your dev server running?`, + { unreachable: true }, + ); + } + throw new ScreenshotError( + name === "TimeoutError" + ? `${url} did not finish loading within ${TIMEOUT / 1000} s.` + : `${url} failed to load (${code ?? message.split("\n")[0]}).`, + ); + } + if (response && !response.ok()) { + const status = [response.status(), response.statusText()].join(" "); + throw new ScreenshotError(`${url} returned ${status.trim()}.`); + } +} + +/** + * Let a page finish rendering before capture. A full-page screenshot doesn't + * move the viewport, so lazy images and IntersectionObserver content would + * stay blank. + * + * Scrolls until the viewport can go no further, re-reading the page each step + * so sections that load taller than their placeholders are followed. At that + * apparent bottom it waits for the network — covering late page data and what + * scrolling started — then tries again: a response that appends content has + * to be scrolled through too, or its own lazy content is sliced but blank. + * The bottom is stable once a wait there brings no growth; only then does it + * return to the top. No explicit font wait: Playwright's screenshot already + * awaits `document.fonts.ready`. + */ +async function settle( + page: PlaywrightModule.Page, + network: NetworkWatch, + warn: (message: string) => void, +): Promise { + const deadline = Date.now() + SETTLE_DEADLINE; + let steps = 0; + // Whether the network has gone quiet since the last scroll + let waitedHere = false; + for (;;) { + // `instant` overrides CSS `scroll-behavior: smooth`, which would leave + // `scrollY` unchanged when read back and end the walk early + const moved = await page.evaluate(`(() => { + const before = scrollY; + scrollTo({ top: before + innerHeight, behavior: "instant" }); + return scrollY !== before; + })()`); + if (moved) { + waitedHere = false; + if (++steps >= SETTLE_STEPS || Date.now() >= deadline) { + const reached = await page.evaluate("scrollY + innerHeight"); + warn( + `page kept growing while scrolling; content below ${reached.toLocaleString("en-US")} px may not have loaded.`, + ); + break; + } + await delay(SETTLE_DWELL); + } else if (waitedHere) { + break; + } else { + await network.idle(deadline - Date.now()); + waitedHere = true; + } + } + + await page.evaluate(`scrollTo({ top: 0, behavior: "instant" })`); +} + +export interface NetworkWatch { + /** + * Resolves after the quiet window with nothing in flight, or after `limit` + * ms — never longer than the cap. + */ + idle(limit?: number): Promise; +} + +/** The part of a Playwright page that reports requests. */ +export interface RequestEvents { + on( + event: "request" | "requestfinished" | "requestfailed", + listener: (request: unknown) => void, + ): unknown; +} + +/** + * Track requests from before navigation onward. Not Playwright's `networkidle` + * load state: once reached it resolves immediately, so waiting for it after + * scrolling misses exactly the requests scrolling started. + * + * Quiet is measured from the last request event, not sampled: a request that + * starts and finishes between two polls still restarts the window. + */ +export function watchNetwork( + page: RequestEvents, + { quiet = NETWORK_QUIET, cap = NETWORK_IDLE } = {}, +): NetworkWatch { + const inflight = new Set(); + let lastActivity = Date.now(); + page.on("request", (request) => { + inflight.add(request); + lastActivity = Date.now(); + }); + const settled = (request: unknown) => { + inflight.delete(request); + lastActivity = Date.now(); + }; + page.on("requestfinished", settled); + page.on("requestfailed", settled); + + return { + async idle(limit = cap) { + const deadline = Date.now() + Math.min(limit, cap); + while (Date.now() < deadline) { + if (!inflight.size && Date.now() - lastActivity >= quiet) return; + await delay(Math.min(50, deadline - Date.now())); + } + }, + }; +} + +/** + * The whole page at CSS scale, for layout. Best-effort: very tall pages exceed + * Chromium's texture limits, and an image past the upload limit can't be + * attached. Detail slices cover the page either way. + */ +async function captureOverview( + page: PlaywrightModule.Page, + options: PlaywrightModule.PageScreenshotOptions, + warn: (message: string) => void, +): Promise { + let data: Uint8Array; + try { + data = await page.screenshot({ ...options, fullPage: true, scale: "css" }); + } catch (error) { + warn( + `skipped the whole-page overview: ${(error as Error).message.split("\n")[0]}. Detail slices are complete.`, + ); + return undefined; + } + if (data.byteLength > CHATGPT_MAX_IMAGE_BYTES) { + warn( + `skipped the whole-page overview: ${Math.ceil(data.byteLength / 1024 / 1024)} MB is over ChatGPT's 20 MB per-image limit. Detail slices are complete.`, + ); + return undefined; + } + return data; +} + +/** + * Match install commands to `npm_config_user_agent`, defaulting to npm when + * the invoking package manager is unknown. + */ +export function packageManager(userAgent = ""): { add: string; exec: string } { + switch (userAgent.split("/")[0]) { + case "bun": + return { add: "bun add -d", exec: "bunx" }; + case "pnpm": + return { add: "pnpm add -D", exec: "pnpm exec" }; + case "yarn": + return { add: "yarn add -D", exec: "yarn" }; + default: + return { add: "npm install -D", exec: "npx" }; + } +} diff --git a/tests/e2e/cli.test.ts b/tests/e2e/cli.test.ts index 267db0e..01d70b8 100644 --- a/tests/e2e/cli.test.ts +++ b/tests/e2e/cli.test.ts @@ -658,6 +658,54 @@ describe("cli", () => { ); }); + test("should skip on-demand bundles unless named", async () => { + await mkdir(join(project, "src"), { recursive: true }); + await writeFile(join(project, "src/index.ts"), "export const x = 1;\n"); + await writeFile( + join(project, "srcpack.config.ts"), + `export default { bundles: { + code: "src/**/*", + slow: { include: "src/**/*", onDemand: true }, + } };`, + ); + + const full = await runCli([], { cwd: project }); + + expect(full.exitCode).toBe(0); + expect(full.stdout).toContain("Bundled: 1 bundle,"); + expect(full.stdout).toContain("On demand: slow"); + expect(await Bun.file(join(project, ".srcpack/slow.txt")).exists()).toBe( + false, + ); + + const named = await runCli(["slow"], { cwd: project }); + + expect(named.exitCode).toBe(0); + expect(named.stdout).not.toContain("On demand:"); + expect(await Bun.file(join(project, ".srcpack/slow.txt")).exists()).toBe( + true, + ); + }); + + test("should still empty outDir when every bundle is on demand", async () => { + await mkdir(project, { recursive: true }); + await writeFile(join(project, "a.md"), "# a\n"); + await writeFile( + join(project, "srcpack.config.ts"), + `export default { bundles: { slow: { include: "*.md", onDemand: true } } };`, + ); + const output = join(project, ".srcpack/slow.txt"); + await runCli(["slow"], { cwd: project }); + expect(await Bun.file(output).exists()).toBe(true); + + const result = await runCli([], { cwd: project }); + + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("On demand: slow"); + // A full run rebuilds .srcpack, however few bundles it builds + expect(await Bun.file(output).exists()).toBe(false); + }); + test("should not bundle a previous run's output", async () => { await mkdir(join(project, "src"), { recursive: true }); await writeFile(join(project, "src/index.ts"), "export const x = 1;\n"); diff --git a/tests/e2e/screenshot.test.ts b/tests/e2e/screenshot.test.ts new file mode 100644 index 0000000..47fa2da --- /dev/null +++ b/tests/e2e/screenshot.test.ts @@ -0,0 +1,471 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + test, +} from "bun:test"; +import { mkdir, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { chromium, type Browser } from "playwright"; + +// Real Chromium, driven through the CLI. Pixels are checked by drawing the PNG +// into a canvas in a browser the test already has — no image dependency. + +const CLI_PATH = join(import.meta.dir, "../../src/cli.ts"); +// Each run launches a browser of its own +const TIMEOUT = 60_000; + +const WHITE = [255, 255, 255]; +const GREEN = [0, 128, 0]; + +async function runCli( + args: string[], + cwd: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const proc = Bun.spawn(["bun", CLI_PATH, ...args], { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { stdout, stderr, exitCode: await proc.exited }; +} + +/** Width and height from a PNG's IHDR chunk. */ +async function pngSize(path: string): Promise<[number, number]> { + const view = new DataView(await Bun.file(path).arrayBuffer()); + return [view.getUint32(16), view.getUint32(20)]; +} + +function html(body: string): Response { + return new Response( + `${body}`, + { headers: { "content-type": "text/html" } }, + ); +} + +// Reaching B inserts a spacer that pushes C below the page's original height; +// C paints itself green only once it has been scrolled into view. A capture +// that doesn't re-read the height, or doesn't scroll, never sees green. +const GROWING_PAGE = ` +
+
+
+ `; + +const OVERLAY_PAGE = ` + + + + `; + +// The bottom section fetches its color only once scrolled into view, and the +// server answers slowly: the capture has to wait for the request, not just the +// scroll. Red until the response arrives. +const LAZY_PAGE = ` +
+
+ `; + +// Reaching the bottom starts a slow request whose response appends a tall +// section, and that section paints itself green only once scrolled into view. +// Settling has to wait at the bottom and then keep going: growth that lands +// after it has left is sliced but was never visited. +const APPENDING_PAGE = ` +
+
+ `; + +let shrinkingHeight = 4000; + +const server = Bun.serve({ + port: 0, + async fetch(request) { + switch (new URL(request.url).pathname) { + case "/lazy": + return html(LAZY_PAGE); + case "/appending": + return html(APPENDING_PAGE); + case "/more": + await Bun.sleep(1000); + return new Response("ok"); + case "/lazy-data": + await Bun.sleep(1500); + return Response.json({ color: "rgb(0, 128, 0)" }); + case "/tall": + return html(`
`); + case "/shrinking": + return html(`
`); + case "/growing": + return html(GROWING_PAGE); + case "/overlays": + return html(OVERLAY_PAGE); + case "/user-agent": { + const mobile = request.headers.get("user-agent")?.includes("Mobile"); + const color = mobile ? "rgb(0, 128, 0)" : "rgb(255, 0, 0)"; + return html(`
`); + } + default: + return new Response("Not Found", { status: 404 }); + } + }, +}); +const base = `http://localhost:${server.port}`; + +let browser: Browser; + +beforeAll(async () => { + browser = await chromium.launch(); +}); + +afterAll(async () => { + await browser.close(); + server.stop(true); +}); + +async function pixelAt(path: string, x: number, y: number): Promise { + const png = Buffer.from(await Bun.file(path).arrayBuffer()).toString( + "base64", + ); + const page = await browser.newPage(); + try { + return await page.evaluate(`(async () => { + const image = new Image(); + image.src = "data:image/png;base64,${png}"; + await image.decode(); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d"); + context.drawImage(image, 0, 0); + return Array.from(context.getImageData(${x}, ${y}, 1, 1).data.slice(0, 3)); + })()`); + } finally { + await page.close(); + } +} + +const projects: string[] = []; + +async function project(config?: object): Promise { + const dir = join( + tmpdir(), + `srcpack-screenshot-${Date.now()}-${projects.length}`, + ); + projects.push(dir); + await mkdir(dir, { recursive: true }); + if (config) { + await writeFile( + join(dir, "srcpack.config.ts"), + `export default ${JSON.stringify(config)};`, + ); + } + return dir; +} + +afterEach(async () => { + await Promise.all( + projects.splice(0).map((dir) => rm(dir, { recursive: true, force: true })), + ); +}); + +describe("screenshot bundles", () => { + test( + "should capture a tall page as an overview plus overlapping slices", + async () => { + const dir = await project({ + bundles: { home: { screenshot: `${base}/tall` } }, + }); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + "home 3 images page 1440×4,000 → .srcpack/home-00.png … home-02.png", + ); + expect(result.stdout).toContain("Bundled: 1 bundle, 3 images"); + const out = join(dir, ".srcpack"); + expect((await readdir(out)).sort()).toEqual([ + "home-00.png", + "home-01.png", + "home-02.png", + ]); + expect(await pngSize(join(out, "home-00.png"))).toEqual([1440, 4000]); + // Two slices sharing the page with a 160 px overlap, not two full 2,200s + expect(await pngSize(join(out, "home-01.png"))).toEqual([1440, 2080]); + expect(await pngSize(join(out, "home-02.png"))).toEqual([1440, 2080]); + }, + TIMEOUT, + ); + + test( + "should settle a page that grows while it is scrolled", + async () => { + const dir = await project({ + bundles: { home: { screenshot: `${base}/growing` } }, + }); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + const overview = join(dir, ".srcpack/home-00.png"); + // 3,500 px as served; 5,500 once the spacer went in + expect(await pngSize(overview)).toEqual([1440, 5500]); + expect(await pixelAt(overview, 720, 5250)).toEqual(GREEN); + // Details too: 1,940 px slices start at 0, 1,780 and 3,560, so C's + // middle sits at 1,690 in the last. A capture repeating y=0 shows white. + const last = join(dir, ".srcpack/home-03.png"); + expect(await pngSize(last)).toEqual([1440, 1940]); + expect(await pixelAt(last, 720, 1690)).toEqual(GREEN); + }, + TIMEOUT, + ); + + test( + "should wait for data a scrolled-to section requests", + async () => { + const dir = await project({ + bundles: { home: { screenshot: `${base}/lazy` } }, + }); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + // 3,500 px page: 1,830 px slices at 0 and 1,670. The response lands + // well after scrolling ends, so only the network wait makes it green. + expect( + await pixelAt(join(dir, ".srcpack/home-02.png"), 720, 1580), + ).toEqual(GREEN); + expect( + await pixelAt(join(dir, ".srcpack/home-00.png"), 720, 3250), + ).toEqual(GREEN); + }, + TIMEOUT, + ); + + test( + "should visit content a late response appends at the bottom", + async () => { + const dir = await project({ + bundles: { home: { screenshot: `${base}/appending` } }, + }); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + const overview = join(dir, ".srcpack/home-00.png"); + // 3,010 px as served; 5,510 once the response appended its sections + expect(await pngSize(overview)).toEqual([1440, 5510]); + expect(await pixelAt(overview, 720, 5260)).toEqual(GREEN); + // 1,944 px slices at 0, 1,783 and 3,566: the last section's middle is + // at 1,694 in the final one + expect( + await pixelAt(join(dir, ".srcpack/home-03.png"), 720, 1694), + ).toEqual(GREEN); + }, + TIMEOUT, + ); + + test( + "should hide overlays and write text and images for a mixed bundle", + async () => { + const dir = await project({ + bundles: { + home: { + include: "*.md", + screenshot: { url: `${base}/overlays`, hide: ["#banner"] }, + }, + }, + }); + await writeFile(join(dir, "notes.md"), "# notes\n"); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + "home 1 file 1 line → .srcpack/home.txt", + ); + expect(result.stdout).toContain( + "home 1 image page 1440×900 → .srcpack/home-00.png", + ); + // A page that fits one slice is its own overview + expect((await readdir(join(dir, ".srcpack"))).sort()).toEqual([ + "home-00.png", + "home.txt", + ]); + const image = join(dir, ".srcpack/home-00.png"); + expect(await pixelAt(image, 720, 50)).toEqual(WHITE); // #banner + expect(await pixelAt(image, 720, 850)).toEqual(WHITE); // dev toolbar + // Next's portal also carries its error overlay, so it stays visible + // unless listed in `hide` + expect(await pixelAt(image, 720, 450)).toEqual([128, 0, 128]); + }, + TIMEOUT, + ); + + test( + "should remove stale images when the page shrinks, and nothing else", + async () => { + shrinkingHeight = 4000; + const dir = await project({ + bundles: { home: { screenshot: `${base}/shrinking`, onDemand: true } }, + }); + expect((await runCli(["home"], dir)).exitCode).toBe(0); + await writeFile(join(dir, ".srcpack/keep.txt"), "keep\n"); + // Another bundle's family, which a prefix match would take for `home`'s + await writeFile(join(dir, ".srcpack/home-01-02.png"), "home-01's\n"); + + try { + shrinkingHeight = 500; + const rerun = await runCli(["home"], dir); + expect(rerun.exitCode).toBe(0); + } finally { + shrinkingHeight = 4000; + } + + expect((await readdir(join(dir, ".srcpack"))).sort()).toEqual([ + "home-00.png", + "home-01-02.png", + "keep.txt", + ]); + }, + TIMEOUT, + ); + + test( + "should render the mobile markup a server sends to a phone", + async () => { + const dir = await project({ + bundles: { + phone: { + screenshot: { url: `${base}/user-agent`, viewport: "mobile" }, + }, + }, + }); + + const result = await runCli([], dir); + + expect(result.exitCode).toBe(0); + const image = join(dir, ".srcpack/phone-00.png"); + // 412 CSS px at DPR 2 + expect((await pngSize(image))[0]).toBe(824); + expect(await pixelAt(image, 20, 20)).toEqual(GREEN); + }, + TIMEOUT, + ); + + test( + "should fail cleanly and keep previous output when a page can't be captured", + async () => { + const closed = Bun.serve({ port: 0, fetch: () => new Response() }); + const closedPort = closed.port; + closed.stop(true); + const dir = await project({ + bundles: { + home: { screenshot: `http://localhost:${closedPort}/` }, + missing: { screenshot: `${base}/missing`, onDemand: true }, + }, + }); + await mkdir(join(dir, ".srcpack")); + const previous = join(dir, ".srcpack/home-00.png"); + await writeFile(previous, "previous run"); + + const unreachable = await runCli([], dir); + + expect(unreachable.exitCode).toBe(1); + expect(unreachable.stderr).toContain( + `Bundle "home": http://localhost:${closedPort}/ is not reachable (connection refused). Is your dev server running? Set onDemand: true to capture it only when named.`, + ); + expect(unreachable.stderr).not.toContain(" at "); + + const notFound = await runCli(["missing"], dir); + + expect(notFound.exitCode).toBe(1); + expect(notFound.stderr).toContain( + `Bundle "missing": ${base}/missing returned 404`, + ); + expect(await Bun.file(previous).text()).toBe("previous run"); + }, + TIMEOUT, + ); + + test( + "should capture an ad-hoc URL without a config file", + async () => { + const dir = await project(); + + const result = await runCli( + [ + "--screenshot", + `localhost:${server.port}/user-agent`, + "--viewport", + "mobile", + ], + dir, + ); + + expect(result.exitCode).toBe(0); + const image = join(dir, ".srcpack/screenshot-00.png"); + expect((await pngSize(image))[0]).toBe(824); + }, + TIMEOUT, + ); + + test("should preview without requesting the page", async () => { + // Nothing listens on port 1, so a request would fail the run + const dir = await project({ + bundles: { home: { screenshot: "http://localhost:1/" } }, + }); + + const result = await runCli(["--dry-run"], dir); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + "home screenshot http://localhost:1/ desktop → .srcpack/home-NN.png", + ); + expect(result.stdout).toContain("Dry run: 1 bundle"); + expect(await Bun.file(join(dir, ".srcpack")).exists()).toBe(false); + }); +}); diff --git a/tests/unit/args.test.ts b/tests/unit/args.test.ts new file mode 100644 index 0000000..6773381 --- /dev/null +++ b/tests/unit/args.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "bun:test"; +import { parseCliArgs, UsageError } from "../../src/args.ts"; + +describe("parseCliArgs", () => { + test("should default to a full run that uploads", () => { + expect(parseCliArgs([])).toEqual({ + bundles: [], + adHoc: null, + dryRun: false, + emptyOutDir: undefined, + upload: true, + }); + }); + + test("should read bundle names and flags in any order", () => { + expect( + parseCliArgs(["web", "--dry-run", "api", "--no-upload", "--emptyOutDir"]), + ).toEqual({ + bundles: ["web", "api"], + adHoc: null, + dryRun: true, + emptyOutDir: true, + upload: false, + }); + expect(parseCliArgs(["--no-emptyOutDir"]).emptyOutDir).toBe(false); + }); + + test("should keep the value of --since out of the bundle names", () => { + // `init` here is a branch, not a bundle or the wizard + expect(parseCliArgs(["--since", "init"])).toMatchObject({ + bundles: [], + adHoc: { name: "since", source: ["git:init", "git:untracked"] }, + }); + expect(parseCliArgs(["--since=main"]).adHoc?.source).toEqual([ + "git:main", + "git:untracked", + ]); + }); + + test("should build an ad-hoc screenshot bundle", () => { + expect( + parseCliArgs(["--screenshot", "localhost:5173/pricing"]).adHoc, + ).toEqual({ + name: "screenshot", + source: { + screenshot: { + url: "http://localhost:5173/pricing", + viewport: "desktop", + }, + }, + }); + expect( + parseCliArgs([ + "--viewport", + "mobile", + "--screenshot=https://example.com/", + ]).adHoc, + ).toEqual({ + name: "screenshot", + source: { + screenshot: { url: "https://example.com/", viewport: "mobile" }, + }, + }); + }); + + test("should add a scheme even when the URL embeds another one", () => { + const source = parseCliArgs([ + "--screenshot", + "localhost:5173/?next=https://example.com", + ]).adHoc?.source; + + expect(source).toMatchObject({ + screenshot: { url: "http://localhost:5173/?next=https://example.com" }, + }); + }); + + test("should reject a malformed screenshot request", () => { + expect(() => parseCliArgs(["--viewport", "mobile"])).toThrow( + "--viewport applies to --screenshot .", + ); + expect(() => parseCliArgs(["--screenshot"])).toThrow("Missing URL"); + expect(() => parseCliArgs(["--screenshot", "--dry-run"])).toThrow( + "Missing URL", + ); + expect(() => + parseCliArgs(["--screenshot", "localhost", "--viewport", "wat"]), + ).toThrow('--viewport must be "desktop" or "mobile", got "wat".'); + expect(() => parseCliArgs(["--screenshot", "ftp://example.com/"])).toThrow( + "--screenshot takes an http(s) URL", + ); + expect(() => parseCliArgs(["--screenshot", "localhost", "web"])).toThrow( + "Cannot combine --screenshot with named bundles.", + ); + expect(() => + parseCliArgs(["--screenshot", "localhost", "--staged"]), + ).toThrow("Cannot combine --staged and --screenshot."); + }); + + // Each of these reads as a real option and would otherwise do the opposite + // of what was typed, or nothing at all + test.each([ + ["--upload"], + ["--no-dry-run"], + ["--no-staged"], + ["--no-uplaod"], + // A terminator parseArgs would accept, turning `srcpack --` into a full run + ["--"], + ])("should reject %p as an unknown option", (flag) => { + expect(() => parseCliArgs([flag])).toThrow( + `Unknown option: ${flag}\nRun \`srcpack --help\``, + ); + }); + + test.each([[["--since"]], [["--since", "-x"]], [["--since="]]])( + "should require a revision for %p", + (argv) => { + expect(() => parseCliArgs(argv)).toThrow("Missing revision"); + }, + ); + + test("should reject a boolean flag given a value", () => { + expect(() => parseCliArgs(["--dry-run=1"])).toThrow(UsageError); + }); + + test("should reject contradictory and exclusive combinations", () => { + expect(() => parseCliArgs(["--emptyOutDir", "--no-emptyOutDir"])).toThrow( + "Cannot combine --emptyOutDir with --no-emptyOutDir.", + ); + expect(() => parseCliArgs(["--staged", "--since", "main"])).toThrow( + "Cannot combine --staged and --since.", + ); + expect(() => parseCliArgs(["web", "--dirty"])).toThrow( + "Cannot combine --dirty with named bundles.", + ); + expect(() => parseCliArgs(["--since", "main..HEAD"])).toThrow( + "not a range", + ); + // Last-one-wins would bundle a different change set than the one asked for + expect(() => parseCliArgs(["--since", "main", "--since", "HEAD"])).toThrow( + '--since takes one value; got "main" and "HEAD".', + ); + expect(() => + parseCliArgs(["--screenshot", "a", "--screenshot", "b"]), + ).toThrow('--screenshot takes one value; got "a" and "b".'); + }); +}); diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index 206c251..6f334ca 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -77,16 +77,15 @@ describe("parseConfig", () => { }); }); - test("should default index to true for object config", () => { + test("should leave index unset when omitted", () => { + // Defaulted where it is read, so the schema can tell set from absent const config = parseConfig({ bundles: { web: { include: "src/**/*" }, }, }); - expect( - (config.bundles.web as { include: string; index: boolean }).index, - ).toBe(true); + expect(config.bundles.web).toEqual({ include: "src/**/*" }); }); test("should accept multiple bundles with mixed formats", () => { @@ -426,7 +425,7 @@ describe("parseConfig", () => { test("should accept the team shorthand", () => { const config = parseConfig({ bundles: { backlog: { linear: "ENG" } } }); - expect(config.bundles.backlog).toEqual({ linear: "ENG", index: true }); + expect(config.bundles.backlog).toEqual({ linear: "ENG" }); }); test("should default includeClosed to false", () => { @@ -516,6 +515,94 @@ describe("parseConfig", () => { }); }); +describe("screenshot source", () => { + const messageOf = (bundles: Record): string => { + try { + parseConfig({ bundles }); + } catch (e) { + return (e as ConfigError).message; + } + throw new Error("should have thrown"); + }; + + test("should accept the URL shorthand and the object form", () => { + const config = parseConfig({ + bundles: { + home: { screenshot: "http://localhost:5173/", onDemand: true }, + phone: { + screenshot: { + url: "https://example.com/pricing", + viewport: "mobile", + hide: ["#cookie-banner"], + }, + }, + }, + }); + + expect(config.bundles.home).toEqual({ + screenshot: "http://localhost:5173/", + onDemand: true, + }); + expect(config.bundles.phone).toMatchObject({ + screenshot: { viewport: "mobile", hide: ["#cookie-banner"] }, + }); + }); + + test("should name a missing scheme rather than a bad protocol", () => { + // `new URL("localhost:5173")` parses, as protocol `localhost:` + expect(messageOf({ home: { screenshot: "localhost:5173" } })).toBe( + 'bundles.home.screenshot: URL needs a scheme, e.g. "http://localhost:5173/"', + ); + expect(messageOf({ home: { screenshot: { url: "localhost:5173" } } })).toBe( + 'bundles.home.screenshot.url: URL needs a scheme, e.g. "http://localhost:5173/"', + ); + }); + + test("should reject non-http URLs and unknown options", () => { + expect(messageOf({ home: { screenshot: "ftp://example.com/" } })).toBe( + "bundles.home.screenshot: Expected an http(s) URL", + ); + expect( + messageOf({ home: { screenshot: { url: "http://x/", wait: 1 } } }), + ).toBe('bundles.home.screenshot: Unrecognized key: "wait"'); + expect( + messageOf({ home: { screenshot: { url: "http://x/", viewport: "tv" } } }), + ).toContain("bundles.home.screenshot.viewport"); + }); + + test.each([ + ["prompt", "Review this."], + ["index", false], + ["outfile", "home.txt"], + ])("should reject %p on a screenshot-only bundle", (key, value) => { + expect( + messageOf({ home: { screenshot: "http://x/", [key]: value } }), + ).toStartWith(`bundles.home.${key}: "${key}" applies to the text file`); + }); + + test("should accept text options once the bundle also writes text", () => { + const config = parseConfig({ + bundles: { + pricing: { + include: "src/pages/pricing/**/*", + screenshot: "http://localhost:5173/pricing", + prompt: + "Review this implementation against the attached screenshots.", + index: false, + }, + }, + }); + + expect(config.bundles.pricing).toMatchObject({ index: false }); + }); + + test("should still require some source", () => { + expect(messageOf({ web: { onDemand: true } })).toContain( + '"include" patterns, "linear", or "screenshot"', + ); + }); +}); + describe("defineConfig", () => { test("should return config unchanged", () => { const input: ConfigInput = { diff --git a/tests/unit/plan.test.ts b/tests/unit/plan.test.ts new file mode 100644 index 0000000..c388a00 --- /dev/null +++ b/tests/unit/plan.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ConfigError, parseConfig } from "../../src/config.ts"; +import { planOutputs, selectBundles } from "../../src/plan.ts"; + +describe("planOutputs", () => { + const root = join(tmpdir(), `srcpack-plan-${Date.now()}`); + + afterEach(async () => { + await rm(root, { recursive: true, force: true }); + }); + + const bundlesOf = (bundles: Record) => + parseConfig({ bundles }).bundles; + + test("should derive each active destination once", async () => { + const configured = bundlesOf({ + web: "src/**/*", + docs: { include: "docs/**", outfile: "out/docs.md" }, + }); + + const plan = await planOutputs(root, ".srcpack", configured, [ + ["docs", configured.docs!], + ]); + + expect(plan.bundles).toEqual([ + { + name: "docs", + source: configured.docs!, + text: { outfile: join(root, "out/docs.md") }, + }, + ]); + // A bundle that isn't active still claims its outputs + await expect( + planOutputs(root, ".srcpack", configured, [["web", configured.web!]]), + ).resolves.toMatchObject({ bundles: [{ name: "web" }] }); + // Inactive bundles' outputs are still srcpack's, and never bundled + expect(plan.ownOutputs).toContain(join(root, ".srcpack/web.txt")); + }); + + test("should reject configured bundles sharing a file on any run", async () => { + const configured = bundlesOf({ + Web: { include: "src/**", outfile: ".srcpack/Context.txt" }, + web: { include: "src/**", outfile: ".srcpack/context.txt" }, + }); + + // Neither bundle is active: a config error doesn't depend on the request + await expect(planOutputs(root, ".srcpack", configured, [])).rejects.toThrow( + 'Bundles "Web" and "web" both write to ".srcpack/context.txt"', + ); + }); + + test("should reject outfiles aliased through a symlinked directory", async () => { + await mkdir(join(root, ".srcpack"), { recursive: true }); + await symlink(join(root, ".srcpack"), join(root, "alias")); + const configured = bundlesOf({ + a: { include: "src/**", outfile: ".srcpack/ctx.txt" }, + b: { include: "src/**", outfile: "alias/ctx.txt" }, + }); + + await expect(planOutputs(root, ".srcpack", configured, [])).rejects.toThrow( + "both write to", + ); + }); + + test("should let an ad-hoc bundle shadow a configured one of its name", async () => { + const configured = bundlesOf({ staged: ["git:staged", "!bun.lock"] }); + + const plan = await planOutputs(root, ".srcpack", configured, [ + ["staged", ["git:staged"]], + ]); + + expect(plan.bundles.map((b) => b.name)).toEqual(["staged"]); + }); + + test("should plan text, images, or both", async () => { + const configured = bundlesOf({ + home: { screenshot: "http://localhost:5173/" }, + pricing: { + include: "src/**", + screenshot: { + url: "http://localhost:5173/pricing", + viewport: "mobile", + }, + }, + }); + + const { bundles } = await planOutputs(root, ".srcpack", configured, [ + ["home", configured.home!], + ["pricing", configured.pricing!], + ]); + + // A screenshot-only bundle reserves no .txt + expect(bundles[0]!.text).toBeUndefined(); + expect(bundles[0]!.images).toEqual({ + target: { url: "http://localhost:5173/", viewport: "desktop", hide: [] }, + dir: join(root, ".srcpack"), + }); + expect(bundles[1]!.text).toEqual({ + outfile: join(root, ".srcpack/pricing.txt"), + }); + expect(bundles[1]!.images?.target.viewport).toBe("mobile"); + }); + + test("should reject an outfile inside its own bundle's image family", async () => { + const configured = bundlesOf({ + home: { + include: "src/**", + screenshot: "http://x/", + outfile: ".srcpack/home-00.png", + }, + }); + + await expect(planOutputs(root, ".srcpack", configured, [])).rejects.toThrow( + 'Bundle "home" writes its text and its images to ".srcpack/home-00.png"', + ); + }); + + test("should reject an outfile reaching a family through a symlink", async () => { + await mkdir(join(root, ".srcpack"), { recursive: true }); + await symlink(join(root, ".srcpack"), join(root, "alias")); + const configured = bundlesOf({ + home: { screenshot: "http://x/" }, + notes: { include: "docs/**", outfile: "alias/HOME-03.png" }, + }); + + await expect(planOutputs(root, ".srcpack", configured, [])).rejects.toThrow( + 'Bundles "home" and "notes" both write to "alias/HOME-03.png"', + ); + }); + + test("should tell apart families whose names only share a prefix", async () => { + const configured = bundlesOf({ + home: { screenshot: "http://x/" }, + "home-01": { screenshot: "http://x/01" }, + notes: { include: "docs/**", outfile: ".srcpack/home-01.txt" }, + }); + + await expect( + planOutputs(root, ".srcpack", configured, []), + ).resolves.toBeDefined(); + }); + + test("should reject image families differing only by case", async () => { + const configured = bundlesOf({ + Web: { screenshot: "http://x/" }, + web: { screenshot: "http://x/" }, + }); + + await expect(planOutputs(root, ".srcpack", configured, [])).rejects.toThrow( + 'Bundles "Web" and "web" both write to ".srcpack/web-NN.png". Rename one of them.', + ); + }); + + test("should let --screenshot shadow a configured screenshot bundle", async () => { + const configured = bundlesOf({ + screenshot: { screenshot: "http://localhost:5173/" }, + }); + + const { bundles } = await planOutputs(root, ".srcpack", configured, [ + ["screenshot", { screenshot: { url: "http://localhost:3000/" } }], + ]); + + expect(bundles[0]!.images?.target.url).toBe("http://localhost:3000/"); + }); + + test("should reject --screenshot over another bundle's outfile", async () => { + const configured = bundlesOf({ + notes: { include: "docs/**", outfile: ".srcpack/screenshot-00.png" }, + }); + + await expect( + planOutputs(root, ".srcpack", configured, [ + ["screenshot", { screenshot: { url: "http://localhost:3000/" } }], + ]), + ).rejects.toThrow('Bundles "notes" and "screenshot" both write to'); + }); + + test("should reject an ad-hoc bundle writing over another bundle's file", async () => { + const configured = bundlesOf({ + review: { include: "git:staged", outfile: ".srcpack/staged.txt" }, + }); + + await expect( + planOutputs(root, ".srcpack", configured, [["staged", ["git:staged"]]]), + ).rejects.toThrow('Bundles "review" and "staged" both write to'); + }); +}); + +describe("selectBundles", () => { + const { bundles } = parseConfig({ + bundles: { + code: "src/**/*", + docs: { include: "docs/**/*", onDemand: false }, + home: { include: "pages/**/*", onDemand: true }, + backlog: { linear: "ENG", onDemand: true }, + }, + }); + + test("should skip on-demand bundles in a full run", () => { + expect(selectBundles(bundles, [])).toEqual({ + names: ["code", "docs"], + skipped: ["home", "backlog"], + }); + }); + + test("should build a named bundle whether or not it is on demand", () => { + expect(selectBundles(bundles, ["home", "code"])).toEqual({ + names: ["home", "code"], + skipped: [], + }); + }); + + test("should select nothing when every bundle is on demand", () => { + const config = parseConfig({ + bundles: { home: { include: "x", onDemand: true } }, + }); + expect(selectBundles(config.bundles, [])).toEqual({ + names: [], + skipped: ["home"], + }); + }); + + test("should build a bundle named twice once", () => { + expect(selectBundles(bundles, ["home", "home"]).names).toEqual(["home"]); + }); + + test("should reject an unknown or inherited name", () => { + expect(() => selectBundles(bundles, ["nope"])).toThrow(ConfigError); + expect(() => selectBundles(bundles, ["toString"])).toThrow( + "Unknown bundle: toString", + ); + }); +}); diff --git a/tests/unit/screenshot.test.ts b/tests/unit/screenshot.test.ts new file mode 100644 index 0000000..28483b8 --- /dev/null +++ b/tests/unit/screenshot.test.ts @@ -0,0 +1,354 @@ +import { describe, expect, test } from "bun:test"; +import { EventEmitter } from "node:events"; +import { pathKey } from "../../src/fs.ts"; +import { + imageFileName, + isImageOf, + launchBrowser, + loadPlaywright, + packageManager, + planSlices, + ScreenshotError, + toScreenshotTarget, + watchNetwork, + type Playwright, + type Require, +} from "../../src/screenshot.ts"; + +describe("planSlices", () => { + test("should return the whole page when it fits in one slice", () => { + expect(planSlices(900, 1)).toEqual([{ y: 0, height: 900 }]); + expect(planSlices(2200, 1)).toEqual([{ y: 0, height: 2200 }]); + }); + + test("should cover the page with overlapping slices ending flush", () => { + for (const [height, dpr] of [ + [2201, 1], + [4000, 1], + [4240, 1], // 2 × 2,200 − 160: exactly two slices + [9000, 1], + [9480, 2], + [45_000, 1], + ] as const) { + const slices = planSlices(height, dpr); + expect(slices[0]!.y).toBe(0); + const last = slices.at(-1)!; + expect(last.y + last.height).toBe(height); + for (let i = 1; i < slices.length; i++) { + const overlap = slices[i - 1]!.y + slices[i - 1]!.height - slices[i]!.y; + expect(overlap).toBeGreaterThanOrEqual(160 / dpr); + // Evenly spread: no gap wider than the budget allows + expect(slices[i]!.y - slices[i - 1]!.y).toBeLessThanOrEqual( + (2200 - 160) / dpr, + ); + } + } + }); + + test("should end flush even where floating point rounds down", () => { + for (const [height, dpr] of [ + [1_048_598, 2], + [4000.5, 1], + ] as const) { + const last = planSlices(height, dpr).at(-1)!; + expect(last.y + last.height).toBe(height); + } + }); + + test("should use the minimum number of slices", () => { + expect(planSlices(4240, 1)).toHaveLength(2); + expect(planSlices(4241, 1)).toHaveLength(3); + }); + + test("should size slices in device pixels", () => { + // DPR 2 doubles pixel dimensions, so a slice covers half as much page + expect(planSlices(1100, 2)).toEqual([{ y: 0, height: 1100 }]); + expect(planSlices(1101, 2)).toEqual([ + { y: 0, height: 591 }, + { y: 510, height: 591 }, + ]); + }); + + test("should shrink slices to share the page instead of duplicating it", () => { + // Two near-identical 2,200 px images would each cost a model's attention + expect(planSlices(2201, 1)).toEqual([ + { y: 0, height: 1181 }, + { y: 1020, height: 1181 }, + ]); + for (const [height, dpr] of [ + [2201, 1], + [4000, 1], + [9480, 2], + [45_000, 1], + ] as const) { + const slices = planSlices(height, dpr); + for (let i = 1; i < slices.length; i++) { + const overlap = slices[i - 1]!.y + slices[i - 1]!.height - slices[i]!.y; + // Rounding to whole pixels adds at most a couple + expect(overlap).toBeLessThanOrEqual(160 / dpr + 3); + } + expect(slices[0]!.height).toBeLessThanOrEqual(2200 / dpr); + } + }); +}); + +describe("imageFileName", () => { + test("should pad to two digits, or more past 99", () => { + expect(imageFileName("home", 0, 3)).toBe("home-00.png"); + expect(imageFileName("home", 9, 9)).toBe("home-09.png"); + expect(imageFileName("home", 10, 10)).toBe("home-10.png"); + expect(imageFileName("home", 99, 99)).toBe("home-99.png"); + expect(imageFileName("home", 7, 100)).toBe("home-007.png"); + expect(imageFileName("home", 100, 100)).toBe("home-100.png"); + }); + + test("should keep a logical index when the overview is omitted", () => { + expect([1, 2].map((i) => imageFileName("home", i, 2))).toEqual([ + "home-01.png", + "home-02.png", + ]); + }); +}); + +describe("isImageOf", () => { + test("should match a bundle's own numbered images", () => { + expect(isImageOf("home", "home-00.png")).toBe(true); + expect(isImageOf("home", "home-100.png")).toBe(true); + }); + + test("should match exactly, leaving folding to the caller", () => { + // Cleanup deletes what matches, so it must not widen by case + expect(isImageOf("Web", "web-01.PNG")).toBe(false); + // Collision checks fold both sides: one entry on macOS and Windows + expect(isImageOf(pathKey("Web"), pathKey("web-01.PNG"))).toBe(true); + }); + + test("should not cross-match similar bundle names", () => { + expect(isImageOf("home", "home-01-02.png")).toBe(false); + expect(isImageOf("home-01", "home-01-02.png")).toBe(true); + expect(isImageOf("home", "home.v2-00.png")).toBe(false); + expect(isImageOf("home", "home-0.png")).toBe(false); + expect(isImageOf("home", "home-00.png.tmp")).toBe(false); + expect(isImageOf("home", "home.txt")).toBe(false); + }); +}); + +describe("toScreenshotTarget", () => { + test("should apply defaults to both forms", () => { + expect(toScreenshotTarget("http://localhost:5173/")).toEqual({ + url: "http://localhost:5173/", + viewport: "desktop", + hide: [], + }); + expect( + toScreenshotTarget({ + url: "http://x/", + viewport: "mobile", + hide: ["#a"], + }), + ).toEqual({ url: "http://x/", viewport: "mobile", hide: ["#a"] }); + }); +}); + +describe("loadPlaywright", () => { + const notFound = () => + Object.assign(new Error("Cannot find module"), { + code: "MODULE_NOT_FOUND", + }); + + /** A require that knows only the given packages. */ + function fakeRequire(packages: Record): Require { + return (id) => { + const name = id.replace(/\/package\.json$/, ""); + const version = packages[name]; + if (version === undefined) throw notFound(); + return id.endsWith("/package.json") + ? { version } + : { chromium: `${name}@${version}`, devices: {} }; + }; + } + + const load = ( + project: Record, + own: Record, + userAgent?: string, + ) => + loadPlaywright( + "/project", + [ + [fakeRequire(project), "playwright"], + [fakeRequire(project), "@playwright/test"], + [fakeRequire(own), "playwright"], + ], + userAgent, + ).chromium as unknown as string; + + test("should prefer the project's copy over srcpack's", () => { + expect(load({ playwright: "1.50.0" }, { playwright: "1.63.0" })).toBe( + "playwright@1.50.0", + ); + }); + + test("should fall back to @playwright/test, then srcpack's own", () => { + expect( + load({ "@playwright/test": "1.45.1" }, { playwright: "1.63.0" }), + ).toBe("@playwright/test@1.45.1"); + expect(load({}, { playwright: "1.63.0" })).toBe("playwright@1.63.0"); + }); + + test("should reject a version outside the supported range", () => { + expect(() => load({ playwright: "1.38.0" }, {})).toThrow( + "Playwright 1.38.0 is too old; srcpack needs 1.41 or newer.", + ); + expect(() => load({ playwright: "2.0.0" }, {})).toThrow(ScreenshotError); + }); + + test("should print the install command for the invoking package manager", () => { + expect(() => load({}, {}, "bun/1.4.2 npm/? node/v24")).toThrow( + "screenshots need Playwright. Install it with:\n bun add -d playwright && bunx playwright install chromium", + ); + expect(() => load({}, {}, undefined)).toThrow( + "npm install -D playwright && npx playwright install chromium", + ); + }); + + test("should not hide a broken installation behind 'not installed'", () => { + const broken: Require = () => { + throw new SyntaxError("Unexpected token"); + }; + expect(() => loadPlaywright("/project", [[broken, "playwright"]])).toThrow( + SyntaxError, + ); + }); +}); + +describe("launchBrowser", () => { + const browser = { close: async () => {} }; + + function fakeChromium( + outcomes: Record unknown>, + ): Playwright & { calls: (string | undefined)[] } { + const calls: (string | undefined)[] = []; + const chromium = { + launch: async (options?: { channel?: string }) => { + calls.push(options?.channel); + return outcomes[options?.channel ?? "bundled"]!(); + }, + }; + return { chromium, devices: {}, calls } as never; + } + + const missing = () => { + throw new Error("browserType.launch: Executable doesn't exist at /x"); + }; + + test("should use Playwright's Chromium when installed", async () => { + const pw = fakeChromium({ bundled: () => browser }); + expect(await launchBrowser(pw)).toBe(browser as never); + expect(pw.calls).toEqual([undefined]); + }); + + test("should fall back to system Chrome when Chromium is missing", async () => { + const pw = fakeChromium({ bundled: missing, chrome: () => browser }); + expect(await launchBrowser(pw)).toBe(browser as never); + expect(pw.calls).toEqual([undefined, "chrome"]); + }); + + test("should print the install command when neither is available", async () => { + const pw = fakeChromium({ + bundled: missing, + chrome: () => { + throw new Error( + "browserType.launch: Chromium distribution 'chrome' is not found at /opt/google/chrome/chrome", + ); + }, + }); + await expect(launchBrowser(pw, "pnpm/9.0.0")).rejects.toThrow( + "screenshots need a browser. Install Chromium with:\n pnpm exec playwright install chromium", + ); + }); + + test("should rethrow a system Chrome that exists but fails to start", async () => { + // "Install Chromium" would send the user after the wrong problem + const pw = fakeChromium({ + bundled: missing, + chrome: () => { + throw new Error("browserType.launch: Target crashed (sandbox)"); + }, + }); + await expect(launchBrowser(pw)).rejects.toThrow("Target crashed"); + }); + + test("should rethrow a launch failure that isn't a missing browser", async () => { + const pw = fakeChromium({ + bundled: () => { + throw new Error("Target page, context or browser has been closed"); + }, + }); + await expect(launchBrowser(pw)).rejects.toThrow("has been closed"); + expect(pw.calls).toEqual([undefined]); + }); +}); + +describe("watchNetwork", () => { + const quiet = 100; + + test("should wait out requests that start and finish between polls", async () => { + const events = new EventEmitter(); + const network = watchNetwork(events, { quiet, cap: 5_000 }); + + // Each request completes within one tick, so sampling how many are in + // flight would never see one. Measured against the last request actually + // sent, so a slow timer can't make a correct wait look early. + let lastRequest = 0; + const timer = setInterval(() => { + const request = {}; + lastRequest = Date.now(); + events.emit("request", request); + events.emit("requestfinished", request); + }, 20); + const stopped = Bun.sleep(300).then(() => clearInterval(timer)); + + await network.idle(); + const idleAt = Date.now(); + await stopped; + + // Resolving while requests were still arriving would put one after idleAt + expect(idleAt - lastRequest).toBeGreaterThanOrEqual(quiet); + }); + + test("should give up on a request that never finishes at the cap", async () => { + const events = new EventEmitter(); + const network = watchNetwork(events, { quiet, cap: 300 }); + events.emit("request", {}); + + const started = Date.now(); + await network.idle(); + + expect(Date.now() - started).toBeGreaterThanOrEqual(300); + }); + + test("should stop at a shorter limit than the cap", async () => { + // Settling passes what is left of its own deadline + const events = new EventEmitter(); + const network = watchNetwork(events, { quiet, cap: 5_000 }); + events.emit("request", {}); + + const started = Date.now(); + await network.idle(150); + const waited = Date.now() - started; + + expect(waited).toBeGreaterThanOrEqual(150); + expect(waited).toBeLessThan(1_000); + }); +}); + +describe("packageManager", () => { + test.each([ + ["npm/10.8.0 node/v24.0.0", "npx"], + ["yarn/4.5.0 npm/? node/v24", "yarn"], + ["", "npx"], + ])("should map %p", (userAgent, exec) => { + expect(packageManager(userAgent).exec).toBe(exec); + }); +});