diff --git a/.github/workflows/site-preview.yml b/.github/workflows/site-preview.yml index f836d816b..c79500c9d 100644 --- a/.github/workflows/site-preview.yml +++ b/.github/workflows/site-preview.yml @@ -26,7 +26,8 @@ jobs: if: >- github.event_name == 'workflow_dispatch' || (github.event.action != 'labeled' && github.event.action != 'unlabeled') || - startsWith(github.event.label.name, 'docs-translations-') + startsWith(github.event.label.name, 'docs-translations-') || + github.event.label.name == 'docs-preview-show-remotes' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -114,20 +115,26 @@ jobs: echo "merge_sha=$merge_sha" >> "$GITHUB_OUTPUT" echo "trust=$trust" >> "$GITHUB_OUTPUT" - - name: Select translated collections from pull request labels - id: translations + - name: Select preview options from pull request labels + id: preview-options env: GH_TOKEN: ${{ github.token }} PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }} + TRUST: ${{ steps.pull-request.outputs.trust }} run: | mapfile -t labels < <( gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST_NUMBER/labels" --jq '.[].name' ) known_locales=(ar es fr ja ko pt-BR ru zh) include_translations=false + include_remotes=false for label in "${labels[@]}"; do normalized="${label,,}" + if [[ "$normalized" == "docs-preview-show-remotes" ]]; then + include_remotes=true + continue + fi if [[ "$normalized" == "docs-translations-all" ]]; then include_translations=true continue @@ -159,8 +166,25 @@ jobs: locale_summary="English only" fi + if [[ "$include_remotes" == true ]]; then + if [[ "$TRUST" != "trusted-branch" ]]; then + echo "docs-preview-show-remotes can only be used for pull requests from ClickHouse/mintlify-docs-dev." >&2 + exit 1 + fi + remote_scope="all" + preview_environment="connect-preview" + remote_summary="all registered remotes" + else + remote_scope="none" + preview_environment="" + remote_summary="no remotes" + fi + echo "scope=$locale_scope" >> "$GITHUB_OUTPUT" echo "summary=$locale_summary" >> "$GITHUB_OUTPUT" + echo "remote_scope=$remote_scope" >> "$GITHUB_OUTPUT" + echo "preview_environment=$preview_environment" >> "$GITHUB_OUTPUT" + echo "remote_summary=$remote_summary" >> "$GITHUB_OUTPUT" # GitHub owns refs/pull//merge in the primary repository, even # when the pull request head is in a fork. Vercel fetches that immutable @@ -174,8 +198,11 @@ jobs: HEAD_REPOSITORY: ${{ steps.pull-request.outputs.head_repository }} HEAD_SHA: ${{ steps.pull-request.outputs.head_sha }} PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }} - TRANSLATION_SCOPE: ${{ steps.translations.outputs.scope }} - TRANSLATION_SUMMARY: ${{ steps.translations.outputs.summary }} + TRANSLATION_SCOPE: ${{ steps.preview-options.outputs.scope }} + TRANSLATION_SUMMARY: ${{ steps.preview-options.outputs.summary }} + REMOTE_SCOPE: ${{ steps.preview-options.outputs.remote_scope }} + PREVIEW_ENVIRONMENT: ${{ steps.preview-options.outputs.preview_environment }} + REMOTE_SUMMARY: ${{ steps.preview-options.outputs.remote_summary }} GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} TRUST: ${{ steps.pull-request.outputs.trust }} @@ -192,9 +219,9 @@ jobs: local updated_at="$(date -u '+%Y-%m-%d %H:%M UTC')" local marker="" local docs_url="${preview_url%/}/docs" - local components="English: $english_state; translations: Production fallback" + local components="English: $english_state; $REMOTE_SUMMARY; translations: Production fallback" if [[ -n "${translations_url:-}" ]]; then - components="English: $english_state; [translations]($translations_url): $translations_state" + components="English: $english_state; $REMOTE_SUMMARY; [translations]($translations_url): $translations_state" fi local body local comment_id @@ -406,6 +433,8 @@ jobs: --arg project_id "$VERCEL_PROJECT_ID" \ --arg project_name "$project_name" \ --arg pull_request "$PULL_REQUEST_NUMBER" \ + --arg remote_scope "$REMOTE_SCOPE" \ + --arg preview_environment "$PREVIEW_ENVIRONMENT" \ --arg repository "$repository_name" \ --arg repository_owner "$repository_owner" \ --arg trust "$TRUST" \ @@ -423,7 +452,7 @@ jobs: DOCS_DEPLOY_TARGET: "english", DOCS_LOCALES: "none", DOCS_AVAILABLE_LOCALES: "all", - DOCS_REMOTES: "none" + DOCS_REMOTES: $remote_scope }}, meta: { buildScope: "english-preview", @@ -434,7 +463,8 @@ jobs: approvedSha: $approved_sha, trust: $trust } - }' + } + + (if $preview_environment == "connect-preview" then {customEnvironmentSlugOrId: $preview_environment} else {} end)' )" if ! english_response="$( diff --git a/bin/check-gt-navigation.ts b/bin/check-gt-navigation.ts index 250ad0c18..74de5526d 100644 --- a/bin/check-gt-navigation.ts +++ b/bin/check-gt-navigation.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { readScope } from "../src/lib/scope.ts"; const repositoryRoot = process.cwd(); const configPath = path.join(repositoryRoot, "gt.config.json"); @@ -23,6 +24,35 @@ function readJson(filePath: string): unknown { return JSON.parse(fs.readFileSync(filePath, "utf8")); } +function omittedRemoteMounts(): Set { + const manifest = asRecord( + readJson(path.join(repositoryRoot, "remotes.json")), + "remotes.json", + ); + if (!Array.isArray(manifest.remotes)) { + fail("remotes.json must contain a remotes array"); + } + + const scope = readScope(repositoryRoot); + const mounts = new Set(); + for (const [index, remote] of manifest.remotes.entries()) { + const definition = asRecord(remote, `remotes.json remote ${index}`); + const name = definition.name; + const mount = definition.mount; + if (typeof name !== "string" || !name.trim()) { + fail(`remotes.json remote ${index} must have a non-empty name`); + } + if (typeof mount !== "string" || !mount.trim()) { + fail(`remotes.json remote ${index} must have a non-empty mount`); + } + + const included = scope.remotes + && (!scope.remotePreview || scope.remotePreview.name === name); + if (!included) mounts.add(mount.replace(/^\/+|\/+$/g, "")); + } + return mounts; +} + function asRecord( value: unknown, description: string, @@ -163,6 +193,7 @@ function assertNavigationCoverage(): void { } const visitedFiles = new Set(); + const omittedMounts = omittedRemoteMounts(); let labelCount = 0; function visit(value: unknown, sourceDirectory: string): void { @@ -181,6 +212,10 @@ function assertNavigationCoverage(): void { fail(`navigation reference escapes the repository: ${record.$ref}`); } if (!fs.existsSync(referencedPath)) { + const isOmittedRemote = [...omittedMounts].some((mount) => { + return relativePath === mount || relativePath.startsWith(`${mount}/`); + }); + if (isOmittedRemote) return; fail(`navigation reference does not exist: ${relativePath}`); } if (visitedFiles.has(referencedPath)) return; diff --git a/bin/fetch-remotes.ts b/bin/fetch-remotes.ts index 835bb82ec..2ca321b8c 100644 --- a/bin/fetch-remotes.ts +++ b/bin/fetch-remotes.ts @@ -1,9 +1,8 @@ // Fetches docs owned by other repositories into their mount directories so the // primary collection builds them at their current URLs. Sources, in order: -// 1. a GitHub tarball authenticated by a Vercel Connect token, -// 2. a GitHub tarball authenticated by GH_TOKEN / GITHUB_TOKEN outside Vercel, -// 3. an anonymous GitHub tarball for public repositories, -// 4. a shallow GitHub SSH checkout for local development. +// 1. a shallow, blob-filtered Git checkout with only the configured path, +// 2. a GitHub tarball fallback when sparse checkout is unavailable, +// 3. a shallow, blob-filtered Git SSH checkout for local private remotes. // Production always reads each repository's `main` branch. A source-repository // preview replaces exactly one source with an immutable commit SHA and omits // the other remote sources. Base-repository previews omit every remote. Only @@ -35,10 +34,12 @@ fs.mkdirSync(stateDir, { recursive: true }); type Authentication = "anonymous" | "environment-token" | "vercel-connect"; -async function githubAuthentication(remote: Remote, repository: string): Promise<{ +interface GitHubAuthentication { authentication: Authentication; token?: string; -}> { +} + +async function githubAuthentication(remote: Remote, repository: string): Promise { const environmentToken = (process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN ?? "").trim(); if (environmentToken) { if (process.env.VERCEL === "1") { @@ -78,13 +79,52 @@ async function githubAuthentication(remote: Remote, repository: string): Promise return { authentication: "anonymous" }; } +function gitEnvironment(token?: string): NodeJS.ProcessEnv { + if (!token) return process.env; + return { + ...process.env, + // Keep the token out of command arguments and repository configuration. + // This child is the only process allowed to use remote credentials. + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "http.https://github.com/.extraHeader", + GIT_CONFIG_VALUE_0: `Authorization: Bearer ${token}`, + }; +} + +function checkoutSparseGit( + repositoryUrl: string, + ref: string, + sourcePath: string, + destination: string, + token?: string, +): string { + const environment = gitEnvironment(token); + const checkout = path.join(destination, "checkout"); + const git = (args: string[]) => execFileSync("git", args, { env: environment, stdio: "pipe" }); + + git(["init", checkout]); + git(["-C", checkout, "remote", "add", "origin", repositoryUrl]); + if (sourcePath !== ".") { + git(["-C", checkout, "sparse-checkout", "init", "--cone"]); + git(["-C", checkout, "sparse-checkout", "set", "--cone", sourcePath]); + } + // Fetching the requested ref directly supports immutable pull-request SHAs + // as well as the normal main branch. With sparse checkout enabled, Git only + // downloads blobs required by sourcePath during checkout. + git(["-C", checkout, "fetch", "--depth", "1", "--filter=blob:none", "origin", ref]); + git(["-C", checkout, "checkout", "--detach", "FETCH_HEAD"]); + return path.join(checkout, sourcePath); +} + async function downloadGitHubArchive( remote: Remote, repository: string, ref: string, destination: string, + credentials: GitHubAuthentication, ): Promise { - let { authentication, token } = await githubAuthentication(remote, repository); + const { authentication } = credentials; + let token = credentials.token; try { if (remote.private && !token) { throw new Error( @@ -207,43 +247,48 @@ for (const r of manifest.remotes) { } let source: string | null = null; - let sourceKind: "github-api" | "github-ssh" | null = null; + let sourceKind: "github-sparse" | "github-api" | "github-ssh-sparse" | null = null; let authentication: Authentication | "ssh" | null = null; let temporaryDirectory: string | null = null; let commit = "unknown"; try { - const canUseGitHubApi = Boolean( - process.env.GH_TOKEN - || process.env.GITHUB_TOKEN - || process.env.DOCS_GITHUB_CONNECTOR - || process.env.VERCEL_OIDC_TOKEN - || !r.private, - ); - if (canUseGitHubApi) { + const credentials = await githubAuthentication(r, sourceRepository); + if (!r.private || credentials.token) { const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`)); temporaryDirectory = tmp; - const tar = path.join(tmp, "src.tgz"); - authentication = await downloadGitHubArchive(r, sourceRepository, ref, tar); - execFileSync("tar", ["-xzf", tar, "-C", tmp]); - const extracted = fs.readdirSync(tmp).find((directory) => directory !== "src.tgz"); - if (!extracted) throw new Error(`fetch-remotes: archive for ${r.name} contained no root directory`); - source = path.join(tmp, extracted, r.path); - sourceKind = "github-api"; - commit = extracted.split("-").pop() ?? "unknown"; + try { + source = checkoutSparseGit( + `https://github.com/${sourceRepository}.git`, + ref, + r.path, + tmp, + credentials.token, + ); + sourceKind = "github-sparse"; + authentication = credentials.authentication; + commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + } catch { + // GitHub archives remain a portable fallback for hosts without partial + // clone support. Do not surface child-process output because it may + // include transport details from an authenticated request. + fs.rmSync(path.join(tmp, "checkout"), { recursive: true, force: true }); + const tar = path.join(tmp, "src.tgz"); + authentication = await downloadGitHubArchive(r, sourceRepository, ref, tar, credentials); + execFileSync("tar", ["-xzf", tar, "-C", tmp]); + const extracted = fs.readdirSync(tmp).find((directory) => directory !== "src.tgz"); + if (!extracted) throw new Error(`fetch-remotes: archive for ${r.name} contained no root directory`); + source = path.join(tmp, extracted, r.path); + sourceKind = "github-api"; + commit = extracted.split("-").pop() ?? "unknown"; + } } else if (!process.env.CI) { const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`)); temporaryDirectory = tmp; - const checkout = path.join(tmp, "checkout"); - execFileSync( - "git", - ["clone", "--depth", "1", "--branch", ref, `git@github.com:${sourceRepository}.git`, checkout], - { stdio: "inherit" }, - ); - source = path.join(checkout, r.path); - sourceKind = "github-ssh"; + source = checkoutSparseGit(`git@github.com:${sourceRepository}.git`, ref, r.path, tmp); + sourceKind = "github-ssh-sparse"; authentication = "ssh"; - commit = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); } if (!source) { diff --git a/bin/gen-sidebar.ts b/bin/gen-sidebar.ts index 9fd391ce5..e78c0ada3 100644 --- a/bin/gen-sidebar.ts +++ b/bin/gen-sidebar.ts @@ -35,6 +35,44 @@ fs.mkdirSync(outDir, { recursive: true }); function readJson(p: string): Json { return JSON.parse(fs.readFileSync(p, "utf8")); } +const remotesFile = path.join(root, "remotes.json"); +const remotes: Array<{ name: string; label: string; repo: string; mount: string; sourceRef: string }> = fs.existsSync(remotesFile) + ? (readJson(remotesFile) as { remotes: Array<{ name: string; label: string; repo: string; mount: string }> }).remotes.map((r) => ({ + ...r, + // Mintlify's sourceRef names the GitHub repository (sometimes under an older name). + sourceRef: r.repo, + })) + : []; +const previewRemote = scope.remotePreview ? remotes.find((remote) => remote.name === scope.remotePreview?.name) : undefined; +if (scope.remotePreview && !previewRemote) { + throw new Error(`gen-sidebar: preview scope names unknown remote "${scope.remotePreview.name}"`); +} + +function remoteIsOmitted(remote: (typeof remotes)[number]): boolean { + if (previewRemote && previewRemote.name !== remote.name) return true; + const stateFile = path.join(root, ".remote", `${remote.name}.json`); + if (!fs.existsSync(stateFile)) return false; + const state = readJson(stateFile) as Obj; + if (!state.skipped) return false; + const reason = String(state.reason ?? ""); + if ( + reason !== "excluded-from-base-preview" + && reason !== "excluded-from-remote-preview" + && reason !== "excluded-from-untrusted-vercel-preview" + ) { + throw new Error(`gen-sidebar: remote ${remote.name} has an unknown omission reason: ${reason}`); + } + return true; +} + +function isInOmittedRemote(filePath: string): boolean { + const relativePath = path.relative(root, filePath).replaceAll("\\", "/"); + return remotes.some((remote) => { + return remoteIsOmitted(remote) + && (relativePath === remote.mount || relativePath.startsWith(`${remote.mount}/`)); + }); +} + function resolveRefs(node: Json, baseDir: string): Json { if (Array.isArray(node)) return node.map((n) => resolveRefs(n, baseDir)); if (node && typeof node === "object") { @@ -43,6 +81,7 @@ function resolveRefs(node: Json, baseDir: string): Json { // `{ "$ref": "./x.json" }` is replaced by the file; sibling keys (e.g. // `{ "language": "es", "$ref": "./es/docs.json" }`) are kept on top. const p = path.normalize(path.join(baseDir, o.$ref)); + if (!fs.existsSync(p) && isInOmittedRemote(p)) return null; const target = resolveRefs(readJson(p), path.dirname(p)); const rest = Object.fromEntries(Object.entries(o).filter(([k]) => k !== "$ref").map(([k, v]) => [k, resolveRefs(v, baseDir)])); return target && typeof target === "object" && !Array.isArray(target) ? { ...(target as Obj), ...rest } : target; @@ -133,37 +172,6 @@ function configuredLink(link: string): string { return pageLink(link.replace(/^\/+/, "")); } -// ---------------------------------------------------------------- remotes -const remotesFile = path.join(root, "remotes.json"); -const remotes: Array<{ name: string; label: string; repo: string; mount: string; sourceRef: string }> = fs.existsSync(remotesFile) - ? (readJson(remotesFile) as { remotes: Array<{ name: string; label: string; repo: string; mount: string }> }).remotes.map((r) => ({ - ...r, - // Mintlify's sourceRef names the GitHub repository (sometimes under an older name). - sourceRef: r.repo, - })) - : []; -const previewRemote = scope.remotePreview ? remotes.find((remote) => remote.name === scope.remotePreview?.name) : undefined; -if (scope.remotePreview && !previewRemote) { - throw new Error(`gen-sidebar: preview scope names unknown remote "${scope.remotePreview.name}"`); -} - -function remoteIsOmitted(remote: (typeof remotes)[number]): boolean { - if (previewRemote && previewRemote.name !== remote.name) return true; - const stateFile = path.join(root, ".remote", `${remote.name}.json`); - if (!fs.existsSync(stateFile)) return false; - const state = readJson(stateFile) as Obj; - if (!state.skipped) return false; - const reason = String(state.reason ?? ""); - if ( - reason !== "excluded-from-base-preview" - && reason !== "excluded-from-remote-preview" - && reason !== "excluded-from-untrusted-vercel-preview" - ) { - throw new Error(`gen-sidebar: remote ${remote.name} has an unknown omission reason: ${reason}`); - } - return true; -} - /** Prefix every page path in a remote navigation tree with its mount directory. */ function prefixPages(nodes: Json[], mount: string): Json[] { return nodes.map((n) => { diff --git a/remotes.json b/remotes.json index b007d4111..c12c72339 100644 --- a/remotes.json +++ b/remotes.json @@ -15,6 +15,14 @@ } ], "private": true + }, + { + "name": "dbt-clickhouse", + "label": "dbt", + "repo": "ClickHouse/dbt-clickhouse", + "path": "docs", + "mount": "integrations/connectors/data-ingestion/etl-tools/dbt", + "editPattern": "https://github.com/{repo}/edit/{ref}/{path}" } ] } diff --git a/src/README.md b/src/README.md index a99fb70fe..a47bc48f7 100644 --- a/src/README.md +++ b/src/README.md @@ -28,7 +28,7 @@ Mintlify-flavoured MDX build (see `src/plugins/vite-mintlify-snippets.ts` and | `DOCS_INCLUDE` | Comma-separated globs restricting the English collection (spikes, scoped previews). | | `DOCS_LOCALE` | A singular locale build (`en`, `es`, `pt-BR`, and so on); normally set only by the Vercel shard orchestrator. | | `DOCS_LOCALES` | Translations included in a `translations` or legacy `combined` artifact: `none`, `all`, or a comma-separated list such as `es,fr`. The translations project uses `all`. | -| `DOCS_REMOTES` | Registered remote-source scope: `none` for a base-repository preview and `all` for source previews and production. | +| `DOCS_REMOTES` | Registered remote-source scope: `none` for an ordinary base-repository preview and `all` for source previews, production, and a base preview carrying `docs-preview-show-remotes`. | | `DOCS_REMOTE_NAME`, `DOCS_REMOTE_REPOSITORY`, `DOCS_REMOTE_REF` | CI-only tuple selecting one registered remote at an immutable commit for an English pull-request preview. | | `DOCS_REMOTE_SOURCE_REPOSITORY` | CI-derived repository that owns the preview SHA. It defaults to the registered repository and differs only for a fork PR. | | `DOCS_REMOTES_PREFETCHED=1` | Requires the remote mounts and fetch-state files supplied by the credentialed CI fetch job. | @@ -68,7 +68,13 @@ registered source, repository, and open pull request number. Standard Vercel Preview deployments are deliberately tokenless. Base-repository pull requests use this environment and set `DOCS_REMOTES=none`, regardless of -whether their head branch belongs to the primary repository or a fork. +whether their head branch belongs to the primary repository or a fork. To inspect +remote navigation and content in a base-repository preview, apply the +`docs-preview-show-remotes` label to a pull request from a trusted +`ClickHouse/mintlify-docs-dev` branch. The preview then runs in the +`connect-preview` environment with `DOCS_REMOTES=all`. The workflow rejects that +label on fork pull requests, because the connected environment may read private +remote repositories. Nimbus application pull requests use `.github/workflows/site-preview.yml`. The base-branch workflow resolves GitHub's immutable @@ -140,7 +146,9 @@ Vercel must be provisioned as follows: not in standard Preview. 7. Keep standard Preview free of secrets and privileged integrations. Every base-repository pull request builds from the primary repository's synthetic - merge ref in this environment and omits registered remotes. + merge ref in this environment and omits registered remotes by default. A + trusted-branch pull request with the `docs-preview-show-remotes` label instead + uses `connect-preview` to fetch all registered remotes. 8. Add `VERCEL_TOKEN`, `VERCEL_ORG_ID`, and `VERCEL_PROJECT_ID` as repository secrets under GitHub Actions. Add the translations project's ID as the repository variable `VERCEL_TRANSLATIONS_PROJECT_ID`; project IDs are not