Docs: add dbt-clickhouse remote

.github/workflows/site-preview.yml

@@ -26,7 +26,8 @@ jobs:
     if: >-
       github.event_name == 'workflow_dispatch' ||
       (github.event.action != 'labeled' && github.event.action != 'unlabeled') ||
-      startsWith(github.event.label.name, 'docs-translations-')
+      startsWith(github.event.label.name, 'docs-translations-') ||
+      github.event.label.name == 'docs-preview-show-remotes'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -114,20 +115,26 @@ jobs:
           echo "merge_sha=$merge_sha" >> "$GITHUB_OUTPUT"
           echo "trust=$trust" >> "$GITHUB_OUTPUT"
 
-      - name: Select translated collections from pull request labels
-        id: translations
+      - name: Select preview options from pull request labels
+        id: preview-options
         env:
           GH_TOKEN: ${{ github.token }}
           PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }}
+          TRUST: ${{ steps.pull-request.outputs.trust }}
         run: |
           mapfile -t labels < <(
             gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST_NUMBER/labels" --jq '.[].name'
           )
           known_locales=(ar es fr ja ko pt-BR ru zh)
           include_translations=false
+          include_remotes=false
 
           for label in "${labels[@]}"; do
             normalized="${label,,}"
+            if [[ "$normalized" == "docs-preview-show-remotes" ]]; then
+              include_remotes=true
+              continue
+            fi
             if [[ "$normalized" == "docs-translations-all" ]]; then
               include_translations=true
               continue
@@ -159,8 +166,25 @@ jobs:
             locale_summary="English only"
           fi
 
+          if [[ "$include_remotes" == true ]]; then
+            if [[ "$TRUST" != "trusted-branch" ]]; then
+              echo "docs-preview-show-remotes can only be used for pull requests from ClickHouse/mintlify-docs-dev." >&2
+              exit 1
+            fi
+            remote_scope="all"
+            preview_environment="connect-preview"
+            remote_summary="all registered remotes"
+          else
+            remote_scope="none"
+            preview_environment=""
+            remote_summary="no remotes"
+          fi
+
           echo "scope=$locale_scope" >> "$GITHUB_OUTPUT"
           echo "summary=$locale_summary" >> "$GITHUB_OUTPUT"
+          echo "remote_scope=$remote_scope" >> "$GITHUB_OUTPUT"
+          echo "preview_environment=$preview_environment" >> "$GITHUB_OUTPUT"
+          echo "remote_summary=$remote_summary" >> "$GITHUB_OUTPUT"
 
       # GitHub owns refs/pull/<number>/merge in the primary repository, even
       # when the pull request head is in a fork. Vercel fetches that immutable
@@ -174,8 +198,11 @@ jobs:
           HEAD_REPOSITORY: ${{ steps.pull-request.outputs.head_repository }}
           HEAD_SHA: ${{ steps.pull-request.outputs.head_sha }}
           PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || inputs.pull_request_number }}
-          TRANSLATION_SCOPE: ${{ steps.translations.outputs.scope }}
-          TRANSLATION_SUMMARY: ${{ steps.translations.outputs.summary }}
+          TRANSLATION_SCOPE: ${{ steps.preview-options.outputs.scope }}
+          TRANSLATION_SUMMARY: ${{ steps.preview-options.outputs.summary }}
+          REMOTE_SCOPE: ${{ steps.preview-options.outputs.remote_scope }}
+          PREVIEW_ENVIRONMENT: ${{ steps.preview-options.outputs.preview_environment }}
+          REMOTE_SUMMARY: ${{ steps.preview-options.outputs.remote_summary }}
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           TRUST: ${{ steps.pull-request.outputs.trust }}
@@ -192,9 +219,9 @@ jobs:
             local updated_at="$(date -u '+%Y-%m-%d %H:%M UTC')"
             local marker="<!-- nimbus-site-preview -->"
             local docs_url="${preview_url%/}/docs"
-            local components="English: $english_state; translations: Production fallback"
+            local components="English: $english_state; $REMOTE_SUMMARY; translations: Production fallback"
             if [[ -n "${translations_url:-}" ]]; then
-              components="English: $english_state; [translations]($translations_url): $translations_state"
+              components="English: $english_state; $REMOTE_SUMMARY; [translations]($translations_url): $translations_state"
             fi
             local body
             local comment_id
@@ -406,6 +433,8 @@ jobs:
               --arg project_id "$VERCEL_PROJECT_ID" \
               --arg project_name "$project_name" \
               --arg pull_request "$PULL_REQUEST_NUMBER" \
+              --arg remote_scope "$REMOTE_SCOPE" \
+              --arg preview_environment "$PREVIEW_ENVIRONMENT" \
               --arg repository "$repository_name" \
               --arg repository_owner "$repository_owner" \
               --arg trust "$TRUST" \
@@ -423,7 +452,7 @@ jobs:
                   DOCS_DEPLOY_TARGET: "english",
                   DOCS_LOCALES: "none",
                   DOCS_AVAILABLE_LOCALES: "all",
-                  DOCS_REMOTES: "none"
+                  DOCS_REMOTES: $remote_scope
                 }},
                 meta: {
                   buildScope: "english-preview",
@@ -434,7 +463,8 @@ jobs:
                   approvedSha: $approved_sha,
                   trust: $trust
                 }
-              }'
+              }
+              + (if $preview_environment == "connect-preview" then {customEnvironmentSlugOrId: $preview_environment} else {} end)'
           )"
 
           if ! english_response="$(

bin/check-gt-navigation.ts

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
+import { readScope } from "../src/lib/scope.ts";
 
 const repositoryRoot = process.cwd();
 const configPath = path.join(repositoryRoot, "gt.config.json");
@@ -23,6 +24,35 @@ function readJson(filePath: string): unknown {
   return JSON.parse(fs.readFileSync(filePath, "utf8"));
 }
 
+function omittedRemoteMounts(): Set<string> {
+  const manifest = asRecord(
+    readJson(path.join(repositoryRoot, "remotes.json")),
+    "remotes.json",
+  );
+  if (!Array.isArray(manifest.remotes)) {
+    fail("remotes.json must contain a remotes array");
+  }
+
+  const scope = readScope(repositoryRoot);
+  const mounts = new Set<string>();
+  for (const [index, remote] of manifest.remotes.entries()) {
+    const definition = asRecord(remote, `remotes.json remote ${index}`);
+    const name = definition.name;
+    const mount = definition.mount;
+    if (typeof name !== "string" || !name.trim()) {
+      fail(`remotes.json remote ${index} must have a non-empty name`);
+    }
+    if (typeof mount !== "string" || !mount.trim()) {
+      fail(`remotes.json remote ${index} must have a non-empty mount`);
+    }
+
+    const included = scope.remotes
+      && (!scope.remotePreview || scope.remotePreview.name === name);
+    if (!included) mounts.add(mount.replace(/^\/+|\/+$/g, ""));
+  }
+  return mounts;
+}
+
 function asRecord(
   value: unknown,
   description: string,
@@ -163,6 +193,7 @@ function assertNavigationCoverage(): void {
   }
 
   const visitedFiles = new Set<string>();
+  const omittedMounts = omittedRemoteMounts();
   let labelCount = 0;
 
   function visit(value: unknown, sourceDirectory: string): void {
@@ -181,6 +212,10 @@ function assertNavigationCoverage(): void {
         fail(`navigation reference escapes the repository: ${record.$ref}`);
       }
       if (!fs.existsSync(referencedPath)) {
+        const isOmittedRemote = [...omittedMounts].some((mount) => {
+          return relativePath === mount || relativePath.startsWith(`${mount}/`);
+        });
+        if (isOmittedRemote) return;
         fail(`navigation reference does not exist: ${relativePath}`);
       }
       if (visitedFiles.has(referencedPath)) return;

bin/fetch-remotes.ts

@@ -1,9 +1,8 @@
 // Fetches docs owned by other repositories into their mount directories so the
 // primary collection builds them at their current URLs. Sources, in order:
-//   1. a GitHub tarball authenticated by a Vercel Connect token,
-//   2. a GitHub tarball authenticated by GH_TOKEN / GITHUB_TOKEN outside Vercel,
-//   3. an anonymous GitHub tarball for public repositories,
-//   4. a shallow GitHub SSH checkout for local development.
+//   1. a shallow, blob-filtered Git checkout with only the configured path,
+//   2. a GitHub tarball fallback when sparse checkout is unavailable,
+//   3. a shallow, blob-filtered Git SSH checkout for local private remotes.
 // Production always reads each repository's `main` branch. A source-repository
 // preview replaces exactly one source with an immutable commit SHA and omits
 // the other remote sources. Base-repository previews omit every remote. Only
@@ -35,10 +34,12 @@ fs.mkdirSync(stateDir, { recursive: true });
 
 type Authentication = "anonymous" | "environment-token" | "vercel-connect";
 
-async function githubAuthentication(remote: Remote, repository: string): Promise<{
+interface GitHubAuthentication {
   authentication: Authentication;
   token?: string;
-}> {
+}
+
+async function githubAuthentication(remote: Remote, repository: string): Promise<GitHubAuthentication> {
   const environmentToken = (process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN ?? "").trim();
   if (environmentToken) {
     if (process.env.VERCEL === "1") {
@@ -78,13 +79,52 @@ async function githubAuthentication(remote: Remote, repository: string): Promise
   return { authentication: "anonymous" };
 }
 
+function gitEnvironment(token?: string): NodeJS.ProcessEnv {
+  if (!token) return process.env;
+  return {
+    ...process.env,
+    // Keep the token out of command arguments and repository configuration.
+    // This child is the only process allowed to use remote credentials.
+    GIT_CONFIG_COUNT: "1",
+    GIT_CONFIG_KEY_0: "http.https://github.com/.extraHeader",
+    GIT_CONFIG_VALUE_0: `Authorization: Bearer ${token}`,
+  };
+}
+
+function checkoutSparseGit(
+  repositoryUrl: string,
+  ref: string,
+  sourcePath: string,
+  destination: string,
+  token?: string,
+): string {
+  const environment = gitEnvironment(token);
+  const checkout = path.join(destination, "checkout");
+  const git = (args: string[]) => execFileSync("git", args, { env: environment, stdio: "pipe" });
+
+  git(["init", checkout]);
+  git(["-C", checkout, "remote", "add", "origin", repositoryUrl]);
+  if (sourcePath !== ".") {
+    git(["-C", checkout, "sparse-checkout", "init", "--cone"]);
+    git(["-C", checkout, "sparse-checkout", "set", "--cone", sourcePath]);
+  }
+  // Fetching the requested ref directly supports immutable pull-request SHAs
+  // as well as the normal main branch. With sparse checkout enabled, Git only
+  // downloads blobs required by sourcePath during checkout.
+  git(["-C", checkout, "fetch", "--depth", "1", "--filter=blob:none", "origin", ref]);
+  git(["-C", checkout, "checkout", "--detach", "FETCH_HEAD"]);
+  return path.join(checkout, sourcePath);
+}
+
 async function downloadGitHubArchive(
   remote: Remote,
   repository: string,
   ref: string,
   destination: string,
+  credentials: GitHubAuthentication,
 ): Promise<Authentication> {
-  let { authentication, token } = await githubAuthentication(remote, repository);
+  const { authentication } = credentials;
+  let token = credentials.token;
   try {
     if (remote.private && !token) {
       throw new Error(
@@ -207,43 +247,48 @@ for (const r of manifest.remotes) {
   }
 
   let source: string | null = null;
-  let sourceKind: "github-api" | "github-ssh" | null = null;
+  let sourceKind: "github-sparse" | "github-api" | "github-ssh-sparse" | null = null;
   let authentication: Authentication | "ssh" | null = null;
   let temporaryDirectory: string | null = null;
   let commit = "unknown";
 
   try {
-    const canUseGitHubApi = Boolean(
-      process.env.GH_TOKEN
-      || process.env.GITHUB_TOKEN
-      || process.env.DOCS_GITHUB_CONNECTOR
-      || process.env.VERCEL_OIDC_TOKEN
-      || !r.private,
-    );
-    if (canUseGitHubApi) {
+    const credentials = await githubAuthentication(r, sourceRepository);
+    if (!r.private || credentials.token) {
       const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`));
       temporaryDirectory = tmp;
-      const tar = path.join(tmp, "src.tgz");
-      authentication = await downloadGitHubArchive(r, sourceRepository, ref, tar);
-      execFileSync("tar", ["-xzf", tar, "-C", tmp]);
-      const extracted = fs.readdirSync(tmp).find((directory) => directory !== "src.tgz");
-      if (!extracted) throw new Error(`fetch-remotes: archive for ${r.name} contained no root directory`);
-      source = path.join(tmp, extracted, r.path);
-      sourceKind = "github-api";
-      commit = extracted.split("-").pop() ?? "unknown";
+      try {
+        source = checkoutSparseGit(
+          `https://github.com/${sourceRepository}.git`,
+          ref,
+          r.path,
+          tmp,
+          credentials.token,
+        );
+        sourceKind = "github-sparse";
+        authentication = credentials.authentication;
+        commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+      } catch {
+        // GitHub archives remain a portable fallback for hosts without partial
+        // clone support. Do not surface child-process output because it may
+        // include transport details from an authenticated request.
+        fs.rmSync(path.join(tmp, "checkout"), { recursive: true, force: true });
+        const tar = path.join(tmp, "src.tgz");
+        authentication = await downloadGitHubArchive(r, sourceRepository, ref, tar, credentials);
+        execFileSync("tar", ["-xzf", tar, "-C", tmp]);
+        const extracted = fs.readdirSync(tmp).find((directory) => directory !== "src.tgz");
+        if (!extracted) throw new Error(`fetch-remotes: archive for ${r.name} contained no root directory`);
+        source = path.join(tmp, extracted, r.path);
+        sourceKind = "github-api";
+        commit = extracted.split("-").pop() ?? "unknown";
+      }
     } else if (!process.env.CI) {
       const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`));
       temporaryDirectory = tmp;
-      const checkout = path.join(tmp, "checkout");
-      execFileSync(
-        "git",
-        ["clone", "--depth", "1", "--branch", ref, `git@github.com:${sourceRepository}.git`, checkout],
-        { stdio: "inherit" },
-      );
-      source = path.join(checkout, r.path);
-      sourceKind = "github-ssh";
+      source = checkoutSparseGit(`git@github.com:${sourceRepository}.git`, ref, r.path, tmp);
+      sourceKind = "github-ssh-sparse";
       authentication = "ssh";
-      commit = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+      commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
     }
 
     if (!source) {