diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d97e02f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + cooldown: + default-days: 7 + groups: + github-actions: + patterns: ["*"] + - package-ecosystem: npm + directory: / + schedule: + interval: monthly + cooldown: + default-days: 7 + groups: + docusaurus: + patterns: ["@docusaurus/*"] + open-pull-requests-limit: 5 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..93aa44f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,16 @@ +## What and why + + + +## Checklist + +- [ ] Every new or changed fact is verified (code at `origin/main`, a test, or engineering), per region where it can differ (AGENTS.md) +- [ ] Other pages that restate the same fact are updated too (`grep -rn` the old wording across `docs/`, `static/`, `docusaurus.config.js`) +- [ ] Destructive, irreversible, or cost-affecting steps carry a `:::warning` / `:::danger` + +## Mirrors + + + +- [ ] `static/llms.txt` and `static/llms-full.txt` updated +- [ ] No mirror change needed: diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..03e0a2b --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,72 @@ +name: Build + +# Runs on every PR and on pushes to main. Holds no secrets: a PR's MDX runs +# during the build. See CONTRIBUTING.md for what each check catches. + +on: + pull_request: + types: [opened, synchronize, reopened, edited, ready_for_review] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 + with: + node-version: 24 + cache: npm + + - run: npm ci --ignore-scripts --no-audit --no-fund + + # MDX errors, broken internal links (onBrokenLinks: throw), bad frontmatter. + - run: npm run build + + # Every page has an entry in both llms mirrors, and the mirrors link no dead pages. + - run: node scripts/check-mirrors.mjs + + # Python/JS/Go/Bash/JSON snippets parse; no untagged blocks; no real-looking keys. + # Python, Go and Bash come preinstalled on ubuntu-latest. + - run: node scripts/check-snippets.mjs + + # A PR that changes docs/** but neither mirror must say why, so mirror drift + # is a decision someone made, not an oversight. Only reads the PR body. + mirror-impact: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Require a mirror update or a reason + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + # Template placeholders () don't count as a reason. + body=$(printf '%s' "$PR_BODY" | perl -0pe 's///gs') + changed=$(git diff --name-only "$BASE_SHA"...HEAD) + if ! grep -q '^docs/' <<<"$changed"; then echo "No docs/** changes."; exit 0; fi + if grep -q '^static/llms' <<<"$changed"; then echo "Mirrors updated."; exit 0; fi + if grep -qiE '^\s*-\s*\[x\]\s*No mirror change needed:\s*\S' <<<"$body"; then + echo "Author says no mirror change is needed."; exit 0 + fi + echo "::error::This PR changes docs/** but not static/llms.txt or static/llms-full.txt." + echo "Update both mirrors (AGENTS.md), or tick 'No mirror change needed:' in the PR description and say why (e.g. a typo fix)." + exit 1 diff --git a/AGENTS.md b/AGENTS.md index 2561b21..4714e00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,3 +41,15 @@ npm run build ``` This catches MDX syntax errors (broken JSX, unclosed tags, invalid frontmatter) that won't show up just from reading the file. + +## Checks that run on every PR + +`.github/workflows/build.yaml` runs these, and you can run them locally: + +```bash +npm run build # MDX errors and broken internal links +node scripts/check-mirrors.mjs # every page has an entry in both llms mirrors +node scripts/check-snippets.mjs # snippets parse, no untagged blocks, no real-looking keys +``` + +A PR that changes `docs/**` but neither mirror fails unless its description ticks **No mirror change needed** with a reason. None of these checks can tell whether a fact is *true*; that is what the rules above and [CONTRIBUTING.md](CONTRIBUTING.md) are for. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..aba3651 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,65 @@ +# Contributing to the Fil One docs + +The docs are part of the product. A reader should be able to trust every page as much as the console it describes. [`AGENTS.md`](AGENTS.md) has the accuracy rules: verify facts, check each region, keep the surfaces separate, and keep the mirrors in sync. This page covers **when** docs must change and **how** we keep them from drifting. + +## When a docs change is required + +A product change needs a docs change when a customer could see it or rely on it: + +- a console screen, label, button, flow step, empty state or error message +- S3 API behaviour: a supported or unsupported operation, a header, an error code, a region difference +- a limit, a price, a plan or trial term, a region +- a security property: key permissions, MFA, encryption, retention, deletion + +Refactors, tests, internal tooling and changes behind a feature flag that is off in production don't need one. Feature-flagged work gets its docs at launch: a public docs PR announces the feature. + +**Definition of done for a user-facing product PR:** the product PR links a filone-docs PR, or says why no docs change is needed. That requirement lives in the product repo's PR template (see [Product-side check](#product-side-check)). + +## Where a product change usually lands in the docs + +Use this map to find the pages to check. It is a starting point, not a complete list, so grep the docs for the label or fact you changed. + +| Product change (`fil-one/fil-one`) | Docs to check | +|---|---| +| `packages/website/src/routes/_app/buckets*`, bucket pages and components | `docs/app/buckets.mdx`, `docs/storage/buckets.mdx`, `docs/storage/objects.mdx`, `docs/quickstart/index.mdx` | +| `.../_app/api-keys*`, `packages/shared/src/access-key-permissions.ts` | `docs/app/api-keys.mdx`, `docs/security/api-keys.mdx`, `docs/security/authentication.mdx`, `docs/sdk/ai-agents.mdx` | +| `.../_app/organization*`, `members*`, `edit-organization*`, `packages/shared/src/permissions.ts` | `docs/app/organization.mdx`, `docs/faq.mdx` | +| `.../_app/billing*`, `packages/shared/src/api/billing.ts`, `usage.ts`, plan constants | `docs/app/billing.mdx`, `docs/billing/*`, `docs/limits.mdx`, JSON-LD in `docusaurus.config.js` | +| `.../_app/dashboard*`, `get-started*` | `docs/app/dashboard.mdx`, `docs/quickstart/index.mdx` | +| `.../_app/settings*`, auth / MFA / passkeys (`packages/shared/src/auth.ts`) | `docs/app/settings.mdx`, `docs/security/account-security.mdx` | +| `.../_app/support*` | `docs/app/support.mdx` | +| Regions, endpoints, S3 gateway behaviour, per-region grants | `docs/reference/s3-compatibility.mdx` first, then every page that restates it (quickstart, `docs/sdk/*`, `docs/reference/overview.mdx`) | +| Object Lock, versioning, retention, deletion | `docs/storage/object-lock.mdx`, `docs/storage/versioning.mdx`, `docs/faq.mdx` | +| Encryption, compliance claims | `docs/security/encryption.mdx`, `docs/faq.mdx` | + +Every fact you change also goes into `static/llms.txt` and `static/llms-full.txt`. + +## What CI checks, and what it can't + +`.github/workflows/build.yaml` runs on every PR. It has no secrets. + +| Check | Catches | Doesn't catch | +|---|---|---| +| `npm run build` | MDX syntax errors, bad frontmatter, broken internal links (`onBrokenLinks: "throw"`) | Raw `` links in HTML card grids. Docusaurus doesn't check those. | +| `scripts/check-mirrors.mjs` | A page added, renamed or removed without its `llms.txt` link and `llms-full.txt` section; mirror links to pages that no longer exist | A fact that differs between a page and its mirror | +| `scripts/check-snippets.mjs` | Python, JS, Go, Bash and JSON snippets that don't parse; untagged code blocks; values that look like real access keys | Whether a snippet works against Fil One | +| mirror-impact gate | A PR that changes `docs/**` but neither mirror and gives no reason | Whether the reason is right | + +To opt a deliberate fragment out of the snippet check, add `nocheck` to its fence (`` ```python nocheck ``) and explain the fragment in the surrounding text. + +## Testing code snippets + +CI only checks that snippets parse. Before you merge a new or changed snippet, run it: + +1. **Against a local S3 emulator** for request shape: endpoint, path-style addressing, flags, profile config. For example, run `uvx --from 'moto[server]' moto_server -p 5055`, then substitute only the endpoint and dummy credentials. This doesn't prove Fil One behaviour. +2. **Against Fil One, with a throwaway key** for anything region- or behaviour-specific. Use a key scoped to a test bucket, set in environment variables and never in the snippet. Delete the key afterwards. Never paste a real key into a doc, an issue, or a chat with an AI tool. + +Not automated yet: running the snippets against Fil One in CI. That needs a scoped test key stored as a CI secret and a dedicated test bucket per region, so it's a decision for the team. It would run on a schedule, not on PRs, so a PR's code never sees the key. + +## Periodic audit + +Checks catch structure; they can't catch a console label that changed in a product PR that skipped its docs box. Run a full docs audit at least once a month and after any large product launch. The audit compares every docs area with the product code at `origin/main` and with the live sites, and writes a report to `audits/`. Today the `docs-audit` Claude Code skill that runs it lives on the docs maintainer's machine, not in this repo. + +## Product-side check + +The product repo gets the other half: a PR template with a **Docs** section, a "Documentation" section in its `AGENTS.md`, and a small `docs-impact` workflow. When a PR touches user-facing paths, the workflow requires its description to link a filone-docs PR, give a reason no docs change is needed, or carry the `docs:gated` label. It holds no secrets and calls no model, so it only makes the decision visible. It can't tell whether the decision is right. That's what the audit is for. diff --git a/README.md b/README.md index 1b10e65..64cea50 100644 --- a/README.md +++ b/README.md @@ -60,12 +60,13 @@ docs/ # All documentation pages (MDX) app/ # Dashboard guides reference/ # API reference, S3 compat, errors sdk/ # SDK examples per language - bucket-intelligence.mdx faq.mdx limits.mdx introduction.mdx docusaurus.config.js # Site config (navbar, theme, JSON-LD structured data) sidebars.js # Sidebar navigation structure +scripts/ # CI checks: mirror coverage, snippet syntax +.github/ # CI workflow, PR template, Dependabot src/ # Custom React components and CSS css/custom.css # Custom styles static/ # Served as-is at the site root @@ -82,4 +83,7 @@ vercel.json # Vercel deployment config 2. Edit or add `.mdx` files in `docs/`. 3. Follow [`AGENTS.md`](./AGENTS.md) — it covers keeping `static/llms.txt` / `static/llms-full.txt` in sync, checking `docusaurus.config.js` for claims outside `docs/**`, and verifying facts before publishing them. 4. Run `npm start` to preview locally. -5. Open a PR. Vercel will create a preview deployment. +5. Run the checks in [`AGENTS.md`](./AGENTS.md#checks-that-run-on-every-pr). CI runs the same ones. +6. Open a PR and fill in the template. Vercel creates a preview deployment. + +[`CONTRIBUTING.md`](./CONTRIBUTING.md) covers when a product change needs a docs change, which pages a product change usually affects, and how to test snippets. diff --git a/scripts/check-mirrors.mjs b/scripts/check-mirrors.mjs new file mode 100644 index 0000000..5eebeec --- /dev/null +++ b/scripts/check-mirrors.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node +// Checks that static/llms.txt and static/llms-full.txt cover the same pages as +// docs/**, so a page can't be added, renamed or removed without its mirrors. +// +// - Every docs page has a link in llms.txt and a "Source: " section in +// llms-full.txt. +// - Every docs.filone.com page URL in either mirror points at a page that exists +// (or at a redirect in vercel.json). +// +// It cannot tell whether a *fact* inside a page and its mirror agree; that is +// what the PR checklist and review are for (see CONTRIBUTING.md). + +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; + +const ROOT = new URL("..", import.meta.url).pathname; +const HOST = "https://docs.filone.com"; + +function walk(dir) { + return readdirSync(dir).flatMap((name) => { + const p = join(dir, name); + return statSync(p).isDirectory() ? walk(p) : /\.mdx?$/.test(p) ? [p] : []; + }); +} + +function routeOf(file) { + const src = readFileSync(file, "utf8"); + const fm = src.match(/^---\n([\s\S]*?)\n---/); + if (fm && /^draft:\s*true/m.test(fm[1])) return null; + const slug = fm?.[1].match(/^slug:\s*["']?([^"'\n]+)/m)?.[1].trim(); + if (slug) return slug.startsWith("/") ? slug : "/" + slug; + return "/" + relative(join(ROOT, "docs"), file).replace(/\.mdx?$/, "").replace(/(^|\/)index$/, ""); +} + +const norm = (u) => (u.replace(/[#?].*$/, "").replace(/\/+$/, "") || "/"); +const pages = new Map(walk(join(ROOT, "docs")).map((f) => [norm(routeOf(f) ?? ""), relative(ROOT, f)])); +pages.delete(""); +const redirects = new Set(JSON.parse(readFileSync(join(ROOT, "vercel.json"), "utf8")).redirects?.map((r) => norm(r.source)) ?? []); + +const llms = readFileSync(join(ROOT, "static/llms.txt"), "utf8"); +const full = readFileSync(join(ROOT, "static/llms-full.txt"), "utf8"); +const urlsIn = (text) => [...text.matchAll(/https:\/\/docs\.filone\.com(\/[^\s)>\]"'`,]*)?/g)].map((m) => norm(m[1] ?? "/")); +const linked = new Set(urlsIn(llms)); +const sources = new Set([...full.matchAll(/^Source:\s*https:\/\/docs\.filone\.com(\/\S*)?/gm)].map((m) => norm(m[1] ?? "/"))); + +const errors = []; +for (const [route, file] of pages) { + if (!linked.has(route)) errors.push(`${file}: no link to ${HOST}${route} in static/llms.txt`); + if (!sources.has(route)) errors.push(`${file}: no "Source: ${HOST}${route}" section in static/llms-full.txt`); +} +const assets = new Set(["/llms.txt", "/llms-full.txt", "/sitemap.xml", "/robots.txt"]); +for (const [name, text] of [["static/llms.txt", llms], ["static/llms-full.txt", full]]) { + for (const route of new Set(urlsIn(text))) { + if (!pages.has(route) && !redirects.has(route) && !assets.has(route)) { + errors.push(`${name}: links to ${HOST}${route}, which is not a docs page`); + } + } +} + +if (errors.length) { + console.error(`Mirror check failed (${errors.length}):\n` + errors.map((e) => " " + e).join("\n")); + console.error("\nAdd or remove the page's entry in both static/llms.txt and static/llms-full.txt (AGENTS.md)."); + process.exit(1); +} +console.log(`Mirror check passed: ${pages.size} pages, each linked in llms.txt and mirrored in llms-full.txt.`); diff --git a/scripts/check-snippets.mjs b/scripts/check-snippets.mjs new file mode 100644 index 0000000..e7ffe7b --- /dev/null +++ b/scripts/check-snippets.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node +// Syntax-checks fenced code blocks in docs/**/*.mdx and scans them for +// credential-shaped values. +// +// What it catches: a Python/JS/Go/Bash/JSON snippet that would not even parse +// (a reader who copies it gets a syntax error), a code block with no language +// tag, and anything that looks like a real access key or secret. +// +// What it does NOT catch: whether a snippet works against Fil One. That needs +// credentials and a live bucket; see CONTRIBUTING.md ("Testing code snippets"). +// +// Opt a single block out by adding `nocheck` to its fence meta: +// ```python nocheck +// Use this only for deliberate fragments, and say why in the surrounding text. +// +// Usage: node scripts/check-snippets.mjs [files...] (default: all docs) + +import { readFileSync, writeFileSync, mkdtempSync, rmSync, readdirSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; + +const ROOT = new URL("..", import.meta.url).pathname; + +function walk(dir) { + return readdirSync(dir).flatMap((name) => { + const p = join(dir, name); + return statSync(p).isDirectory() ? walk(p) : p.endsWith(".mdx") || p.endsWith(".md") ? [p] : []; + }); +} + +function extractBlocks(file) { + const lines = readFileSync(file, "utf8").split("\n"); + const blocks = []; + let open = null; + lines.forEach((line, i) => { + const m = line.match(/^(\s*)(`{3,}|~{3,})(.*)$/); + if (!m) { + if (open) open.body.push(line.slice(Math.min(open.indent, line.length - line.trimStart().length))); + return; + } + if (!open) { + const [lang = "", ...meta] = m[3].trim().split(/\s+/); + open = { fence: m[2], indent: m[1].length, lang: lang.toLowerCase(), meta, line: i + 1, body: [] }; + } else if (m[2].startsWith(open.fence[0]) && m[2].length >= open.fence.length && !m[3].trim()) { + blocks.push({ ...open, code: open.body.join("\n") }); + open = null; + } else { + open.body.push(line); + } + }); + return blocks; +} + +function have(cmd) { + return spawnSync(cmd, ["--version"], { stdio: "ignore" }).status === 0 || + spawnSync(cmd, ["version"], { stdio: "ignore" }).status === 0; +} + +const tools = { python3: have("python3"), bash: have("bash"), gofmt: have("go") }; +const tmp = mkdtempSync(join(tmpdir(), "snippets-")); +let n = 0; + +function run(cmd, args, code, ext) { + const f = join(tmp, `s${n++}${ext}`); + writeFileSync(f, code); + const r = spawnSync(cmd, [...args, f], { encoding: "utf8" }); + return r.status === 0 ? null : (r.stderr || r.stdout).replaceAll(f, "").trim().split("\n").slice(-3).join(" | "); +} + +const checkers = { + json: (c) => { try { JSON.parse(c); return null; } catch (e) { return e.message; } }, + python: (c) => tools.python3 ? run("python3", ["-m", "py_compile"], c, ".py") : undefined, + javascript: (c) => run(process.execPath, ["--check"], c, ".mjs"), + js: (c) => checkers.javascript(c), + bash: (c) => tools.bash ? run("bash", ["-n"], c, ".sh") : undefined, + sh: (c) => checkers.bash(c), + go: (c) => { + if (!tools.gofmt) return undefined; + // Fragments (no package clause) are wrapped in a function body; a leading + // import block stays at file level. + let src = c; + if (!/^\s*package\s+\w+/m.test(c)) { + const m = c.match(/^\s*((?:import\s*(?:\([\s\S]*?\)|"[^"]*")\s*)+)/); + src = `package snippet\n\n${m ? m[1] : ""}\nfunc _() {\n${c.slice(m ? m[0].length : 0)}\n}\n`; + } + return run("gofmt", ["-e", "-l"], src, ".go"); + }, +}; + +// Credential-shaped values. AWS's documented example values contain "EXAMPLE". +const SECRET_PATTERNS = [ + [/\b(AKIA|ASIA)[0-9A-Z]{16}\b/g, "AWS-style access key ID"], + [/(secret[_-]?access[_-]?key|aws_secret_access_key|secretAccessKey)\s*[:=]\s*["']?([A-Za-z0-9/+]{40})\b/gi, "40-char secret key"], + [/-----BEGIN [A-Z ]*PRIVATE KEY-----/g, "private key"], +]; + +const files = process.argv.slice(2).length ? process.argv.slice(2) : walk(join(ROOT, "docs")); +const errors = []; +const skipped = new Set(); +let checked = 0; + +for (const file of files) { + const rel = relative(ROOT, file); + for (const b of extractBlocks(file)) { + const where = `${rel}:${b.line}`; + if (!b.lang) errors.push(`${where} code block has no language tag (use \`text\` for plain output)`); + for (const [re, what] of SECRET_PATTERNS) { + for (const m of b.code.matchAll(re)) { + if (!/EXAMPLE|YOUR_|<|\.\.\./.test(m[0])) errors.push(`${where} looks like a real ${what}; use a placeholder`); + } + } + if (b.meta.includes("nocheck")) continue; + const check = checkers[b.lang]; + if (!check) continue; + const err = check(b.code); + if (err === undefined) { skipped.add(b.lang); continue; } + checked++; + if (err) errors.push(`${where} ${b.lang} does not parse: ${err}`); + } +} + +rmSync(tmp, { recursive: true, force: true }); +for (const lang of skipped) console.warn(`warning: ${lang} toolchain not found; ${lang} snippets were not syntax-checked`); +if (errors.length) { + console.error(`Snippet check failed (${errors.length}):\n` + errors.map((e) => " " + e).join("\n")); + console.error("\nFix the snippet, or add `nocheck` to the fence if it is a deliberate fragment."); + process.exit(1); +} +console.log(`Snippet check passed: ${checked} blocks syntax-checked across ${files.length} files.`);