Migrate documentation from Sphinx to Markdown and Jekyll

.github/workflows/pages.yml

@@ -1,22 +1,70 @@
-name: Documentation on github pages
+name: Documentation
 
 on:
+  pull_request:
   push:
-    branches:
-      - main
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  github-pages:
+  build:
     runs-on: ubuntu-latest
+    env:
+      BUNDLE_WITHOUT: "development:test"
+      JEKYLL_ENV: production
     steps:
       - uses: actions/checkout@v6.0.2
+        with:
+          persist-credentials: false
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Build documentation
+        run: bundle exec jekyll build --source docs --destination build/docs
+
+      - name: Check links and anchors
+        run: bundle exec htmlproofer build/docs --disable-external --allow-missing-href --no-enforce-https --swap-urls '^/ghostferry/main/:/'
 
-      - name: Build documentations
-        run: .github/workflows/build-docs.sh
+      - name: Upload documentation
+        uses: actions/upload-artifact@v7.0.1
+        with:
+          name: docs-html
+          path: build/docs
+          if-no-files-found: error
+          retention-days: 7
 
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'Shopify/ghostferry'
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Download documentation
+        uses: actions/download-artifact@v8.0.1
+        with:
+          name: docs-html
+          path: build/docs
 
-      - name: Deploy github pages
+      - name: Deploy current documentation
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./docs/build/ghostferry-pages
+          publish_branch: gh-pages
+          publish_dir: ./build/docs
+          destination_dir: main
+          keep_files: false
+          force_orphan: false
+          enable_jekyll: false

.github/workflows/tests.yml

@@ -80,7 +80,7 @@ jobs:
 
     env:
       CI: "true"
-      BUNDLE_WITHOUT: "development"
+      BUNDLE_WITHOUT: "development:docs"
       MYSQL_VERSION: ${{ matrix.mysql }}
       GHOSTFERRY_LOG_BACKEND: ${{ matrix.log_backend }}
 

Gemfile

@@ -17,4 +17,12 @@ group :development do
   gem "pry-byebug"
 end
 
+group :docs do
+  gem "jekyll", "~> 4.4"
+  gem "just-the-docs", "~> 0.12"
+  gem "jekyll-relative-links", "~> 0.7"
+  gem "jekyll-optional-front-matter", "~> 0.3"
+  gem "html-proofer", "~> 5.0"
+end
+
 gem "mutex_m", "~> 0.3.0"

Gemfile.lock

@@ -1,14 +1,119 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    Ascii85 (2.0.1)
+    addressable (2.9.0)
+      public_suffix (>= 2.0.2, < 8.0)
+    afm (1.0.0)
     ansi (1.6.0)
+    async (2.46.0)
+      console (~> 1.29)
+      fiber-annotation
+      io-event (~> 1.21)
+    base64 (0.3.0)
+    benchmark (0.5.0)
     bigdecimal (4.1.1)
     builder (3.3.0)
     byebug (13.0.0)
       reline (>= 0.6.0)
     coderay (1.1.3)
+    colorator (1.1.0)
+    concurrent-ruby (1.3.8)
+    console (1.38.0)
+      fiber-annotation
+      fiber-local (~> 1.1)
+      json
+    csv (3.3.6)
+    em-websocket (0.5.3)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0)
+    ethon (0.18.0)
+      ffi (>= 1.15.0)
+      logger
+    eventmachine (1.2.7)
+    ffi (1.17.4)
+    ffi (1.17.4-arm64-darwin)
+    ffi (1.17.4-x86_64-linux-gnu)
+    fiber-annotation (0.2.0)
+    fiber-local (1.1.0)
+      fiber-storage
+    fiber-storage (1.0.1)
+    forwardable-extended (2.6.0)
+    google-protobuf (4.36.2)
+      bigdecimal
+      rake (~> 13.3)
+    google-protobuf (4.36.2-arm64-darwin)
+      bigdecimal
+      rake (~> 13.3)
+    google-protobuf (4.36.2-x86_64-linux-gnu)
+      bigdecimal
+      rake (~> 13.3)
+    hashery (2.1.2)
+    html-proofer (5.2.2)
+      addressable (~> 2.3)
+      async (~> 2.1)
+      benchmark (~> 0.5)
+      nokogiri (~> 1.13)
+      pdf-reader (~> 2.11)
+      rainbow (~> 3.0)
+      typhoeus (~> 1.3)
+      yell (~> 2.0)
+      zeitwerk (~> 2.5)
+    http_parser.rb (0.8.1)
+    i18n (1.15.2)
+      concurrent-ruby (~> 1.0)
     io-console (0.8.2)
+    io-event (1.22.1)
+    jekyll (4.4.1)
+      addressable (~> 2.4)
+      base64 (~> 0.2)
+      colorator (~> 1.0)
+      csv (~> 3.0)
+      em-websocket (~> 0.5)
+      i18n (~> 1.0)
+      jekyll-sass-converter (>= 2.0, < 4.0)
+      jekyll-watch (~> 2.0)
+      json (~> 2.6)
+      kramdown (~> 2.3, >= 2.3.1)
+      kramdown-parser-gfm (~> 1.0)
+      liquid (~> 4.0)
+      mercenary (~> 0.3, >= 0.3.6)
+      pathutil (~> 0.9)
+      rouge (>= 3.0, < 5.0)
+      safe_yaml (~> 1.0)
+      terminal-table (>= 1.8, < 4.0)
+      webrick (~> 1.7)
+    jekyll-include-cache (0.2.2)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-optional-front-matter (0.3.3)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-relative-links (0.8.0)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-sass-converter (3.1.0)
+      sass-embedded (~> 1.75)
+    jekyll-seo-tag (2.9.0)
+      jekyll (>= 3.8, < 5.0)
+    jekyll-watch (2.2.1)
+      listen (~> 3.0)
+    json (2.21.2)
+    just-the-docs (0.12.0)
+      jekyll (>= 3.8.5)
+      jekyll-include-cache
+      jekyll-seo-tag (>= 2.0)
+      rake (>= 12.3.1)
+    kramdown (2.5.2)
+      rexml (>= 3.4.4)
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    liquid (4.0.4)
+    listen (3.10.0)
+      logger
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.7.0)
+    mercenary (0.4.0)
     method_source (1.1.0)
+    mini_portile2 (2.8.9)
     minitest (5.27.0)
     minitest-fail-fast (0.1.0)
       minitest (~> 5)
@@ -24,24 +129,65 @@ GEM
     mutex_m (0.3.0)
     mysql2 (0.5.7)
       bigdecimal
+    nokogiri (1.19.4)
+      mini_portile2 (~> 2.8.2)
+      racc (~> 1.4)
+    nokogiri (1.19.4-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.19.4-x86_64-linux-gnu)
+      racc (~> 1.4)
+    pathutil (0.16.2)
+      forwardable-extended (~> 2.6)
+    pdf-reader (2.16.0)
+      Ascii85 (>= 1.0, < 3.0, != 2.0.0)
+      afm (>= 0.2.1, < 2)
+      hashery (~> 2.0)
+      ttfunk
     pry (0.16.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
       reline (>= 0.6.0)
     pry-byebug (3.12.0)
       byebug (~> 13.0)
       pry (>= 0.13, < 0.17)
+    public_suffix (7.0.5)
+    racc (1.8.1)
+    rainbow (3.1.1)
     rake (13.4.1)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.11.1)
+      ffi (~> 1.0)
     reline (0.6.3)
       io-console (~> 0.5)
+    rexml (3.4.4)
+    rouge (4.7.0)
     ruby-progressbar (1.13.0)
+    safe_yaml (1.0.5)
+    sass-embedded (1.105.0)
+      google-protobuf (~> 4.31)
+      rake (~> 13.3)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
     tqdm (0.4.1)
+    ttfunk (1.7.0)
+    typhoeus (1.6.0)
+      ethon (>= 0.18.0)
+    unicode-display_width (2.6.0)
     webrick (1.9.2)
+    yell (2.2.2)
+    zeitwerk (2.8.3)
 
 PLATFORMS
+  arm64-darwin-23
   ruby
+  x86_64-linux
 
 DEPENDENCIES
+  html-proofer (~> 5.0)
+  jekyll (~> 4.4)
+  jekyll-optional-front-matter (~> 0.3)
+  jekyll-relative-links (~> 0.7)
+  just-the-docs (~> 0.12)
   minitest
   minitest-fail-fast (~> 0.1.0)
   minitest-hooks

README.md

@@ -29,6 +29,29 @@ On a high-level, Ghostferry is broken into several components, enabling it to
 copy data. This is documented at
 https://shopify.github.io/ghostferry/main/technicaloverview.html
 
+Documentation
+-------------
+
+The documentation is written in Markdown under `docs/` and can be read
+directly on GitHub, starting at the [Documentation source](docs/index.md). The
+published site is built with [Jekyll](https://jekyllrb.com/) and the
+[Just the Docs](https://just-the-docs.com/) theme; its settings, page titles
+and navigation order live in `docs/_config.yml`.
+
+Internal contributors get the gems from `dev up`, then can run `dev docs`
+(live preview) or `dev docs-build` (build plus link check). Otherwise:
+
+```bash
+bundle install
+bundle exec jekyll build --source docs --destination build/docs
+bundle exec htmlproofer build/docs --disable-external --allow-missing-href --no-enforce-https --swap-urls '^/ghostferry/main/:/'
+bundle exec jekyll serve --source docs --destination build/docs --host 127.0.0.1 --port 4000
+```
+
+The build writes the site to `build/docs/`; `htmlproofer` fails on broken
+internal links or anchors. The live preview is served at
+http://127.0.0.1:4000/ghostferry/main/. None of these commands deploy anything.
+
 Development Setup
 -----------------
 

dev.yml

@@ -31,3 +31,11 @@ commands:
   test-ruby:
     desc: Run the ruby test suite.
     run: make test-ruby
+  docs:
+    desc: Serve the documentation locally at http://127.0.0.1:4000/ghostferry/main/ (no deploy).
+    run: bundle exec jekyll serve --source docs --destination build/docs --host 127.0.0.1 --port 4000
+  docs-build:
+    desc: Build the documentation into build/docs and check its links and anchors (no deploy).
+    run: |
+      bundle exec jekyll build --source docs --destination build/docs
+      bundle exec htmlproofer build/docs --disable-external --allow-missing-href --no-enforce-https --swap-urls '^/ghostferry/main/:/'