diff --git a/.github/workflows/clickhouse_upgrade_test.yml b/.github/workflows/clickhouse_upgrade_test.yml new file mode 100644 index 00000000..6a710941 --- /dev/null +++ b/.github/workflows/clickhouse_upgrade_test.yml @@ -0,0 +1,396 @@ +# Runs the ClickHouse cluster upgrade test (scripts/clickhouse-upgrade-test/) +# against a throwaway 3-node cluster shaped like production's oonidata_cluster. +# +# Each hop / node-upgrade is its own job step (via ci_step.py) rather than one +# opaque "run the whole thing" step, so a failure localizes to exactly which +# version transition broke, with its own checkmark and log in the GitHub UI. +# +# See scripts/clickhouse-upgrade-test/README.md and sql/001_schema.sql for +# the full writeup of why the upgrade path is staged through LTS releases +# instead of jumping straight to latest. +on: + workflow_dispatch: + inputs: + scenario: + description: "Which upgrade path(s) to run" + type: choice + options: + - both + - staged + - direct + - real-data + - all + default: both + pull_request: + paths: + - "scripts/clickhouse-upgrade-test/**" + - "ansible/group_vars/clickhouse/vars.yml" + - ".github/workflows/clickhouse_upgrade_test.yml" + push: + branches: + - main + paths: + - "scripts/clickhouse-upgrade-test/**" + - "ansible/group_vars/clickhouse/vars.yml" + +jobs: + # Recommended path: 24.8.6.70 -> 25.3.14.14 -> 25.8.29.51 -> 26.3.17.110 -> + # 26.7.3.19, one LTS hop at a time, one node at a time within each hop. + # This is the upgrade path OONI should actually follow. + staged-upgrade: + if: github.event_name != 'workflow_dispatch' || github.event.inputs.scenario == 'both' || github.event.inputs.scenario == 'staged' + runs-on: ubuntu-latest + # Bumped from 60 -> 90: the bisection hops (see harness/versions.py) add + # 4 more full hops (16 more steps + 4 new image pulls), so the old + # budget got tighter even in the all-pass case. + timeout-minutes: 90 + defaults: + run: + working-directory: scripts/clickhouse-upgrade-test + steps: + - uses: actions/checkout@v4 + + # Fails loudly if harness/versions.py is edited without updating the + # hardcoded step list below (or vice versa) -- the two have no other + # link between them, so silent drift would otherwise just quietly test + # the wrong path. + - name: Sanity-check version ladder matches harness/versions.py + run: | + python3 - <<'EOF' + from harness.versions import BASE_VERSION, LATEST_VERSION, LTS_HOPS + # 25.4.13.22 .. 25.7.8.71 are bisection steps inserted between the + # 25.3 and 25.8 LTS releases -- see harness/versions.py's module + # docstring for why (a real mark-file-format incompatibility hit + # partway through that hop in ooni/devops#477 CI run 32044578317). + expected_hops = [ + "25.3.14.14", + "25.4.13.22", + "25.5.11.15", + "25.6.13.41", + "25.7.8.71", + "25.8.29.51", + "26.3.17.110", + "26.7.3.19", + ] + actual_hops = [v for v, _ in LTS_HOPS[1:]] + assert BASE_VERSION == "24.8.6.70", ( + f"harness/versions.py BASE_VERSION is now {BASE_VERSION!r} -- " + "update this workflow's steps (and the assertions here) to match" + ) + assert LATEST_VERSION == "26.7.3.19", ( + f"harness/versions.py LATEST_VERSION is now {LATEST_VERSION!r} -- " + "update this workflow's steps (and the assertions here) to match" + ) + assert actual_hops == expected_hops, ( + f"harness/versions.py LTS_HOPS is now {actual_hops} -- " + "update this workflow's steps (and the assertions here) to match" + ) + print("OK: workflow version ladder matches harness/versions.py") + EOF + + - name: Validate docker-compose.yml + run: docker compose config + + - name: Set up cluster at 24.8.6.70 (current production version) and load schema + seed data + run: python3 ci_step.py setup --base-version 24.8.6.70 --label setup + + # --- Hop 1/8: 24.8.6.70 -> 25.3.14.14 (LTS) --- + - name: "Hop 1/8 (-> 25.3.14.14): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.3.14.14 --label hop1-ch1 + continue-on-error: true + - name: "Hop 1/8 (-> 25.3.14.14): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.3.14.14 --label hop1-ch2 + continue-on-error: true + - name: "Hop 1/8 (-> 25.3.14.14): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.3.14.14 --label hop1-ch3 + continue-on-error: true + - name: "Hop 1/8 (-> 25.3.14.14): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.3.14.14 --label hop1-verify-ddl + continue-on-error: true + + # --- Hops 2-5/8: bisection steps through the monthly (non-LTS) stable + # releases between 25.3 and 25.8, inserted to pin down exactly which + # release introduces the "Unknown mark file extension" incompatibility + # hit in ooni/devops#477 CI run 32044578317 (see harness/versions.py's + # module docstring for the full writeup and root cause). Whichever of + # these is the first to fail localizes the change to one month's worth + # of ClickHouse commits instead of the whole 25.3-25.8 span. --- + - name: "Hop 2/8 (-> 25.4.13.22): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.4.13.22 --label hop2-ch1 + continue-on-error: true + - name: "Hop 2/8 (-> 25.4.13.22): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.4.13.22 --label hop2-ch2 + continue-on-error: true + - name: "Hop 2/8 (-> 25.4.13.22): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.4.13.22 --label hop2-ch3 + continue-on-error: true + - name: "Hop 2/8 (-> 25.4.13.22): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.4.13.22 --label hop2-verify-ddl + continue-on-error: true + + - name: "Hop 3/8 (-> 25.5.11.15): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.5.11.15 --label hop3-ch1 + continue-on-error: true + - name: "Hop 3/8 (-> 25.5.11.15): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.5.11.15 --label hop3-ch2 + continue-on-error: true + - name: "Hop 3/8 (-> 25.5.11.15): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.5.11.15 --label hop3-ch3 + continue-on-error: true + - name: "Hop 3/8 (-> 25.5.11.15): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.5.11.15 --label hop3-verify-ddl + continue-on-error: true + + - name: "Hop 4/8 (-> 25.6.13.41): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.6.13.41 --label hop4-ch1 + continue-on-error: true + - name: "Hop 4/8 (-> 25.6.13.41): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.6.13.41 --label hop4-ch2 + continue-on-error: true + - name: "Hop 4/8 (-> 25.6.13.41): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.6.13.41 --label hop4-ch3 + continue-on-error: true + - name: "Hop 4/8 (-> 25.6.13.41): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.6.13.41 --label hop4-verify-ddl + continue-on-error: true + + - name: "Hop 5/8 (-> 25.7.8.71): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.7.8.71 --label hop5-ch1 + continue-on-error: true + - name: "Hop 5/8 (-> 25.7.8.71): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.7.8.71 --label hop5-ch2 + continue-on-error: true + - name: "Hop 5/8 (-> 25.7.8.71): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.7.8.71 --label hop5-ch3 + continue-on-error: true + - name: "Hop 5/8 (-> 25.7.8.71): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.7.8.71 --label hop5-verify-ddl + continue-on-error: true + + # --- Hop 6/8: 25.7.8.71 -> 25.8.29.51 (LTS) --- + - name: "Hop 6/8 (-> 25.8.29.51): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 25.8.29.51 --label hop6-ch1 + continue-on-error: true + - name: "Hop 6/8 (-> 25.8.29.51): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 25.8.29.51 --label hop6-ch2 + continue-on-error: true + - name: "Hop 6/8 (-> 25.8.29.51): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 25.8.29.51 --label hop6-ch3 + continue-on-error: true + - name: "Hop 6/8 (-> 25.8.29.51): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 25.8.29.51 --label hop6-verify-ddl + continue-on-error: true + + # --- Hop 7/8: 25.8.29.51 -> 26.3.17.110 (LTS) --- + - name: "Hop 7/8 (-> 26.3.17.110): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 26.3.17.110 --label hop7-ch1 + continue-on-error: true + - name: "Hop 7/8 (-> 26.3.17.110): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 26.3.17.110 --label hop7-ch2 + continue-on-error: true + - name: "Hop 7/8 (-> 26.3.17.110): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 26.3.17.110 --label hop7-ch3 + continue-on-error: true + - name: "Hop 7/8 (-> 26.3.17.110): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 26.3.17.110 --label hop7-verify-ddl + continue-on-error: true + + # --- Hop 8/8: 26.3.17.110 -> 26.7.3.19 (latest stable) --- + - name: "Hop 8/8 (-> 26.7.3.19): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 26.7.3.19 --label hop8-ch1 + continue-on-error: true + - name: "Hop 8/8 (-> 26.7.3.19): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 26.7.3.19 --label hop8-ch2 + continue-on-error: true + - name: "Hop 8/8 (-> 26.7.3.19): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 26.7.3.19 --label hop8-ch3 + continue-on-error: true + - name: "Hop 8/8 (-> 26.7.3.19): verify ON CLUSTER DDL" + run: python3 ci_step.py verify-ddl --version 26.7.3.19 --label hop8-verify-ddl + continue-on-error: true + + - name: Generate report + if: always() + run: python3 ci_step.py report + + - name: Upload report + per-step results + if: always() + uses: actions/upload-artifact@v4 + with: + name: clickhouse-staged-upgrade-report + path: scripts/clickhouse-upgrade-test/results/ + retention-days: 30 + + - name: Tear down cluster + if: always() + run: python3 ci_step.py teardown + + # Real-data end-to-end scenario (ooni/data#437 follow-up, ooni/data#160): + # does the actual OONI ingestion pipeline -- real measurements, downloaded + # and submitted through fastpath/oonipipeline, then queried through the + # real api-oonimeasurements service -- still work after each hop of + # PRODUCTION_HOPS? Separate from staged-upgrade above, which uses fast + # synthetic seed data to check ClickHouse's own replication mechanics + # across every bisection waypoint; this one is slower, has a real + # external git + network dependency, and walks the actual 4-hop + # production runbook rather than every diagnostic waypoint. See + # harness/real_data.py and README.md's "real-data-upgrade" section for + # the full design (load real data once, re-verify integrity + the real + # pytest suite at every hop). + # + # Deliberately NOT run on pull_request (unlike staged-upgrade / + # direct-jump-upgrade above): it checks out an unmerged external branch + # (ooni/data's add_end_to_end_tests, ooni/data#160) this repo doesn't + # control, plus does real network downloads of OONI measurement data -- + # a flaky/slow dependency there shouldn't block unrelated PRs to this + # repo. Runs on push to main (when these paths change) and on-demand via + # workflow_dispatch (scenario: real-data or all) instead. + real-data-upgrade: + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.scenario == 'all' || github.event.inputs.scenario == 'real-data')) + runs-on: ubuntu-latest + # Real network download of OONI measurement data + oonipipeline/fastpath + # processing (once), plus 4 hops x (3 node upgrades + an integrity + # re-check + a full pytest run against the real API) -- substantially + # slower than staged-upgrade's synthetic-data hops. + timeout-minutes: 120 + defaults: + run: + working-directory: scripts/clickhouse-upgrade-test + steps: + - uses: actions/checkout@v4 + + # ooni/data#160 (branch add_end_to_end_tests) is still an open, + # unmerged PR as of this writing -- see docker-compose.real-data.yml's + # header comment for why this pins to the branch instead of main. + # Update this ref once #160 merges. + - name: Checkout ooni/data (add_end_to_end_tests branch) + uses: actions/checkout@v4 + with: + repository: ooni/data + ref: add_end_to_end_tests + path: external/ooni-data + + - name: Sanity-check hop ladder matches harness/versions.py PRODUCTION_HOPS + run: | + python3 - <<'EOF' + from harness.versions import PRODUCTION_HOPS + expected = [ + "24.8.6.70", + "25.3.14.14", + "25.8.29.51", + "26.3.17.110", + "26.7.3.19", + ] + actual = [v for v, _ in PRODUCTION_HOPS] + assert actual == expected, ( + f"harness/versions.py PRODUCTION_HOPS is now {actual} -- " + "update this workflow's steps (and the assertions here) to match" + ) + print("OK: workflow hop ladder matches harness/versions.py PRODUCTION_HOPS") + EOF + + - name: Validate docker-compose.yml + docker-compose.real-data.yml + run: docker compose -f docker-compose.yml -f docker-compose.real-data.yml config + + - name: Set up cluster at 24.8.6.70 (schema only -- no synthetic seed data) + run: python3 ci_step.py setup-real-data --base-version 24.8.6.70 --label setup-real-data + + - name: Load real OONI data once (oonidata sync + oonipipeline + fastpath) + run: python3 ci_step.py load-real-data --label load-real-data + + - name: Take golden snapshot (row counts + checksums, all 3 nodes must agree) + run: python3 ci_step.py golden-snapshot --label golden-snapshot + + - name: Sanity-check ooni/data's pytest suite passes at 24.8.6.70 before any upgrade + run: python3 ci_step.py verify-e2e --label base-verify-e2e + + # --- Real-data hop 1/4: 24.8.6.70 -> 25.3.14.14 --- + - name: "Real-data hop 1/4 (-> 25.3.14.14): upgrade all 3 nodes, re-check integrity + pytest suite" + run: python3 ci_step.py real-data-hop --version 25.3.14.14 --label rd-hop1 + continue-on-error: true + + # --- Real-data hop 2/4: 25.3.14.14 -> 25.8.29.51 -- see + # harness/versions.py's "self-heals once the lagging node catches up" + # section: expect the trailing node to log hard-looking errors until + # its own upgrade finishes. real_data_hop_step upgrades all 3 nodes + # back-to-back within this single step, matching the operational rule + # documented there for this specific hop. --- + - name: "Real-data hop 2/4 (-> 25.8.29.51): upgrade all 3 nodes, re-check integrity + pytest suite" + run: python3 ci_step.py real-data-hop --version 25.8.29.51 --label rd-hop2 + continue-on-error: true + + # --- Real-data hop 3/4: 25.8.29.51 -> 26.3.17.110 -- same + # self-healing pattern as hop 2, this time the nested-data-type + # serialization change. --- + - name: "Real-data hop 3/4 (-> 26.3.17.110): upgrade all 3 nodes, re-check integrity + pytest suite" + run: python3 ci_step.py real-data-hop --version 26.3.17.110 --label rd-hop3 + continue-on-error: true + + # --- Real-data hop 4/4: 26.3.17.110 -> 26.7.3.19 (latest stable) --- + - name: "Real-data hop 4/4 (-> 26.7.3.19): upgrade all 3 nodes, re-check integrity + pytest suite" + run: python3 ci_step.py real-data-hop --version 26.7.3.19 --label rd-hop4 + continue-on-error: true + + - name: Generate report + if: always() + run: python3 ci_step.py report + + - name: Upload report + per-step results + if: always() + uses: actions/upload-artifact@v4 + with: + name: clickhouse-real-data-upgrade-report + path: scripts/clickhouse-upgrade-test/results/ + retention-days: 30 + + - name: Tear down cluster + real-data stack + if: always() + run: python3 ci_step.py teardown-real-data + + # Naive path: straight from 24.8.6.70 to 26.7.3.19, node by node, skipping + # every intermediate LTS release. Expected to demonstrate ClickHouse's own + # documented ~1 year mixed-version compatibility limit, so this job is + # allowed to fail without failing the whole workflow (continue-on-error) -- + # it's diagnostic/documentary, not a gate. + direct-jump-upgrade: + if: github.event_name != 'workflow_dispatch' || github.event.inputs.scenario == 'both' || github.event.inputs.scenario == 'direct' + runs-on: ubuntu-latest + timeout-minutes: 30 + continue-on-error: true + defaults: + run: + working-directory: scripts/clickhouse-upgrade-test + steps: + - uses: actions/checkout@v4 + + - name: Validate docker-compose.yml + run: docker compose config + + - name: Set up cluster at 24.8.6.70 (current production version) and load schema + seed data + run: python3 ci_step.py setup --base-version 24.8.6.70 --label setup + + - name: "Direct jump (-> 26.7.3.19, skips all LTS hops): upgrade ch1" + run: python3 ci_step.py upgrade-node --node ch1 --version 26.7.3.19 --label direct-ch1 + continue-on-error: true + - name: "Direct jump (-> 26.7.3.19, skips all LTS hops): upgrade ch2" + run: python3 ci_step.py upgrade-node --node ch2 --version 26.7.3.19 --label direct-ch2 + continue-on-error: true + - name: "Direct jump (-> 26.7.3.19, skips all LTS hops): upgrade ch3" + run: python3 ci_step.py upgrade-node --node ch3 --version 26.7.3.19 --label direct-ch3 + continue-on-error: true + + - name: Generate report + if: always() + run: python3 ci_step.py report + + - name: Upload report + per-step results + if: always() + uses: actions/upload-artifact@v4 + with: + name: clickhouse-direct-jump-report + path: scripts/clickhouse-upgrade-test/results/ + retention-days: 30 + + - name: Tear down cluster + if: always() + run: python3 ci_step.py teardown diff --git a/scripts/clickhouse-upgrade-test/.gitignore b/scripts/clickhouse-upgrade-test/.gitignore new file mode 100644 index 00000000..0521f1a8 --- /dev/null +++ b/scripts/clickhouse-upgrade-test/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.pyc + +# Generated test output -- keep the directory (see results/.gitkeep) but not +# whatever a local run drops into it. +results/report.md +results/report.json +results/steps/ diff --git a/scripts/clickhouse-upgrade-test/Makefile b/scripts/clickhouse-upgrade-test/Makefile new file mode 100644 index 00000000..67e4927d --- /dev/null +++ b/scripts/clickhouse-upgrade-test/Makefile @@ -0,0 +1,21 @@ +.PHONY: test test-staged test-direct config down clean + +# Full test: staged (recommended) upgrade path, then the naive direct-jump path +test: + python3 run_test.py --scenario both + +test-staged: + python3 run_test.py --scenario staged + +test-direct: + python3 run_test.py --scenario direct + +# Validate docker-compose.yml without needing registry access +config: + docker compose config + +down: + docker compose down -v + +clean: down + rm -rf results/report.md results/report.json diff --git a/scripts/clickhouse-upgrade-test/README.md b/scripts/clickhouse-upgrade-test/README.md new file mode 100644 index 00000000..569f2be2 --- /dev/null +++ b/scripts/clickhouse-upgrade-test/README.md @@ -0,0 +1,550 @@ +# ClickHouse cluster upgrade test (ooni/devops#437) + +Answers the question behind [ooni/devops#437](https://github.com/ooni/devops/issues/437): +**can OONI's production ClickHouse cluster be upgraded from its current +version to the latest stable release one node at a time, or does it need a +scheduled-downtime, all-nodes-at-once upgrade?** + +## TL;DR + +- Production is on **24.8.6.70** (LTS, Aug 2024) — confirmed from + `ooni/devops` `ansible/group_vars/clickhouse/vars.yml` (`clickhouse_version: 24.8.6.70`), + matching what issue #437 reports. +- Latest stable as of 2026-08-10 is **26.7.3.19** (released 2026-07-22). +- That's about **23 months apart**. ClickHouse's own docs + ([clickhouse.com/docs/operations/update](https://clickhouse.com/docs/operations/update)) + say replicas of the same shard should not run versions more than + **~1 year apart** — beyond that window the docs warn the cluster "may not + work", queries can fail with arbitrary errors, and downgrading stops being + an option. +- **Recommendation, updated after CI run + [32122682392](https://github.com/ooni/devops/actions/runs/32122682392) + completed the full ladder: do a rolling, node-by-node upgrade all the + way to `26.7.3.19`, in 4 hops, landing on each LTS release in turn.** + Two of those hops (`25.3.14.14 -> 25.8.29.51` and + `25.8.29.51 -> 26.3.17.110`) hit real, reproducible ClickHouse + incompatibilities in CI — but both turned out to be transient and + self-healing once the lagging node's own upgrade completes, not + structural blocks. See "Real CI findings" below for what that means + operationally before doing either of those two hops in production. + + ``` + 24.8.6.70 → 25.3.14.14 → 25.8.29.51 → 26.3.17.110 → 26.7.3.19 + (current) LTS LTS (*) LTS (*) (latest) + + (*) upgrade all 3 nodes back-to-back in one sitting for these two hops -- + the trailing node is expected to log hard-looking errors for a + minute or two until its own upgrade finishes. See "Real CI findings". + ``` + + No full-cluster downtime is needed — the risk was never downtime, it + was version skew during the upgrade window, and that skew resolves + itself as long as the rollout actually finishes rather than being left + half-done. + + The monthly (non-LTS) releases between `25.3.14.14` and `25.8.29.51` + (`25.4.13.22`, `25.5.11.15`, `25.6.13.41`, `25.7.8.71`) only exist in + this project's CI ladder (`harness/versions.py`'s `LTS_HOPS`), inserted + to bisect *which* release introduced the incompatibility. Production + has no reason to stop on any of them — see `harness/versions.py`'s + `PRODUCTION_HOPS` for the 4-hop version of this ladder. + +This repo contains a dockerized test that *exercises* this rather than just +asserting it: it spins up a 3-node cluster shaped exactly like OONI's +`oonidata_cluster` (1 shard, 3 replicas, embedded ClickHouse Keeper, same +table schemas), loads it with data, and mechanically upgrades one node at a +time — first via the direct jump (to show what breaks), then via the staged +LTS path (to confirm it doesn't). A third, separate job additionally +replays the real OONI data-ingestion pipeline across the same upgrade path +using actual measurements instead of synthetic data — see "Real-data +end-to-end scenario" below. + +## Where the numbers come from + +| Fact | Source | +|---|---| +| Current version `24.8.6.70` | `ooni/devops` `ansible/group_vars/clickhouse/vars.yml` → `clickhouse_version:` | +| Cluster topology: 1 shard, 3 replicas, embedded Keeper on `data1/2/3.htz-fsn.prod.ooni.nu` | `ooni/devops` `ansible/group_vars/clickhouse/vars.yml` (`clickhouse_remote_servers`, `clickhouse_keeper`, `clickhouse_macros`), `ansible/roles/oonidata_clickhouse/tasks/main.yml`, `ansible/inventory` | +| Production table schemas (`fastpath`, `citizenlab`, `jsonl`, `analysis_web_measurement`, `event_detector_changepoints`, `faulty_measurements`) | `ooni/devops` `scripts/cluster-migration/schema.sql` | +| `obs_web` column list | `ooni/backend` `ooniapi/services/oonimeasurements/tests/fixtures/initdb/clickhouse.sql` | +| Other table column lists (test/CI copies) | `ooni/backend` `ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql` | +| Latest stable / LTS version history | [clickhouse.com/docs/whats-new/changelog](https://clickhouse.com/docs/whats-new/changelog), [endoflife.date/clickhouse](https://endoflife.date/clickhouse) | +| Mixed-version / rolling-upgrade guidance | [clickhouse.com/docs/operations/update](https://clickhouse.com/docs/operations/update) | + +## What the test actually does + +`docker-compose.yml` brings up 3 ClickHouse nodes (`ch1`, `ch2`, `ch3`) on a +private docker network, each running **both** `clickhouse-server` and an +embedded **ClickHouse Keeper** instance (ports 9181/9234) — the same +topology as `data1/data2/data3` in production, just condensed onto one +Docker host. `sql/001_schema.sql` creates the real table schemas +(`ReplicatedReplacingMergeTree`, `ON CLUSTER oonidata_cluster`) and +`harness/seed_data.py` loads synthetic-but-schema-accurate rows into them. + +`run_test.py` then runs one or both scenarios: + +- **`staged`** — walks the version ladder above, upgrading `ch1`, then + `ch2`, then `ch3` at each hop (never more than one node down at a time, + never all 3 nodes on different versions at once), validating after every + single node swap that: + - the node comes back up, + - a write issued anywhere is readable from every replica within the + timeout (`harness/validate.py:probe_write_then_read`), + - row counts converge across all 3 nodes, + - `system.errors` hasn't accumulated any replication/checksum/protocol + errors, + - `system.replication_queue` has no stuck tasks, + - and, once a hop is fully rolled out, an `ALTER TABLE ... ON CLUSTER` + still propagates cluster-wide. +- **`direct`** — does the same node-by-node mechanics but jumps straight + from `24.8.6.70` to `26.7.3.19`, to surface (not just cite) whatever + breaks when replicas are held ~2 years apart in version for the whole + rollout. + +Results land in `results/report.md` (human-readable) and +`results/report.json` (full structured data, including every row-count +snapshot and every error ClickHouse logged). + +## Running it + +Requires Docker + Compose v2, and — this matters — **network access to pull +`clickhouse/clickhouse-server` images from Docker Hub**. (This harness was +built inside a sandboxed environment whose egress is restricted to a small +allowlist that does not include Docker Hub or S3, so it could not be +executed end-to-end there; everything here was validated as far as that +constraint allows — see "What was and wasn't verified" below.) + +```bash +# from this directory +make test # both scenarios (staged, then direct), ~20-40 min depending on image pull speed +make test-staged # just the recommended path +make test-direct # just the naive direct-jump path +make config # sanity-check docker-compose.yml without pulling anything +``` + +Or directly: + +```bash +python3 run_test.py --scenario both +``` + +Add `--keep-up` to leave the cluster running after the test so you can poke +at it manually (`docker compose exec ch1 clickhouse-client`). + +## About the seed data + +The task pointed at `ooni/backend`'s initdb sample data. That repo doesn't +actually vendor the sample rows in git — its test fixtures +(`ooniapi/services/oonimeasurements/tests/conftest.py`) download +`obs_web-sample.sql.gz` and `analysis_web_measurement-sample.sql.gz` at test +time from a public S3 bucket +(`ooni-data-eu-fra.s3.eu-central-1.amazonaws.com`). This sandbox's network +egress couldn't reach S3 either, so `harness/seed_data.py` generates +synthetic rows that conform exactly to the real schemas instead (same +columns, types, nullability, realistic cardinality for things like +`probe_cc`/ASN/test names). That's sufficient for what this test is +checking — replication and on-disk part-format compatibility across +ClickHouse versions — since that behavior depends on schema and volume, not +on the specific measurement content. + +If you have S3 access and want to use the real dump instead: + +```bash +curl -sL https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/obs_web-sample.sql.gz \ + | gunzip -c | docker compose exec -T ch1 clickhouse-client --database ooni +curl -sL https://ooni-data-eu-fra.s3.eu-central-1.amazonaws.com/samples/analysis_web_measurement-sample.sql.gz \ + | gunzip -c | docker compose exec -T ch1 clickhouse-client --database ooni +``` + +(after `sql/001_schema.sql` has been applied, and before running an upgrade +scenario — or just skip the seed step in `harness/scenarios.py:load_schema_and_seed` +and load these instead). + +## Real CI findings: a genuine incompatibility, pinned to exactly 25.8.29.51 + +A real staged-upgrade CI run +([ooni/devops#477](https://github.com/ooni/devops/pull/477), run +[32044578317](https://github.com/ooni/devops/actions/runs/32044578317)) +got cleanly through `24.8.6.70 -> 25.3.14.14` (including the transient, +non-gating connection blips a container recreate is expected to cause — +see "What was and wasn't verified" below) and then hit a **hard, real +failure** during `25.3.14.14 -> 25.8.29.51`, specifically at the point +where ch1 and ch2 were already on `25.8.29.51` and ch3 was still on +`25.3.14.14`: + +- `CHECKSUM_DOESNT_MATCH` logged on both upgraded nodes. +- ch3's replication queue stuck retrying two entries (148 and 147 tries + and climbing) with the identical root cause on both: + `Code: 79. DB::Exception: Unknown mark file extension: '4'. + (INCORRECT_FILE_NAME)`, thrown while ch3 tried to fetch a data part from + a peer. +- The write-then-read-back probe failed on ch3 for the first time in the + whole run, and row counts diverged. + +That's a **materially bigger finding than the one raised in review** — it +shows up a full LTS hop before 26.3, the version the review flagged as the +one to be careful about. + +**Bisected and confirmed** (run +[32047534149](https://github.com/ooni/devops/actions/runs/32047534149), +after inserting every monthly release between the two LTS versions — +`25.4.13.22`, `25.5.11.15`, `25.6.13.41`, `25.7.8.71`, from +[endoflife.date/api/clickhouse.json](https://endoflife.date/api/clickhouse.json)): +**`24.8.6.70 -> 25.3.14.14 -> 25.4.13.22 -> 25.5.11.15 -> 25.6.13.41 -> +25.7.8.71` all upgrade cleanly, node by node, zero hard errors.** The +failure reappears exactly and only at `25.7.8.71 -> 25.8.29.51` — same +failure family, a different specific manifestation this time: +`Code: 226. NO_FILE_IN_DATA_PART: No columns_substreams.txt in part +all_17_17_1`, fetching a part whose mark file has the new `.cmrk4` +extension. This rules out a gradual drift across the whole 25.3-25.8 span +— it's one version boundary, `25.8.29.51`, that changes the on-disk +compact-part format (new manifest file + new mark-file extension) in a +way no earlier binary in this range can read. + +**Corroborating evidence** (not confirmed against the official changelog +text itself — repeated attempts to fetch the relevant section, listed +below, all failed): a v25.12 changelog entry found during this +investigation reads *"Enable advanced shared data for JSON by default... +after that change downgrade to versions before 25.8 will be not possible, +because these versions won't be able to read new data parts with JSON +column."* That's scoped to JSON columns and to downgrading specifically, +but it names 25.8 as the version this substream-based part-serialization +infrastructure was introduced in. `citizenlab` (the table that failed +here) has no JSON column, so this bisection most likely caught that same +infrastructure applying to plain `MergeTree` parts generally — consistent +with, though not proof of, a shared root cause. + +`harness/versions.py`'s `LTS_HOPS` and +`.github/workflows/clickhouse_upgrade_test.yml` both keep the 8-hop +bisection ladder (rather than collapsing back to 4 hops) so this stays +directly re-testable. Production, however, doesn't need to walk the +monthly releases — see `PRODUCTION_HOPS` below. + +## Real CI findings, continued: both incompatibilities self-heal once the lagging node catches up + +The open question from the previous section — does the stuck queue clear +once the lagging node's own upgrade finishes, or is it permanent? — is +answered. `.github/workflows/clickhouse_upgrade_test.yml` was changed to +add `continue-on-error: true` to every upgrade/verify step so a hop's +first failure no longer aborts the job before the remaining nodes get a +chance to upgrade too, and `ci_step.py report` was changed to be the +actual job-level pass/fail gate instead. Run +[32122682392](https://github.com/ooni/devops/actions/runs/32122682392) +then completed the entire 8-hop ladder and found: + +- **`hop6-ch2`** (upgrading ch2 to `25.8.29.51`, leaving ch3 on + `25.7.8.71`) failed exactly as before: ch3 stuck retrying a `GET_PART` + fetch (`NO_FILE_IN_DATA_PART`, missing `columns_substreams.txt`), 17 + tries. **`hop6-ch3`** — ch3's own upgrade to `25.8.29.51`, run + immediately after — passed clean: converged, fully replicated, zero + queue problems. The stuck fetch simply succeeded once ch3 could parse + the new format itself. +- The identical pattern repeats one hop later, and this is the exact + 26.3 nested-type serialization change flagged in the original PR + review: **`hop7-ch1`** logged a hard `CHECKSUM_DOESNT_MATCH` while + briefly the only node on `26.3.17.110`. **`hop7-ch2`** then left ch3 + (still on `25.8.29.51`) stuck retrying with `CORRUPTED_DATA` / + *"Unknown version of serialization infos (1). Should be less or equal + than 0"* — 17 tries. **`hop7-ch3`** — ch3's own upgrade to + `26.3.17.110` — again passed clean. +- **`hop8`** (`26.3.17.110 -> 26.7.3.19`) had zero hard errors anywhere. + +So both incompatibilities are the same underlying mechanism: an +old-format binary can't parse a part written in a new on-disk format, and +the fix is simply for that binary to become new-format too, at which +point its own retry of the identical fetch succeeds. Neither is a +structural block on reaching `26.7.3.19`. + +**What this doesn't tell us**, and shouldn't be assumed: the mixed-version +window in that run was CI-paced — seconds to at most a couple of minutes +between one node finishing and the next starting. Whether the same +self-healing holds if a node is left lagging for hours or days at either +of these two hops specifically hasn't been tested. Nor does this touch +the separate *downgrade*-lossiness warning in the 26.3 changelog entry — +that's about rolling back after the fact, a different risk from the +forward-rolling mixed-version friction this run exercised. + +Given this, `RECOMMENDED_NOW` in `harness/versions.py` is now +`26.7.3.19`, and `PRODUCTION_HOPS` is the 4-hop runbook this project +recommends: skip the monthly bisection releases (they were CI-diagnostic +only), land on each LTS in turn, and for the two hops that hit a real +incompatibility (`25.3.14.14 -> 25.8.29.51`, `25.8.29.51 -> 26.3.17.110`) +upgrade all three nodes back-to-back in one sitting rather than spacing +them out — expect the trailing node to log hard-looking errors for a +minute or two right up until its own upgrade finishes, and treat that as +expected only if it actually clears once that node is fully upgraded. If +it's still stuck minutes after the last node comes back up, stop and +treat it as a real problem rather than assuming it'll resolve. + +One gap before calling `PRODUCTION_HOPS` fully proven: self-healing has +been directly observed for the `25.7.8.71 -> 25.8.29.51` sub-hop (via the +bisection ladder) and for `25.8.29.51 -> 26.3.17.110`, but not yet for a +genuine single-hop `25.3.14.14 -> 25.8.29.51` jump (skipping the +intermediate monthly releases). The original un-bisected 4-hop ladder +(run 32044578317) hit the identical failure signature at that exact +transition, but aborted before ch3 got a chance to complete its own +upgrade — so self-healing there is inferred from the shared mechanism, +not directly confirmed. Worth one more CI run of `PRODUCTION_HOPS` itself +to close this gap. + +## PR #477 review response + +Raised in review on [ooni/devops#477](https://github.com/ooni/devops/pull/477) +(hellais) — addressed here point by point: + +1. **"Read the changelog for tricky breaking changes."** 26.3 ships + ["Propagate data types serialization versions to nested data + types"](https://clickhouse.com/docs/resources/changelogs/oss/2026#263-backward-incompatible-change), + which the changelog itself flags as able to make **downgrading after + upgrading lossy**. That downgrade-lossiness warning is still true and + still unresolved. Separately, real CI turned up a *forward*-upgrade + consequence of this same change too: mixed-version friction while + rolling through 26.3 (see "Real CI findings, continued" above) — which + traces to the identical changelog entry, just a different symptom than + the one originally flagged. Both this and the earlier 25.8.29.51 + mark-file finding turned out to be transient and self-healing rather + than blocking, once the harness could observe a completed rollout — + see "Real CI findings" and "Real CI findings, continued" above for the + full story and the operational caveats that still apply. +2. **"Renamed `searchAny`/`searchAll` to `hasAnyTokens`/`hasAllTokens` + (25.10) — make sure we aren't using these."** Confirmed absent from + `oonipipeline` (`ooni/data`, the data-pipeline repo this cluster feeds — + found via `ansible/roles/oonidata_airflow` / `ansible/roles/notebook`). + **Still open:** `ooni/backend` hasn't been grepped for these yet. +3. **"Disallow truncating replicated databases — might apply to us in the + data pipeline."** It does, and there are two independent production + sites doing it, not one: + - `ooni/data` (`oonipipeline`) `tasks/updaters/citizenlab_test_lists_updater.py` + - `ooni/backend` `analysis/analysis/citizenlab_test_lists_updater.py` + + Both run the identical sequence: `TRUNCATE TABLE citizenlab_flip` + (a `ReplicatedReplacingMergeTree` table, per `sql/001_schema.sql`) → + `INSERT INTO citizenlab_flip` → `EXCHANGE TABLES citizenlab_flip AND + citizenlab` — the swapped-ZK-path pair documented there. It's not yet + confirmed which of these two is the one actually deployed/cron'd today + vs. legacy code left over from a migration between repos — worth a + direct check before assuming only one matters. + + Lower-severity but same category, found while checking test suites for + "run against the target version" (point 4): `ooni/backend`'s + `oonirun` and `ooniprobe` service test fixtures (`tests/conftest.py`) + both call `TRUNCATE TABLE` on `url_priorities` and `faulty_measurements` + respectively — both of which are also `Replicated*` engines per the live + schema. These only run against ephemeral test containers today, but if + those test suites get pointed at a candidate ClickHouse version (see + point 4), a truncate-replicated restriction would surface there too, not + just in the data pipeline. `oonipipeline`'s own `cli/commands.py` + `TRUNCATE TABLE event_detector_cusums SYNC` and its `tests/conftest.py` + truncates are lower risk since `event_detector_cusums`/`_changepoints` + are plain (non-replicated) `ReplacingMergeTree` per the live schema dump. + + **Still open:** pinning down which exact ClickHouse version introduced + the truncate-replicated restriction (the reviewer's comment didn't + include a changelog link for this one) and confirming whether it blocks + this specific truncate-then-swap pattern outright or only under some + conditions (e.g. only for `TRUNCATE ... ON CLUSTER` / whole databases, + not a single replicated table via one node). +4. **"Run the target version against the real API + data pipeline."** + **Addressed** by the new `real-data-upgrade` job — see "Real-data + end-to-end scenario" below for the full design. It downloads real OONI + measurements, ingests them through the actual `oonidata`/`oonipipeline` + pipeline and `fastpath`, and re-runs `ooni/data`'s own pytest suite + against the real `api-oonimeasurements` service at every hop of + `PRODUCTION_HOPS` — not just ClickHouse's own replication mechanics + (which the synthetic scenario above already covers), but the actual + ingestion and query paths OONI's data pipeline depends on. One caveat: + it exercises `ooni/data`'s test suite and `ooni/backend`'s + `api-oonimeasurements` service, not `oonipipeline`'s own test suite + directly, and it depends on an unmerged external branch + (`ooni/data#160`) — see that section for details. +5. **Full changelog sweep, 24.9 through 26.7, for every "Backward + Incompatible Change" entry** (not just the two the reviewer happened to + quote) — in progress, not complete. What's confirmed so far is captured + in points 1-3 above. + +Net effect: `LTS_HOPS` (what this harness's `staged` CI job actually +tests) walks the full ladder to `26.7.3.19`, and as of run 32122682392 it +does so clean — both the mark-file finding and the 26.3 nested-type +serialization change turned out to be transient, self-healing mixed- +version friction rather than structural blocks (see "Real CI findings, +continued" above). The production recommendation (`RECOMMENDED_NOW` / +`PRODUCTION_HOPS`, and the README TL;DR above) now covers the whole +ladder, with an operational caveat (upgrade the trailing node promptly) +attached to the two hops that hit a real incompatibility. Point 4 is now +addressed by the `real-data-upgrade` job (see below). Points 2 and 5 +remain open. + +## Real-data end-to-end scenario (`real-data-upgrade` job) + +A second, separate CI job that answers a different question from the +`staged`/`direct` scenarios above: not "does ClickHouse's own replication +survive the upgrade" (synthetic seed data is sufficient for that, and +fast), but "does OONI's actual data pipeline — real measurements, ingested +through the real `oonidata`/`oonipipeline`/`fastpath` tools, queried +through the real `api-oonimeasurements` service — still work correctly +after each hop of the production runbook, with nothing corrupted or +changed along the way." Implemented in `harness/real_data.py`, wired into +the `real-data-upgrade` job in +`.github/workflows/clickhouse_upgrade_test.yml`. + +**Source of the pipeline itself:** [ooni/data#160](https://github.com/ooni/data/pull/160) +(branch `add_end_to_end_tests`), which adds a `tests/integration/` stack +(single-node ClickHouse + Postgres + Valkey + fastpath + +`api-oonimeasurements` + a pytest `verify` container) that downloads real +OONI measurements and processes them through `oonipipeline`, then tests +the API against the result. This job adapts that stack onto this +project's existing 3-node **replicated** cluster instead of a single +throwaway node — see `docker-compose.real-data.yml`'s header comment for +every deliberate difference from the upstream compose file (auth, dropped +`fastpath2`, no host port exposure, etc.). + +**Design, per what this cluster actually needs to verify:** + +- **Real data is downloaded and ingested exactly once per CI run**, right + after standing up a fresh cluster at `BASE_VERSION` (`24.8.6.70`) — not + re-downloaded at every hop. Re-ingesting fresh data at each of + `PRODUCTION_HOPS`'s 4 checkpoints would multiply this already-slow job's + runtime for no real gain: the question this job answers is "does + upgrading corrupt or break access to what's already there," not "can + fresh data still be ingested at every intermediate version" (a real but + different question — see the `TODO` in `harness/real_data.py`'s module + docstring for a possible follow-up). +- Right after that one ingestion, a **golden snapshot** is taken — row + count + an order-independent content checksum (`sum(cityHash64(*))`, + ClickHouse's own idiom for hashing a whole table without listing columns + by hand) per real-data table, on all 3 nodes, requiring they already + agree with each other. +- **After every hop of `PRODUCTION_HOPS`** (all 3 nodes upgraded + back-to-back, reusing the exact same `upgrade_node_step()` mechanics the + synthetic scenario uses — these are already version/schema-agnostic): + 1. re-snapshot all 3 nodes and diff against the golden baseline — any + difference at all, on any node, in either row count or checksum, is a + hard failure, since nothing should be writing new data during an + upgrade rehearsal; + 2. re-run `ooni/data#160`'s own pytest suite against the real + `api-oonimeasurements` service, to confirm the genuine query/API path + still works — not just that ClickHouse's own replication mechanics + survived (already covered by the synthetic scenario's write-then- + read-back probe). +- Uses `PRODUCTION_HOPS` (4 hops), not the 8-hop `LTS_HOPS` bisection + ladder — this job verifies the actual recommended production upgrade + path, not every diagnostic waypoint used to originally localize the + mark-file incompatibility. A workflow-level sanity check (mirroring the + `staged-upgrade` job's own `LTS_HOPS` check) fails loudly if + `harness/versions.py`'s `PRODUCTION_HOPS` and this job's hardcoded step + list ever drift apart. + +**Does not replace the synthetic scenario.** `harness/seed_data.py` and +`scenario_staged_lts()`/`scenario_direct_jump()` are unchanged and still +run as before — they're fast, self-contained, and sufficient for +verifying replication/on-disk-format compatibility. This is a slower, +additional, more realistic check layered on top. + +**New schema tables**, added to `sql/001_schema.sql` to match what +`oonipipeline`'s observations workflow and `fastpath` actually write +(`fingerprints_dns`, `fingerprints_http`, `obs_web_ctrl`, +`obs_http_middlebox`, `obs_openvpn`) — see that file's own comments for +per-table provenance and which ones are cross-checked against a +known-live schema (`obs_web`) versus derived from `oonipipeline`'s DDL +generator with manual signedness corrections (that generator emits +`Int32`/`Int8` unconditionally for `int`/`bool` fields, which is +known-wrong for at least the ASN and boolean-flag columns it shares with +the already-verified `obs_web` table — see the file for the full +reasoning). `EmbeddedRocksDB` (used by a couple of these) has no +`Replicated` variant in ClickHouse — `ON CLUSTER` replicates the table +*structure* for it, not row contents; that's a real, documented +limitation of this engine, not a gap in this test. + +**Caveats:** + +- `ooni/data#160` is **still an open, unmerged PR** as of this writing. + `docker-compose.real-data.yml` and the workflow both pin to its + `add_end_to_end_tests` branch specifically (that's the branch with the + `oonipipeline` CLI entry point this job needs — `main` doesn't have it + yet). **Update the checkout ref once #160 merges.** + `oonidata sync`/`oonipipeline run --workflow-name observations` and + `fastpath` only populate `fastpath`, `obs_web`, `obs_web_ctrl`, and + `obs_http_middlebox` — `citizenlab`, `fingerprints_dns/http`, and + `obs_openvpn` need different updater/workflow invocations this job + doesn't run, so they're expected to stay empty here and aren't part of + the integrity check. +- **Not run on every pull request**, unlike `staged-upgrade` and + `direct-jump-upgrade` above — it checks out an external, unmerged + branch this repo doesn't control and does real network downloads of + OONI measurement data, so a flaky or slow dependency there shouldn't + block unrelated PRs. It runs on push to `main` (when these paths + change) and on demand via `workflow_dispatch` (`scenario: real-data` or + `all`). Revisit this if `real-data-upgrade` proves reliable enough, or + once `ooni/data#160` merges and the external-branch risk goes away. +- Like the rest of this harness, this could not be executed end-to-end in + the sandbox this was built in (no Docker daemon, restricted egress) — + verified as far as that allows (Python compiles, YAML/SQL parse + structurally) and real validation is deferred to an actual CI run, same + as every other CI-dependent change in this project. + +## What was and wasn't verified + +This harness has now actually run in GitHub Actions four times (see +`.github/workflows/clickhouse_upgrade_test.yml`, exercised on +[ooni/devops#477](https://github.com/ooni/devops/pull/477)), which has real +network access this project's original build/review sandbox didn't: + +- **Run 1** ([32041884883](https://github.com/ooni/devops/actions/runs/32041884883)) + got through `setup` and `ch1`'s `24.8.6.70 -> 25.3.14.14` upgrade, then + flagged `ch2`'s upgrade as failed — a **false positive in this harness's + own error-detection logic**, not a real ClickHouse problem (fixed; see + `harness/validate.py`'s transient-vs-hard error classification). +- **Run 2** ([32044578317](https://github.com/ooni/devops/actions/runs/32044578317)), + after that fix, got all the way through the full `24.8.6.70 -> 25.3.14.14` + hop cleanly (including further transient, non-gating blips, confirming + the fix generalizes) and then hit the real `25.3.14.14 -> 25.8.29.51` + mark-file incompatibility described in "Real CI findings" above. +- **Run 3** ([32047534149](https://github.com/ooni/devops/actions/runs/32047534149)), + with the bisection hops in place, confirmed `24.8.6.70` through + `25.7.8.71` all upgrade cleanly and pinned the failure to exactly the + `25.7.8.71 -> 25.8.29.51` transition — see "Real CI findings" above for + the full detail. +- **Run 4** ([32122682392](https://github.com/ooni/devops/actions/runs/32122682392)), + after adding `continue-on-error` so a hop's first failure no longer + aborts the job, completed the full 8-hop ladder and confirmed both the + `25.8.29.51` and `26.3.17.110` incompatibilities are transient and + self-healing once the lagging node's own upgrade finishes — see "Real + CI findings, continued" above. + +Originally verified only inside a sandbox with restricted egress (no Docker +Hub / S3 access), before any real run: +- `docker-compose.yml` parses and interpolates correctly (`docker compose config`). +- All ClickHouse XML config files (`config/**/*.xml`) are well-formed. +- All Python modules compile and the seed-data generator runs and produces + well-formed `INSERT` statements against the real column lists. + +**Still worth doing:** one more CI run of `PRODUCTION_HOPS` itself (the +4-hop runbook, skipping the monthly bisection releases) to directly +confirm self-healing holds for a genuine single-hop +`25.3.14.14 -> 25.8.29.51` jump, not just the bisected sub-hop; and +separately review the `direct-jump` job's own failure log, which still +hasn't been looked at (it's expected to fail — that's the point of that +job — but it's still worth confirming it fails for the *same* reason and +not something else). + +## Files + +``` +docker-compose.yml 3-node cluster definition, per-node image tag override via env +docker-compose.real-data.yml Overlay: real ooni/data pipeline (postgres/valkey/api/downloader/fastpath/verify) on top of ch1/ch2/ch3 +config/common/ Settings shared by all nodes (remote_servers, zookeeper client, distributed_ddl) +config/ch{1,2,3}/node.xml Per-node macros (shard/replica) + embedded Keeper raft config +sql/001_schema.sql Production table DDL (ReplicatedReplacingMergeTree, ON CLUSTER) + real-data-pipeline tables +real_data/fastpath/ fastpath config + cache dir for the real-data scenario +real_data/data/ Downloaded real OONI measurements land here (gitignored, fetched fresh every run) +harness/seed_data.py Synthetic data generator (see note above on why it's synthetic) +harness/real_data.py Real-data end-to-end scenario (see "Real-data end-to-end scenario" above) +harness/ch_http.py Minimal stdlib-only ClickHouse HTTP client +harness/compose.py docker-compose wrapper (bring up/tear down/recreate one node at a time, multi-file overlay support) +harness/validate.py Cluster health checks (replication convergence, error scraping, write/read probes) +harness/scenarios.py The two synthetic-data upgrade scenarios +harness/report.py Results -> Markdown report renderer (both scenario shapes) +ci_step.py Per-step CLI used by the GitHub Actions workflow (all three jobs) +run_test.py CLI entry point (synthetic scenarios only; real-data scenario is CI-only via ci_step.py) +results/ report.md / report.json / golden snapshot land here after a run +``` diff --git a/scripts/clickhouse-upgrade-test/ci_step.py b/scripts/clickhouse-upgrade-test/ci_step.py new file mode 100644 index 00000000..c75df26e --- /dev/null +++ b/scripts/clickhouse-upgrade-test/ci_step.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +""" +Per-step CLI for running the upgrade test as discrete, individually +pass/fail-able steps -- built for .github/workflows/clickhouse_upgrade_test.yml, +where each hop / node-upgrade gets its own GitHub Actions step (own +checkmark, own timing, own expandable log), rather than one opaque job that +only reports pass/fail for the entire upgrade path at once. + +Each invocation is a fresh process. State that would normally be threaded +through function arguments (which image tag each node is currently running) +is instead recovered by inspecting the already-running containers via +`docker inspect` (see harness/compose.py:current_env()) -- so steps are +just plain sequential shell commands in a workflow, no shared state file to +keep in sync, other than the results/steps/*.json this script writes after +every step (used by the final `report` step to assemble a combined summary +from whichever steps actually ran). + +For local one-shot runs (not CI), use run_test.py / `make test` instead -- +this script is intentionally low-level. + +Usage: + python3 ci_step.py setup --base-version 24.8.6.70 --label setup + python3 ci_step.py upgrade-node --node ch1 --version 25.3.14.14 --label hop-25.3-ch1 + python3 ci_step.py verify-ddl --version 25.3.14.14 --label hop-25.3-verify-ddl + python3 ci_step.py report + python3 ci_step.py teardown + +Real-data scenario (harness/real_data.py) -- separate CLI verbs, since it's +a different flow (load real data once, then hop): + python3 ci_step.py setup-real-data --base-version 24.8.6.70 --label setup-real-data + python3 ci_step.py load-real-data --label load-real-data + python3 ci_step.py golden-snapshot --label golden-snapshot + python3 ci_step.py verify-e2e --label base-verify-e2e + python3 ci_step.py real-data-hop --version 25.3.14.14 --label rd-hop1 + python3 ci_step.py report + python3 ci_step.py teardown-real-data + +See .github/workflows/clickhouse_upgrade_test.yml for the full sequence. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from harness import compose, real_data, report +from harness.scenarios import setup_step, step_ok, upgrade_node_step, verify_ddl_step + +PROJECT_DIR = Path(__file__).resolve().parent +RESULTS_DIR = PROJECT_DIR / "results" +STEPS_DIR = RESULTS_DIR / "steps" + + +def _save_step(label: str, result: dict) -> None: + STEPS_DIR.mkdir(parents=True, exist_ok=True) + safe = label.replace("/", "_") + (STEPS_DIR / f"{safe}.json").write_text(json.dumps(result, indent=2, default=str)) + + +def cmd_setup(args) -> int: + result = setup_step(args.base_version, label=args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'FAILED: ' + str(result.get('error'))}") + return 0 if ok else 1 + + +def cmd_upgrade_node(args) -> int: + step = upgrade_node_step(args.node, args.version, label=args.label) + _save_step(args.label, step) + ok = step_ok(step) + print(f"[{args.label}] {'OK' if ok else 'PROBLEM DETECTED'}") + print(json.dumps(step, indent=2, default=str)) + return 0 if ok else 1 + + +def cmd_verify_ddl(args) -> int: + result = verify_ddl_step(args.version, label=args.label) + _save_step(result["label"], result) + ok = step_ok(result) + print(f"[{result['label']}] {'OK' if ok else 'FAILED: ' + str(result.get('error'))}") + return 0 if ok else 1 + + +def cmd_report(args) -> int: + RESULTS_DIR.mkdir(exist_ok=True) + steps = [] + if STEPS_DIR.exists(): + # Chronological order (steps run strictly sequentially within a CI + # job), not alphabetical -- so mtime, not filename, decides order. + for f in sorted(STEPS_DIR.glob("*.json"), key=lambda p: p.stat().st_mtime): + steps.append(json.loads(f.read_text())) + + md = report.render_ci_steps_report(steps) + (RESULTS_DIR / "report.md").write_text(md) + (RESULTS_DIR / "report.json").write_text(json.dumps(steps, indent=2, default=str)) + print(md) + + gh_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if gh_summary: + with open(gh_summary, "a") as f: + f.write(md) + f.write("\n") + + # Individual upgrade-node/verify-ddl steps now run with + # `continue-on-error: true` (see .github/workflows/clickhouse_upgrade_test.yml) + # so that a failure partway through a hop doesn't abort the job before + # the remaining nodes in that hop get a chance to upgrade too -- e.g. so + # we can observe whether a lagging node's stuck replication queue clears + # once it also reaches the new version. That means this step -- which + # has no continue-on-error and runs with `if: always()` -- is now the + # thing that actually has to fail the job when something stayed broken. + any_fail = any(not step_ok(s) for s in steps) + return 1 if any_fail else 0 + + +def cmd_teardown(args) -> int: + try: + compose.down(volumes=True) + except Exception as e: + print(f"teardown warning (non-fatal): {e}") + return 0 + + +# --------------------------------------------------------------------------- +# Real-data scenario (harness/real_data.py) -- see the `real-data-upgrade` +# job in .github/workflows/clickhouse_upgrade_test.yml for the full step +# sequence these are wired into, and README.md for the design writeup. +# --------------------------------------------------------------------------- + + +def cmd_setup_real_data(args) -> int: + result = real_data.setup_real_data_step(args.base_version, label=args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'FAILED: ' + str(result.get('error'))}") + return 0 if ok else 1 + + +def cmd_load_real_data(args) -> int: + result = real_data.load_real_data_step(label=args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'FAILED'}") + print(json.dumps(result, indent=2, default=str)) + return 0 if ok else 1 + + +def cmd_golden_snapshot(args) -> int: + result = real_data.take_golden_snapshot_step(label=args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'FAILED: nodes disagree before any upgrade -- ' + str(result.get('mismatched_tables'))}") + return 0 if ok else 1 + + +def cmd_verify_e2e(args) -> int: + result = real_data.run_e2e_verify_step(args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'FAILED (exit ' + str(result.get('exit_code')) + ')'}") + return 0 if ok else 1 + + +def cmd_real_data_hop(args) -> int: + result = real_data.real_data_hop_step(args.version, args.label) + _save_step(args.label, result) + ok = step_ok(result) + print(f"[{args.label}] {'OK' if ok else 'PROBLEM DETECTED'}") + print(json.dumps(result, indent=2, default=str)) + return 0 if ok else 1 + + +def cmd_teardown_real_data(args) -> int: + try: + compose.down(volumes=True, files=real_data.COMPOSE_FILES) + except Exception as e: + print(f"teardown warning (non-fatal): {e}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + p_setup = sub.add_parser("setup", help="Tear down any previous state, bring up a fresh cluster, load schema + seed data") + p_setup.add_argument("--base-version", required=True, help="ClickHouse image tag for all 3 nodes at startup") + p_setup.add_argument("--label", default="setup", help="Step label, used as the results/steps/