From 6c9b34352bbca99a86828903300c3f9c4e758ebe Mon Sep 17 00:00:00 2001 From: Ryuz Date: Sat, 14 Nov 2026 21:44:22 +0100 Subject: [PATCH 1/3] Release preflight: check-config gate, OUTER_ENGINE, shared boot validation deploy/release.sh runs the new image's own outer check-config on the host (same --env-file, same read-only /tls mount, same OUTER_HOST_ID handling) after the image lands and before anything changes prod; a non-zero exit aborts with prod untouched. OUTER_ENGINE=docker|podman selects the local engine (auto keeps podman-first); invalid refuses. check-config and serve now share one validation: Config::boot_file_problems (TLS triple read+parse, the part of boot from_env never covered), called by serve before it touches disk/network/sockets. No new side effects, errors name variables never values. Rust tests cover valid/missing/garbage TLS plus a secret-leak sweep; deploy/test-release-preflight.sh stubs ssh and the engines to prove the abort order (fails on the old script). --- .github/workflows/ci.yml | 1 + crates/outer/src/config.rs | 163 +++++++++++++++++++++++++++++++ crates/outer/src/main.rs | 26 ++++- deploy/release.sh | 72 +++++++++++--- deploy/test-release-preflight.sh | 140 ++++++++++++++++++++++++++ 5 files changed, 388 insertions(+), 14 deletions(-) create mode 100755 deploy/test-release-preflight.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 525e8db..1475c2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,4 +96,5 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - run: sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck - run: shellcheck deploy/*.sh scripts/*.sh + - run: bash deploy/test-release-preflight.sh - run: scripts/check-pins.sh diff --git a/crates/outer/src/config.rs b/crates/outer/src/config.rs index fd87d77..f0b39e1 100644 --- a/crates/outer/src/config.rs +++ b/crates/outer/src/config.rs @@ -956,6 +956,28 @@ impl Config { self.tls_cert.is_some() && self.tls_key.is_some() && self.origin_pull_ca.is_some() } + /// Boot-time file checks shared by `serve` and `check-config`: every file + /// the config points at that `serve` must read to listen (the TLS triple) + /// is read and parsed here first. Pure reads — no DB, no network, no + /// sockets — and every failure names the variable, never its value. + /// `validate` (above) already covers presence and consistency plus the + /// `OUTER_REVIEW_LOGINS_FILE` read; this covers readability, which used + /// to fail only at boot, after the old container was already stopping. + pub fn boot_file_problems(&self) -> Vec { + let mut problems = Vec::new(); + if self.tls_enabled() { + let files = crate::http::tls::TlsFiles { + cert: self.tls_cert.clone().expect("tls triple is set"), + key: self.tls_key.clone().expect("tls triple is set"), + ca: self.origin_pull_ca.clone().expect("tls triple is set"), + }; + if let Err(err) = crate::http::tls::load_server_config(&files) { + problems.push(format!("{err:#}")); + } + } + problems + } + /// True when the platform oxfmt binary exists and is executable. /// Formatting is skipped (commits still work) when this is false. pub fn oxfmt_available(&self) -> bool { @@ -1845,4 +1867,145 @@ mod tests { } } } + /// Minimal production vars: everything `validate` requires off localhost. + /// Every secret carries `NEVER-LEAK` so tests can assert errors name the + /// variable without echoing its value. + fn prod_vars(extra: &[(&str, &str)]) -> BTreeMap { + let mut base = vec![ + ("OUTER_DOMAIN", "prod.example"), + ("OUTER_STORE", "r2"), + ( + "OUTER_R2_ENDPOINT", + "https://acct.eu.r2.cloudflarestorage.com", + ), + ("OUTER_R2_BUCKET", "outer"), + ("R2_ACCESS_KEY_ID", "r2-key-NEVER-LEAK-1"), + ("R2_SECRET_ACCESS_KEY", "r2-secret-NEVER-LEAK-2"), + ("CLOUDFLARE_ACCOUNT_ID", "cf-acct"), + ("CLOUDFLARE_API_TOKEN", "cf-token-NEVER-LEAK-3"), + ("STRIPE_SECRET_KEY", "sk-live-NEVER-LEAK-4"), + ("STRIPE_WEBHOOK_SECRET", "whsec-NEVER-LEAK-5"), + ("STRIPE_PRICE_PRO", "price-NEVER-LEAK-6"), + ("OUTER_TERMS_VERSION", "2026-11-14"), + ("OUTER_SECRET", "outer-secret-NEVER-LEAK-7"), + ]; + base.extend_from_slice(extra); + vars(&base) + } + + fn assert_no_secrets(problems: &[String]) { + let shown = problems.join("\n"); + assert!( + !shown.contains("NEVER-LEAK"), + "a secret value leaked: {shown}" + ); + assert!(!shown.contains("$argon2"), "a hash leaked: {shown}"); + } + + #[test] + fn boot_files_accept_localhost_without_tls() { + let cfg = Config::from_vars(&vars(&[("OUTER_STORE", "memory")])).unwrap(); + assert!(cfg.boot_file_problems().is_empty()); + } + + #[test] + fn boot_files_refuse_missing_tls_files_without_echoing_values() { + let dir = tempfile::tempdir().unwrap(); + let missing = |name: &str| dir.path().join(name).to_string_lossy().into_owned(); + let cfg = Config::from_vars(&prod_vars(&[ + ("OUTER_TLS_CERT", missing("origin.pem").as_str()), + ("OUTER_TLS_KEY", missing("origin.key").as_str()), + ( + "OUTER_ORIGIN_PULL_CA", + missing("origin-pull-ca.pem").as_str(), + ), + ])) + .unwrap(); + let problems = cfg.boot_file_problems(); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("OUTER_TLS_CERT"), "{problems:?}"); + assert_no_secrets(&problems); + } + + #[test] + fn boot_files_refuse_garbage_tls_pem_without_echoing_values() { + let dir = tempfile::tempdir().unwrap(); + for name in ["origin.pem", "origin.key", "origin-pull-ca.pem"] { + std::fs::write(dir.path().join(name), "not pem at all\n").unwrap(); + } + let path = |name: &str| dir.path().join(name).to_string_lossy().into_owned(); + let cfg = Config::from_vars(&prod_vars(&[ + ("OUTER_TLS_CERT", path("origin.pem").as_str()), + ("OUTER_TLS_KEY", path("origin.key").as_str()), + ("OUTER_ORIGIN_PULL_CA", path("origin-pull-ca.pem").as_str()), + ])) + .unwrap(); + let problems = cfg.boot_file_problems(); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("OUTER_TLS_CERT"), "{problems:?}"); + assert_no_secrets(&problems); + } + + #[test] + fn boot_files_accept_a_valid_tls_triple() { + let dir = tempfile::tempdir().unwrap(); + let mut ca_params = rcgen::CertificateParams::new(vec!["test CA".to_string()]).unwrap(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().unwrap(); + let ca_cert = ca_params.self_signed(&ca_key).unwrap(); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + let params = rcgen::CertificateParams::new(vec!["outer.test".to_string()]).unwrap(); + let key = rcgen::KeyPair::generate().unwrap(); + let cert = params.signed_by(&key, &issuer).unwrap(); + let blobs = [ + ("OUTER_TLS_CERT", "origin.pem", cert.pem().into_bytes()), + ( + "OUTER_TLS_KEY", + "origin.key", + key.serialize_pem().into_bytes(), + ), + ( + "OUTER_ORIGIN_PULL_CA", + "origin-pull-ca.pem", + ca_cert.pem().into_bytes(), + ), + ]; + let mut map = prod_vars(&[]); + for (var, name, bytes) in &blobs { + let file = dir.path().join(name); + std::fs::write(&file, bytes).unwrap(); + map.insert(var.to_string(), file.to_string_lossy().into_owned()); + } + let cfg = Config::from_vars(&map).unwrap(); + assert!(cfg.boot_file_problems().is_empty(), "{cfg:?}"); + } + + #[test] + fn config_errors_never_echo_secret_values() { + // Missing Stripe off localhost plus a malformed review-logins entry: + // every problem must name its variable while echoing no value. + let dir = tempfile::tempdir().unwrap(); + let logins = dir.path().join("review-logins"); + std::fs::write(&logins, "review@example.com:not-a-phc-NEVER-LEAK-8\n").unwrap(); + let mut map = prod_vars(&[ + ("OUTER_TLS_CERT", "/tls/origin.pem"), + ("OUTER_TLS_KEY", "/tls/origin.key"), + ("OUTER_ORIGIN_PULL_CA", "/tls/origin-pull-ca.pem"), + ( + "OUTER_REVIEW_LOGINS_FILE", + logins.to_string_lossy().as_ref(), + ), + ]); + map.remove("STRIPE_SECRET_KEY"); + let err = Config::from_vars(&map).unwrap_err(); + assert!( + err.iter().any(|p| p.contains("STRIPE_SECRET_KEY")), + "{err:?}" + ); + assert!( + err.iter().any(|p| p.contains("OUTER_REVIEW_LOGINS_FILE")), + "{err:?}" + ); + assert_no_secrets(&err); + } } diff --git a/crates/outer/src/main.rs b/crates/outer/src/main.rs index b27d705..ea9263e 100644 --- a/crates/outer/src/main.rs +++ b/crates/outer/src/main.rs @@ -57,9 +57,18 @@ async fn main() -> ExitCode { report(serve(config).await) } ["check-config"] => match load_config() { - Some(_) => { - println!("configuration ok"); - ExitCode::SUCCESS + Some(config) => { + let problems = config.boot_file_problems(); + if problems.is_empty() { + println!("configuration ok"); + ExitCode::SUCCESS + } else { + eprintln!("outer: invalid configuration:"); + for problem in problems { + eprintln!(" - {problem}"); + } + ExitCode::from(2) + } } None => ExitCode::from(2), }, @@ -725,6 +734,17 @@ async fn app_list(config: Config) -> anyhow::Result<()> { } async fn serve(config: Config) -> anyhow::Result<()> { + // The same file validation `check-config` runs (one shared function): + // refuse an unreadable TLS triple before touching the data dir, the + // bucket, or any socket. The triple is loaded again below, after the + // bind, to build the actual server config. + let file_problems = config.boot_file_problems(); + if !file_problems.is_empty() { + for problem in &file_problems { + eprintln!("outer: invalid configuration:\n - {problem}"); + } + anyhow::bail!("invalid configuration ({} problem(s))", file_problems.len()); + } std::fs::create_dir_all(&config.data_dir)?; if !config.oxfmt_available() { // An explicitly configured binary that isn't executable is a prod diff --git a/deploy/release.sh b/deploy/release.sh index 883279e..a1f6620 100755 --- a/deploy/release.sh +++ b/deploy/release.sh @@ -5,13 +5,24 @@ # # deploy/release.sh # -# Refuses a dirty tree. Prefers podman, falls back to docker. Streams the -# image with save|load (no registry), uploads compose.yaml + the firewall -# files + handoff.sh to /srv/outer, then hands the new image out: the -# newcomer shares 443 via SO_REUSEPORT while the old container drains -# (refuses new deploys, exits once in-flight ones finish), and the compose -# service is recreated last onto the same image. At no point is 443 without -# a listener, and no in-flight deploy dies with its process. +# Engine: OUTER_ENGINE=docker|podman selects the local container engine for +# build and save. OUTER_ENGINE=auto (the default when unset or empty) keeps +# today's behaviour: prefer podman, fall back to docker. Any other value +# refuses at once. The VPS side always uses docker (provision.sh installs +# it; compose and `docker load` run there). +# +# Refuses a dirty tree. Builds the image, streams it with save|load (no +# registry), then runs the new image's own `outer check-config` on the host +# — as a `docker run --rm` one-off with exactly the environment and mounts +# the compose `outer` service gets (`--env-file /srv/outer/outer.env`, the +# read-only /tls mount, same OUTER_HOST_ID handling) — BEFORE anything +# changes prod. A non-zero exit aborts the release here with prod untouched: +# no uploads, no backup, no handoff. Only then uploads compose.yaml + the +# firewall files + handoff.sh to /srv/outer, then hands the new image out: +# the newcomer shares 443 via SO_REUSEPORT while the old container drains +# (refuses new deploys, exits once in-flight ones finish), and the compose +# service is recreated last onto the same image. At no point is 443 without +# a listener, and no in-flight deploy dies with its process. # # Health check from the box: outer serves TLS with required client # certificates (mTLS), so no unauthenticated probe can pass the handshake — @@ -70,17 +81,56 @@ while read -r version checksum; do exit 1 } done <<<"$APPLIED" +# Local container engine for build and save. OUTER_ENGINE=auto (the default +# when unset or empty) keeps today's behaviour: prefer podman, fall back to +# docker. An explicit engine must exist on PATH; anything else refuses. +ENGINE="${OUTER_ENGINE:-auto}" +case "$ENGINE" in +auto) + if command -v podman >/dev/null 2>&1; then + ENGINE=podman + elif command -v docker >/dev/null 2>&1; then + ENGINE=docker + else + echo "release.sh: need podman or docker on PATH (or set OUTER_ENGINE)" >&2 + exit 1 + fi + ;; +docker | podman) ;; +*) + echo "release.sh: OUTER_ENGINE=$ENGINE is not one of: auto, docker, podman" >&2 + exit 1 + ;; +esac +if ! command -v "$ENGINE" >/dev/null 2>&1; then + echo "release.sh: OUTER_ENGINE=$ENGINE is not on PATH" >&2 + exit 1 +fi IMAGE="outer:$SHA" -if command -v podman >/dev/null 2>&1; then +if [[ "$ENGINE" == "podman" ]]; then # podman names unqualified tags localhost/outer:, and docker load # keeps that name; the remote script retags it to the plain name. podman build --format docker -t "$IMAGE" -f docker/Dockerfile . podman save "$IMAGE" | ssh "$TARGET" docker load -elif command -v docker >/dev/null 2>&1; then +else docker build -t "$IMAGE" -f docker/Dockerfile . docker save "$IMAGE" | ssh "$TARGET" docker load -else - echo "release.sh: need podman or docker on PATH" >&2 +fi + +# Preflight: the new image proves it accepts the live config BEFORE anything +# changes prod (no uploads, no backup, no handoff below). Same environment +# and mounts the compose `outer` service gets: --env-file carries +# /srv/outer/outer.env (including OUTER_HOST_ID when the operator set it; +# otherwise outer defaults to the container hostname on both paths), and +# /tls is the same read-only mount (OUTER_REVIEW_LOGINS_FILE and the TLS +# triple resolve under it). check-config has no side effects (no DB, +# no network, no sockets) and names variables, never values. A non-zero +# exit aborts the release here with prod untouched: past aborts this must +# catch are a malformed OUTER_REVIEW_LOGINS PHC, a review-logins path +# missing its /tls mount, and STRIPE_* missing off localhost. +if ! ssh "$TARGET" docker run --rm --env-file /srv/outer/outer.env \ + -v /srv/outer/tls:/tls:ro "$IMAGE" check-config; then + echo "release.sh: preflight check-config on $IMAGE failed; release aborted, prod untouched (fix the config or the image, then release again)" >&2 exit 1 fi diff --git a/deploy/test-release-preflight.sh b/deploy/test-release-preflight.sh new file mode 100755 index 0000000..cb725e8 --- /dev/null +++ b/deploy/test-release-preflight.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# Fail-before test for deploy/release.sh's preflight gate: stub the ssh and +# engine binaries on PATH, run release.sh against a scratch git repo, and +# assert a failing `check-config` aborts the release before any prod-changing +# step (no scp upload, no rollout heredoc, so no release-backup/handoff). +# +# bash deploy/test-release-preflight.sh +# +# RELEASE_SH points at the script under test (default: this repo's +# deploy/release.sh); HEAD's copy fails case 1 (it has no preflight). +# STUB_KEEP=1 keeps the scratch dir for inspection. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE_SH="${RELEASE_SH:-$SCRIPT_DIR/release.sh}" +TMP="$(mktemp -d)" +trap '[[ "${STUB_KEEP:-0}" == 1 ]] || rm -rf "$TMP"' EXIT +BIN="$TMP/bin" +LOG="$TMP/log" +mkdir -p "$BIN" "$LOG" + +# --- stubs --------------------------------------------------------------- +# ssh: log every call; fake the prod migration query (no rows); the +# check-config one-off exits $STUB_CHECK_CONFIG_EXIT; everything else exits 0 +# without executing (the rollout heredoc is recorded, never run). +cat >"$BIN/ssh" <<'STUB' +#!/usr/bin/env bash +n=0 +while [[ -e "$STUB_LOG/stdin-$n.bin" ]]; do n=$((n + 1)); done +if [ ! -t 0 ]; then cat >"$STUB_LOG/stdin-$n.bin"; else : >"$STUB_LOG/stdin-$n.bin"; fi +printf 'SSH: %s\n' "$*" >>"$STUB_LOG/ssh.log" +case "$*" in +*sqlite3*) exit 0 ;; +*check-config*) exit "${STUB_CHECK_CONFIG_EXIT:-0}" ;; +*) exit 0 ;; +esac +STUB +# scp: log and succeed. Engines: log and succeed (save streams nothing; the +# pipe still exercises the `save | ssh load` shape). +cat >"$BIN/scp" <<'STUB' +#!/usr/bin/env bash +printf 'SCP: %s\n' "$*" >>"$STUB_LOG/scp.log" +STUB +cat >"$BIN/docker" <<'STUB' +#!/usr/bin/env bash +printf 'DOCKER: %s\n' "$*" >>"$STUB_LOG/engine.log" +exit 0 +STUB +sed 's/DOCKER:/PODMAN:/' "$BIN/docker" >"$BIN/podman-stub" +chmod +x "$BIN/ssh" "$BIN/scp" "$BIN/docker" "$BIN/podman-stub" + +export STUB_LOG="$LOG" +# Hermetic PATH: symlinks to exactly the real tools the test and release.sh +# need, so a workstation podman/docker can never leak in and the cases +# control engine presence by installing stubs. +for tool in bash git sha384sum cut grep sed head mktemp rm mkdir chmod cp cat dirname; do + ln -s "$(command -v "$tool")" "$BIN/$tool" +done +export PATH="$BIN" + +# --- scratch repo (release.sh refuses a dirty tree and reads HEAD) -------- +FIX="$TMP/fixture" +mkdir -p "$FIX" +git -C "$FIX" init -q +git -C "$FIX" -c user.email=t@t -c user.name=t commit -q --allow-empty -m fixture +fail() { + echo "test-release-preflight: FAIL: $1" >&2 + sed 's/^/ release-err: /' "$LOG/err.log" >&2 || true + exit 1 +} + +# $1: comma-separated stubs to install ("docker", "docker,podman"). +reset_case() { + rm -f "$LOG"/ssh.log "$LOG"/scp.log "$LOG"/engine.log "$LOG"/stdin-*.bin "$LOG"/out.log "$LOG"/err.log + rm -f "$BIN/podman" + if [[ "$1" == *podman* ]]; then cp "$BIN/podman-stub" "$BIN/podman"; fi + chmod +x "$BIN/ssh" "$BIN/scp" "$BIN/docker" + [[ -f "$BIN/podman" ]] && chmod +x "$BIN/podman" || true + unset OUTER_ENGINE || true + export STUB_CHECK_CONFIG_EXIT=0 +} + +run_release() { + (cd "$FIX" && bash "$RELEASE_SH" fake-target "$LOG/out.log" 2>"$LOG/err.log"; echo "$?") +} + +# --- case 1: failing check-config aborts before any prod-changing step ----- +reset_case "docker,podman" +export STUB_CHECK_CONFIG_EXIT=1 +code="$(run_release)" +[[ "$code" != 0 ]] || fail "failing check-config exited 0 (expected abort)" +grep -q "check-config" "$LOG/ssh.log" || fail "preflight check-config never ran" +[[ ! -f "$LOG/scp.log" ]] || fail "scp upload ran after a failed preflight" +if grep -q "bash -s" "$LOG/ssh.log"; then fail "rollout ran after a failed preflight"; fi +grep -q "prod untouched" "$LOG/err.log" || fail "abort message does not say prod is untouched" +echo "case 1 ok: failed check-config aborts before scp/rollout (exit $code)" + +# --- case 2: passing check-config proceeds, in order ----------------------- +reset_case "docker,podman" +code="$(run_release)" +[[ "$code" == 0 ]] || fail "passing check-config exited $code" +pre="$(grep -n "check-config" "$LOG/ssh.log" | head -n1 | cut -d: -f1)" +roll="$(grep -n "bash -s" "$LOG/ssh.log" | head -n1 | cut -d: -f1)" +[[ -n "$pre" && -n "$roll" && "$pre" -lt "$roll" ]] || fail "preflight did not precede the rollout" +[[ -f "$LOG/scp.log" ]] || fail "passing preflight never reached the upload" +grep -l "handoff_to" "$LOG"/stdin-*.bin >/dev/null || fail "rollout heredoc missing handoff_to" +grep -l "release-backup" "$LOG"/stdin-*.bin >/dev/null || fail "rollout heredoc missing release-backup" +echo "case 2 ok: passing check-config proceeds to upload then rollout (exit 0)" + +# --- case 3: invalid engine refuses ---------------------------------------- +reset_case "docker" +export OUTER_ENGINE=bogus +code="$(run_release)" +[[ "$code" != 0 ]] || fail "OUTER_ENGINE=bogus exited 0" +grep -q "OUTER_ENGINE=bogus" "$LOG/err.log" || fail "invalid engine message names no value" +[[ ! -f "$LOG/engine.log" ]] || fail "an engine ran despite an invalid OUTER_ENGINE" +echo "case 3 ok: invalid OUTER_ENGINE refuses" + +# --- case 4: explicit engine absent from PATH refuses ----------------------- +reset_case "docker" +export OUTER_ENGINE=podman +code="$(run_release)" +[[ "$code" != 0 ]] || fail "missing podman engine exited 0" +grep -q "OUTER_ENGINE=podman is not on PATH" "$LOG/err.log" || fail "missing engine message unclear" +echo "case 4 ok: explicit engine absent from PATH refuses" + +# --- case 5: auto keeps today's behaviour (podman first, else docker) ------ +reset_case "docker,podman" +code="$(run_release)" +[[ "$code" == 0 ]] || fail "auto with both engines exited $code" +head -n1 "$LOG/engine.log" | grep -q "^PODMAN: build" || fail "auto did not prefer podman" +reset_case "docker" +code="$(run_release)" +[[ "$code" == 0 ]] || fail "auto with docker only exited $code" +head -n1 "$LOG/engine.log" | grep -q "^DOCKER: build" || fail "auto did not fall back to docker" +echo "case 5 ok: auto prefers podman, falls back to docker" + +echo "test-release-preflight: all cases passed" From 22311ab8a07d623028a8fb3382dea8a19113317e Mon Sep 17 00:00:00 2001 From: Ryuz Date: Sat, 14 Nov 2026 21:55:18 +0100 Subject: [PATCH 2/3] Release preflight fixes: early podman retag, never pull local-only images Podman-built images arrive as localhost/outer:; retag to the plain name in its own ssh step right after save|load (one place, before the preflight) instead of inside the rollout heredoc. docker run with a missing local image would pull a same-named registry image and run it with the production env: --pull=never on the preflight, the release-backup one-off and the outer-next run (which also keeps its image-inspect guard), plus pull_policy: never on the compose outer service. Test gains the retag-order and --pull=never cases; both fail on the pre-fix script. --- deploy/compose.yaml | 5 +++- deploy/handoff.sh | 2 +- deploy/release.sh | 51 +++++++++++++++++++------------- deploy/test-release-preflight.sh | 19 ++++++++++-- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/deploy/compose.yaml b/deploy/compose.yaml index c0331e1..774aef5 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -17,7 +17,10 @@ name: outer services: outer: image: ${OUTER_IMAGE:?set OUTER_IMAGE in /srv/outer/release.env (deploy/release.sh writes it)} - restart: unless-stopped + # The image is local-only (deploy/release.sh streams it in with + # save|load): never pull, so a missing image fails closed instead of + # running a same-named registry image with the production env. + pull_policy: never network_mode: host # Above outer's stop budget: on SIGTERM outer refuses new deploys and # waits out the in-flight ones (a release handoff drains this way), and diff --git a/deploy/handoff.sh b/deploy/handoff.sh index 59d4918..bcc18a1 100755 --- a/deploy/handoff.sh +++ b/deploy/handoff.sh @@ -170,7 +170,7 @@ handoff_to() { docker rm -f outer-next >/dev/null 2>&1 || true # Same volumes, env, caps and host network as the compose service # (deploy/compose.yaml): the newcomer is the service, started by hand. - docker run -d --name outer-next --network host \ + docker run -d --pull=never --name outer-next --network host \ --cap-add NET_ADMIN --cap-add SETUID --cap-add SETGID --cap-add CHOWN --cap-add SYS_ADMIN \ --security-opt apparmor:unconfined \ --env-file /srv/outer/outer.env \ diff --git a/deploy/release.sh b/deploy/release.sh index a1f6620..be7c62b 100755 --- a/deploy/release.sh +++ b/deploy/release.sh @@ -109,7 +109,7 @@ fi IMAGE="outer:$SHA" if [[ "$ENGINE" == "podman" ]]; then # podman names unqualified tags localhost/outer:, and docker load - # keeps that name; the remote script retags it to the plain name. + # keeps that name; the retag step below restores the plain name. podman build --format docker -t "$IMAGE" -f docker/Dockerfile . podman save "$IMAGE" | ssh "$TARGET" docker load else @@ -117,19 +117,33 @@ else docker save "$IMAGE" | ssh "$TARGET" docker load fi -# Preflight: the new image proves it accepts the live config BEFORE anything -# changes prod (no uploads, no backup, no handoff below). Same environment -# and mounts the compose `outer` service gets: --env-file carries -# /srv/outer/outer.env (including OUTER_HOST_ID when the operator set it; -# otherwise outer defaults to the container hostname on both paths), and -# /tls is the same read-only mount (OUTER_REVIEW_LOGINS_FILE and the TLS -# triple resolve under it). check-config has no side effects (no DB, -# no network, no sockets) and names variables, never values. A non-zero -# exit aborts the release here with prod untouched: past aborts this must -# catch are a malformed OUTER_REVIEW_LOGINS PHC, a review-logins path -# missing its /tls mount, and STRIPE_* missing off localhost. -if ! ssh "$TARGET" docker run --rm --env-file /srv/outer/outer.env \ - -v /srv/outer/tls:/tls:ro "$IMAGE" check-config; then +# A podman-built image arrives named localhost/outer: (docker load +# keeps the name); retag it to the plain name now, before the preflight and +# the rollout consume it. Retagging changes nothing that is running. (A +# docker-built image is already plain-named, so the tag is a silent no-op +# there.) +# SHA is hex-validated above; expanding it here on the workstation is +# intended, same as the rollout heredoc below. +# shellcheck disable=SC2029 + if ssh "$TARGET" docker tag "localhost/outer:$SHA" "outer:$SHA" 2>/dev/null; then + ssh "$TARGET" docker rmi "localhost/outer:$SHA" >/dev/null + fi + + # Preflight: the new image proves it accepts the live config BEFORE anything + # changes prod (no uploads, no backup, no handoff below). Same environment + # and mounts the compose `outer` service gets: --env-file carries + # /srv/outer/outer.env (including OUTER_HOST_ID when the operator set it; + # otherwise outer defaults to the container hostname on both paths), and + # /tls is the same read-only mount (OUTER_REVIEW_LOGINS_FILE and the TLS + # triple resolve under it). check-config has no side effects (no DB, + # no network, no sockets) and names variables, never values. --pull=never: + # a missing local image must fail here, never pull a same-named registry + # image and run it with the production env. A non-zero exit aborts the + # release here with prod untouched: past aborts this must catch are a + # malformed OUTER_REVIEW_LOGINS PHC, a review-logins path missing its /tls + # mount, and STRIPE_* missing off localhost. + if ! ssh "$TARGET" docker run --rm --pull=never --env-file /srv/outer/outer.env \ + -v /srv/outer/tls:/tls:ro "$IMAGE" check-config; then echo "release.sh: preflight check-config on $IMAGE failed; release aborted, prod untouched (fix the config or the image, then release again)" >&2 exit 1 fi @@ -146,11 +160,6 @@ set -euo pipefail SHA="$1" cd /srv/outer -if docker image inspect "localhost/outer:$SHA" >/dev/null 2>&1; then - docker tag "localhost/outer:$SHA" "outer:$SHA" - docker rmi "localhost/outer:$SHA" >/dev/null -fi - [[ -f /srv/outer/outer.env ]] || { echo "remote: /srv/outer/outer.env missing (copy deploy/outer.env.example and fill it)" >&2 exit 1 @@ -188,9 +197,9 @@ trap 'handoff_abort' EXIT # (exit 0) when the bucket holds no snapshot yet. SHA is hex (validated # above), so interpolating it into the docker args is safe. No data mount: # the subcommand only reads config env and the bucket, never the local DB. -# The tls mount (read-only, like the compose service) is required: config # validation reads files under /tls (e.g. OUTER_REVIEW_LOGINS_FILE). -docker run --rm --env-file /srv/outer/outer.env -v /srv/outer/tls:/tls:ro \ +# --pull=never: the image is local-only (same as the preflight above). +docker run --rm --pull=never --env-file /srv/outer/outer.env -v /srv/outer/tls:/tls:ro \ "outer:$SHA" state release-backup --sha "$SHA" handoff_to "outer:$SHA" trap - EXIT diff --git a/deploy/test-release-preflight.sh b/deploy/test-release-preflight.sh index cb725e8..de51400 100755 --- a/deploy/test-release-preflight.sh +++ b/deploy/test-release-preflight.sh @@ -101,10 +101,10 @@ echo "case 1 ok: failed check-config aborts before scp/rollout (exit $code)" reset_case "docker,podman" code="$(run_release)" [[ "$code" == 0 ]] || fail "passing check-config exited $code" -pre="$(grep -n "check-config" "$LOG/ssh.log" | head -n1 | cut -d: -f1)" -roll="$(grep -n "bash -s" "$LOG/ssh.log" | head -n1 | cut -d: -f1)" +pre="$(grep -n "check-config" "$LOG/ssh.log" | head -n1 | cut -d: -f1 || true)" +roll="$(grep -n "bash -s" "$LOG/ssh.log" | head -n1 | cut -d: -f1 || true)" [[ -n "$pre" && -n "$roll" && "$pre" -lt "$roll" ]] || fail "preflight did not precede the rollout" -[[ -f "$LOG/scp.log" ]] || fail "passing preflight never reached the upload" +grep "check-config" "$LOG/ssh.log" | grep -q -- "--pull=never" || fail "preflight run lacks --pull=never" grep -l "handoff_to" "$LOG"/stdin-*.bin >/dev/null || fail "rollout heredoc missing handoff_to" grep -l "release-backup" "$LOG"/stdin-*.bin >/dev/null || fail "rollout heredoc missing release-backup" echo "case 2 ok: passing check-config proceeds to upload then rollout (exit 0)" @@ -137,4 +137,17 @@ code="$(run_release)" head -n1 "$LOG/engine.log" | grep -q "^DOCKER: build" || fail "auto did not fall back to docker" echo "case 5 ok: auto prefers podman, falls back to docker" +# --- case 6: podman images are retagged before the preflight --------------- +# A podman build arrives on the host as localhost/outer:; release.sh +# must retag it to the plain name in its own ssh step before the preflight's +# `docker run … outer:` (the rollout heredoc is too late). +reset_case "docker,podman" +code="$(run_release)" +[[ "$code" == 0 ]] || fail "podman pass-through exited $code" +head -n1 "$LOG/engine.log" | grep -q "^PODMAN: build" || fail "case 6 did not build with podman" +tag="$(grep -n "docker tag.*localhost/outer" "$LOG/ssh.log" | head -n1 | cut -d: -f1 || true)" +pre6="$(grep -n "check-config" "$LOG/ssh.log" | head -n1 | cut -d: -f1 || true)" +[[ -n "$tag" && -n "$pre6" && "$tag" -lt "$pre6" ]] || fail "retag did not precede the preflight" +echo "case 6 ok: podman image retagged before check-config (exit 0)" + echo "test-release-preflight: all cases passed" From 2d7be7f1d2d3099f87cce2d9fe066926df31338c Mon Sep 17 00:00:00 2001 From: Ryuz Date: Sat, 14 Nov 2026 22:10:37 +0100 Subject: [PATCH 3/3] Review fixes: restore restart, strict retag, preflight /data:ro, gate before prepare_host - deploy/compose.yaml: restore restart: unless-stopped (dropped by the pull_policy edit); keep pull_policy: never. - deploy/release.sh: podman retag is required in-branch (set -e, no if swallowing) so a failed retag aborts before the preflight; preflight also mounts /srv/outer/data:/data:ro, mirroring every compose outer volume read-only. - crates/outer/src/main.rs: the shared boot file gate (report_boot_files) runs in the serve arm before prepare_host touches the host; serve keeps no duplicate check. check-config uses the same gate. - deploy/test-release-preflight.sh: case 7 (failed retag aborts before check-config), case 8 (compose/preflight parity: service keys plus volume-set equality with read-only mounts). Both new cases fail on the pre-fix script. --- crates/outer/src/main.rs | 78 +++++++++++++++++++++++++------- deploy/compose.yaml | 1 + deploy/release.sh | 57 ++++++++++++----------- deploy/test-release-preflight.sh | 36 +++++++++++++-- 4 files changed, 122 insertions(+), 50 deletions(-) diff --git a/crates/outer/src/main.rs b/crates/outer/src/main.rs index ea9263e..23be9ee 100644 --- a/crates/outer/src/main.rs +++ b/crates/outer/src/main.rs @@ -50,6 +50,12 @@ async fn main() -> ExitCode { let Some(config) = load_config() else { return ExitCode::from(2); }; + // The same file validation `check-config` runs, before + // `prepare_host` creates dirs and chmods: refuse an unreadable + // TLS triple before touching the host. + if !report_boot_files(&config) { + return ExitCode::from(2); + } init_tracing(&config.log_filter); if let Err(err) = prepare_host(&config) { return report(Err(err)); @@ -58,15 +64,10 @@ async fn main() -> ExitCode { } ["check-config"] => match load_config() { Some(config) => { - let problems = config.boot_file_problems(); - if problems.is_empty() { + if report_boot_files(&config) { println!("configuration ok"); ExitCode::SUCCESS } else { - eprintln!("outer: invalid configuration:"); - for problem in problems { - eprintln!(" - {problem}"); - } ExitCode::from(2) } } @@ -182,6 +183,21 @@ fn load_config() -> Option { } } +/// The one shared boot file check: print `Config::boot_file_problems` and +/// report whether the config is bootable. `serve` runs this before +/// `prepare_host` touches the host; `check-config` is this and nothing else. +fn report_boot_files(config: &Config) -> bool { + let problems = config.boot_file_problems(); + if problems.is_empty() { + return true; + } + eprintln!("outer: invalid configuration:"); + for problem in problems { + eprintln!(" - {problem}"); + } + false +} + fn build_store(config: &Config) -> anyhow::Result { match &config.store { StoreKind::Memory => { @@ -734,17 +750,6 @@ async fn app_list(config: Config) -> anyhow::Result<()> { } async fn serve(config: Config) -> anyhow::Result<()> { - // The same file validation `check-config` runs (one shared function): - // refuse an unreadable TLS triple before touching the data dir, the - // bucket, or any socket. The triple is loaded again below, after the - // bind, to build the actual server config. - let file_problems = config.boot_file_problems(); - if !file_problems.is_empty() { - for problem in &file_problems { - eprintln!("outer: invalid configuration:\n - {problem}"); - } - anyhow::bail!("invalid configuration ({} problem(s))", file_problems.len()); - } std::fs::create_dir_all(&config.data_dir)?; if !config.oxfmt_available() { // An explicitly configured binary that isn't executable is a prod @@ -1232,3 +1237,42 @@ async fn drain_and_wait(pool: sqlx::SqlitePool, drain: Option BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn boot_gate_passes_localhost_and_refuses_garbage_tls() { + // The gate both `serve` (before `prepare_host`) and `check-config` + // share: localhost without TLS passes, an unreadable triple fails. + // Presence alone is not enough — `validate` accepts a set triple + // and only this gate reads it. + let ok = Config::from_vars(&vars(&[("OUTER_STORE", "memory")])).unwrap(); + assert!(report_boot_files(&ok)); + let dir = tempfile::tempdir().unwrap(); + let cert = dir.path().join("origin.pem"); + std::fs::write(&cert, "not pem at all\n").unwrap(); + let missing = dir + .path() + .join("missing.pem") + .to_string_lossy() + .into_owned(); + let bad = Config::from_vars(&vars(&[ + ("OUTER_STORE", "memory"), + ("OUTER_TLS_CERT", cert.to_string_lossy().as_ref()), + ("OUTER_TLS_KEY", missing.as_str()), + ("OUTER_ORIGIN_PULL_CA", missing.as_str()), + ])) + .unwrap(); + assert!(!report_boot_files(&bad)); + } +} diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 774aef5..d97af24 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -17,6 +17,7 @@ name: outer services: outer: image: ${OUTER_IMAGE:?set OUTER_IMAGE in /srv/outer/release.env (deploy/release.sh writes it)} + restart: unless-stopped # The image is local-only (deploy/release.sh streams it in with # save|load): never pull, so a missing image fails closed instead of # running a same-named registry image with the production env. diff --git a/deploy/release.sh b/deploy/release.sh index be7c62b..1200a7d 100755 --- a/deploy/release.sh +++ b/deploy/release.sh @@ -15,7 +15,7 @@ # registry), then runs the new image's own `outer check-config` on the host # — as a `docker run --rm` one-off with exactly the environment and mounts # the compose `outer` service gets (`--env-file /srv/outer/outer.env`, the -# read-only /tls mount, same OUTER_HOST_ID handling) — BEFORE anything +# read-only /tls and /data mounts, same OUTER_HOST_ID handling) — BEFORE anything # changes prod. A non-zero exit aborts the release here with prod untouched: # no uploads, no backup, no handoff. Only then uploads compose.yaml + the # firewall files + handoff.sh to /srv/outer, then hands the new image out: @@ -109,41 +109,40 @@ fi IMAGE="outer:$SHA" if [[ "$ENGINE" == "podman" ]]; then # podman names unqualified tags localhost/outer:, and docker load - # keeps that name; the retag step below restores the plain name. + # keeps that name. Require the retag to the plain name here, before the + # preflight and the rollout consume it: a failed retag aborts (set -e), + # never letting a stale outer:$SHA pass the preflight. Retagging changes + # nothing that is running. + # SHA is hex-validated above; expanding it here on the workstation is + # intended, same as the rollout heredoc below. podman build --format docker -t "$IMAGE" -f docker/Dockerfile . podman save "$IMAGE" | ssh "$TARGET" docker load + # shellcheck disable=SC2029 + ssh "$TARGET" docker tag "localhost/outer:$SHA" "outer:$SHA" + # shellcheck disable=SC2029 + ssh "$TARGET" docker rmi "localhost/outer:$SHA" >/dev/null else docker build -t "$IMAGE" -f docker/Dockerfile . docker save "$IMAGE" | ssh "$TARGET" docker load fi -# A podman-built image arrives named localhost/outer: (docker load -# keeps the name); retag it to the plain name now, before the preflight and -# the rollout consume it. Retagging changes nothing that is running. (A -# docker-built image is already plain-named, so the tag is a silent no-op -# there.) -# SHA is hex-validated above; expanding it here on the workstation is -# intended, same as the rollout heredoc below. -# shellcheck disable=SC2029 - if ssh "$TARGET" docker tag "localhost/outer:$SHA" "outer:$SHA" 2>/dev/null; then - ssh "$TARGET" docker rmi "localhost/outer:$SHA" >/dev/null - fi - - # Preflight: the new image proves it accepts the live config BEFORE anything - # changes prod (no uploads, no backup, no handoff below). Same environment - # and mounts the compose `outer` service gets: --env-file carries - # /srv/outer/outer.env (including OUTER_HOST_ID when the operator set it; - # otherwise outer defaults to the container hostname on both paths), and - # /tls is the same read-only mount (OUTER_REVIEW_LOGINS_FILE and the TLS - # triple resolve under it). check-config has no side effects (no DB, - # no network, no sockets) and names variables, never values. --pull=never: - # a missing local image must fail here, never pull a same-named registry - # image and run it with the production env. A non-zero exit aborts the - # release here with prod untouched: past aborts this must catch are a - # malformed OUTER_REVIEW_LOGINS PHC, a review-logins path missing its /tls - # mount, and STRIPE_* missing off localhost. - if ! ssh "$TARGET" docker run --rm --pull=never --env-file /srv/outer/outer.env \ - -v /srv/outer/tls:/tls:ro "$IMAGE" check-config; then +# Preflight: the new image proves it accepts the live config BEFORE anything +# changes prod (no uploads, no backup, no handoff below). Same environment +# and mounts the compose `outer` service gets: --env-file carries +# /srv/outer/outer.env (including OUTER_HOST_ID when the operator set it; +# otherwise outer defaults to the container hostname on both paths), and +# every compose volume read-only: /tls (OUTER_REVIEW_LOGINS_FILE and the TLS +# triple resolve under it) and /data (the live DB dir; check-config never +# opens it, and read-only keeps even a buggy validation from touching prod). +# check-config has no side effects (no DB, no network, no sockets) and names +# variables, never values. --pull=never: a missing local image must fail +# here, never pull a same-named registry image and run it with the +# production env. A non-zero exit aborts the release here with prod +# untouched: past aborts this must catch are a malformed OUTER_REVIEW_LOGINS +# PHC, a review-logins path missing its /tls mount, and STRIPE_* missing off +# localhost. +if ! ssh "$TARGET" docker run --rm --pull=never --env-file /srv/outer/outer.env \ + -v /srv/outer/tls:/tls:ro -v /srv/outer/data:/data:ro "$IMAGE" check-config; then echo "release.sh: preflight check-config on $IMAGE failed; release aborted, prod untouched (fix the config or the image, then release again)" >&2 exit 1 fi diff --git a/deploy/test-release-preflight.sh b/deploy/test-release-preflight.sh index de51400..031a636 100755 --- a/deploy/test-release-preflight.sh +++ b/deploy/test-release-preflight.sh @@ -23,8 +23,9 @@ mkdir -p "$BIN" "$LOG" # --- stubs --------------------------------------------------------------- # ssh: log every call; fake the prod migration query (no rows); the -# check-config one-off exits $STUB_CHECK_CONFIG_EXIT; everything else exits 0 -# without executing (the rollout heredoc is recorded, never run). +# check-config one-off exits $STUB_CHECK_CONFIG_EXIT and the podman retag +# exits $STUB_RETAG_EXIT; everything else exits 0 without executing (the +# rollout heredoc is recorded, never run). cat >"$BIN/ssh" <<'STUB' #!/usr/bin/env bash n=0 @@ -34,6 +35,7 @@ printf 'SSH: %s\n' "$*" >>"$STUB_LOG/ssh.log" case "$*" in *sqlite3*) exit 0 ;; *check-config*) exit "${STUB_CHECK_CONFIG_EXIT:-0}" ;; +*docker\ tag\ localhost*) exit "${STUB_RETAG_EXIT:-0}" ;; *) exit 0 ;; esac STUB @@ -55,7 +57,7 @@ export STUB_LOG="$LOG" # Hermetic PATH: symlinks to exactly the real tools the test and release.sh # need, so a workstation podman/docker can never leak in and the cases # control engine presence by installing stubs. -for tool in bash git sha384sum cut grep sed head mktemp rm mkdir chmod cp cat dirname; do +for tool in bash git sha384sum cut grep sed head mktemp rm mkdir chmod cp cat dirname sort; do ln -s "$(command -v "$tool")" "$BIN/$tool" done export PATH="$BIN" @@ -79,7 +81,7 @@ reset_case() { chmod +x "$BIN/ssh" "$BIN/scp" "$BIN/docker" [[ -f "$BIN/podman" ]] && chmod +x "$BIN/podman" || true unset OUTER_ENGINE || true - export STUB_CHECK_CONFIG_EXIT=0 + export STUB_CHECK_CONFIG_EXIT=0 STUB_RETAG_EXIT=0 } run_release() { @@ -150,4 +152,30 @@ pre6="$(grep -n "check-config" "$LOG/ssh.log" | head -n1 | cut -d: -f1 || true)" [[ -n "$tag" && -n "$pre6" && "$tag" -lt "$pre6" ]] || fail "retag did not precede the preflight" echo "case 6 ok: podman image retagged before check-config (exit 0)" +# --- case 7: a failed podman retag aborts before check-config --------------- +# The retag is required on the podman path (set -e, no swallowing): a stale +# outer: must never reach the preflight. +reset_case "docker,podman" +export STUB_RETAG_EXIT=1 +code="$(run_release)" +[[ "$code" != 0 ]] || fail "failed retag exited 0 (expected abort)" +grep -q "docker tag.*localhost/outer" "$LOG/ssh.log" || fail "retag never ran" +if grep -q "check-config" "$LOG/ssh.log"; then fail "check-config ran after a failed retag"; fi +echo "case 7 ok: failed retag aborts before check-config (exit $code)" + +# --- case 8: compose and preflight agree on the outer service --------------- +# The single-service file must keep its identity (a past edit dropped +# `restart`), and the preflight's `-v` set must equal the compose `outer` +# volumes (modes ignored) with every mount read-only, so the two can't drift. +for key in "restart: unless-stopped" "pull_policy: never" "stop_grace_period:"; do + grep -q "$key" "$SCRIPT_DIR/compose.yaml" || fail "compose.yaml lost '$key'" +done +compose_vols="$(grep -o -- '- /srv/outer/[^ :]*:[^ :]*' "$SCRIPT_DIR/compose.yaml" | sed 's/^- //; s/:\(ro\|rw\)$//' | sort -u)" +preflight_line="$(grep -- 'check-config' "$RELEASE_SH" || true)" +preflight_vols="$(echo "$preflight_line" | grep -o -- '-v /srv/outer/[^ ]*' | sed 's/^-v //; s/:\(ro\|rw\)$//' | sort -u)" +[[ -n "$preflight_vols" ]] || fail "no preflight -v mounts found in $RELEASE_SH" +[[ "$compose_vols" == "$preflight_vols" ]] || fail "compose volumes != preflight mounts: [$compose_vols] vs [$preflight_vols]" +if echo "$preflight_line" | grep -o -- '-v /srv/outer/[^ ]*' | grep -vq ':ro$'; then fail "a preflight mount is not read-only"; fi +echo "case 8 ok: compose and preflight agree on service identity and mounts" + echo "test-release-preflight: all cases passed"