diff --git a/README.md b/README.md index 37f5797..08477be 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The same sessions can run in a local k3d cluster (spec section 8.2): the API and 1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo. 2. Run `scripts/k8s-up.sh`. It creates the cluster (1 server, 2 agents), builds and imports the images, generates the database password into git-ignored `deploy/k8s/.env`, and deploys everything. Re-run it after a change. 3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page. -4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard (it counts in the cluster's statistics). +4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard. That session plays on a separate database, `rootcause_cluster_test`, through a second API pod, and leaves the real one untouched. 5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. `RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3). diff --git a/deploy/k8s/api.yaml b/deploy/k8s/api.yaml index ed10beb..71168d7 100644 --- a/deploy/k8s/api.yaml +++ b/deploy/k8s/api.yaml @@ -10,8 +10,11 @@ metadata: namespace: rootcause-system spec: type: NodePort + # Only this Deployment's pod: scripts/k8s-prove.sh runs a second API pod + # (instance prove) that shares the app label, and so the network policies. selector: app: rootcause-api + rootcause.instance: main ports: - port: 7300 targetPort: 7300 @@ -33,6 +36,7 @@ spec: metadata: labels: app: rootcause-api + rootcause.instance: main spec: serviceAccountName: rootcause-api # The kubernetes runtime needs the service account's token; the diff --git a/internal/server/cluster_test.go b/internal/server/cluster_test.go index 9510640..a144f30 100644 --- a/internal/server/cluster_test.go +++ b/internal/server/cluster_test.go @@ -21,15 +21,18 @@ import ( // token from the served page, starts perm-01 over the API, works in the web // terminal, and checks. From the terminal it also tries every address in // RC_CLUSTER_PROBES ("name=ip:port,..."), all of which must be unreachable -// from a sandbox. scripts/k8s-prove.sh runs it with RC_CLUSTER_URL set; it is +// from a sandbox. scripts/k8s-prove.sh runs it with RC_CLUSTER_URL set, at a +// second API pod on a test database it names in RC_CLUSTER_DATABASE; it is // skipped otherwise. -// -// It plays on the cluster's own database, so the session counts there. func TestClusterSession(t *testing.T) { base := os.Getenv("RC_CLUSTER_URL") if base == "" { t.Skip("RC_CLUSTER_URL is not set") } + // The API at base must serve a test database, never the learner's. + if db := os.Getenv("RC_CLUSTER_DATABASE"); !strings.HasSuffix(db, "_test") { + t.Fatalf("RC_CLUSTER_DATABASE=%q: the cluster API under test must use a database ending in _test", db) + } c := &clusterClient{t: t, base: strings.TrimSuffix(base, "/")} c.readToken() diff --git a/scripts/k8s-prove.sh b/scripts/k8s-prove.sh index bb9a740..bf74933 100755 --- a/scripts/k8s-prove.sh +++ b/scripts/k8s-prove.sh @@ -10,11 +10,13 @@ # policies reaches the same addresses, so the blocks come from the # NetworkPolicies, not from the probes. # 3. A session runs fully in the cluster from the browser's side: the -# dashboard's API and web terminal on 127.0.0.1:7300 -# (TestClusterSession), with the network probes repeated from the real -# session pod's terminal. +# dashboard's API and web terminal (TestClusterSession), with the network +# probes repeated from the real session pod's terminal. It plays on its +# own database, rootcause_cluster_test, through a second API pod, and the +# real database must be unchanged afterwards (working rule 10). # -# The probe pods and the rootcause-probe namespace are removed at the end. +# The probe pods, the rootcause-probe namespace, the second API pod, its +# Secret and the test database are removed at the end. # Exit 0 only if every check passes. set -euo pipefail @@ -193,18 +195,82 @@ expect rootcause-probe probe-control "postgres-service=$pg_svc_ip:5432" closed expect rootcause-probe probe-control "sandbox=$target_ip:8080" closed echo "== 3. a session from the browser's side" +# The session plays on its own database, never on the cluster's real one +# (rootcause): a second API pod, instance "prove", copied from the running +# API pod, serves rootcause_cluster_test on port 7301 through kubectl +# port-forward. It, its Secret and the database are removed at the end, and +# the real database's session count must not change. +testdb=rootcause_cluster_test +psql() { kc exec -n rootcause-system postgres-0 -- psql -U rootcause -d "$1" -tAc "$2"; } +forward_pid="" +secretdir="$(mktemp -d)" +cleanup_prove() { + if [[ -n "$forward_pid" ]]; then kill "$forward_pid" 2>/dev/null || true; fi + kc delete pod -n rootcause-system rootcause-api-prove --ignore-not-found --grace-period=0 --wait=true >/dev/null 2>&1 || true + kc delete secret -n rootcause-system rootcause-db-prove --ignore-not-found >/dev/null 2>&1 || true + kc exec -n rootcause-system postgres-0 -- dropdb -U rootcause --if-exists --force "$testdb" >/dev/null 2>&1 || true + rm -rf "$secretdir" +} +trap 'cleanup_prove; cleanup' EXIT +cleanup_prove +secretdir="$(mktemp -d)" +real_before="$(psql rootcause "SELECT count(*) FROM session")" +kc exec -n rootcause-system postgres-0 -- createdb -U rootcause "$testdb" +( + umask 077 + password="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$root/deploy/k8s/.env")" + [[ -n "$password" ]] || { echo "k8s-prove: no POSTGRES_PASSWORD in deploy/k8s/.env" >&2; exit 1; } + printf 'postgres://rootcause:%s@postgres:5432/%s?sslmode=disable' "$password" "$testdb" >"$secretdir/database-url" +) +kc create secret generic rootcause-db-prove -n rootcause-system \ + --from-file=database-url="$secretdir/database-url" >/dev/null +rm -f "$secretdir/database-url" +# The running API pod with the test database, its own port and its own +# instance label, so the Service never sends the dashboard to it. +kc get pod -n rootcause-system -l app=rootcause-api,rootcause.instance=main -o json | python3 -c ' +import json, sys +src = json.load(sys.stdin)["items"][0]["spec"] +c = src["containers"][0] +c["args"] = ["serve", "--listen-all", "--port", "7301"] +c["ports"] = [{"containerPort": 7301}] +c["readinessProbe"] = {"tcpSocket": {"port": 7301}, "periodSeconds": 2} +for e in c["env"]: + if e["name"] == "RC_DATABASE_URL": + e["valueFrom"]["secretKeyRef"]["name"] = "rootcause-db-prove" +c["volumeMounts"] = [m for m in c["volumeMounts"] if not m["name"].startswith("kube-api-access")] +spec = {k: v for k, v in src.items() if k not in ("nodeName", "tolerations", "priority", "preemptionPolicy")} +spec["volumes"] = [v for v in src["volumes"] if not v["name"].startswith("kube-api-access")] +json.dump({"apiVersion": "v1", "kind": "Pod", + "metadata": {"name": "rootcause-api-prove", "namespace": "rootcause-system", + "labels": {"app": "rootcause-api", "rootcause.instance": "prove"}}, + "spec": spec}, sys.stdout) +' | kc apply -f - >/dev/null +kc wait --for=condition=Ready pod -n rootcause-system rootcause-api-prove --timeout=120s >/dev/null +kubectl --context "$context" port-forward -n rootcause-system pod/rootcause-api-prove 7301:7301 >/dev/null 2>&1 & +forward_pid=$! +for _ in $(seq 50); do + curl -fs -o /dev/null http://localhost:7301/ && break + sleep 0.2 +done probes="$(tr ' ' ',' <<<"$targets"),other-sandbox=$target_ip:8080" log="$(mktemp)" rc=0 -RC_CLUSTER_URL=http://localhost:7300 RC_CLUSTER_PROBES="$probes" \ +RC_CLUSTER_URL=http://localhost:7301 RC_CLUSTER_DATABASE="$testdb" RC_CLUSTER_PROBES="$probes" \ go test -count=1 -v -run '^TestClusterSession$' ./internal/server/ >"$log" 2>&1 || rc=$? grep -E '^\s+cluster_test|^(--- |ok|FAIL)' "$log" || true if [[ "$rc" -eq 0 ]] && grep -q -- '--- PASS: TestClusterSession' "$log"; then - pass "TestClusterSession" + pass "TestClusterSession on $testdb" else fail "TestClusterSession (exit $rc)" fi rm -f "$log" +played="$(psql "$testdb" "SELECT count(*) FROM session")" +real_after="$(psql rootcause "SELECT count(*) FROM session")" +if [[ "$played" -ge 1 && "$real_before" == "$real_after" ]]; then + pass "the session went to $testdb ($played session); rootcause unchanged ($real_after sessions)" +else + fail "sessions: $testdb has $played, rootcause went from $real_before to $real_after" +fi echo if [[ "$failures" -gt 0 ]]; then