Phase 7

README.md

@@ -28,6 +28,18 @@ The server listens on 127.0.0.1 only. WSL2 forwards `localhost` from Windows to
 
 Exercises that boot systemd (such as svc-02) need a host with cgroup v2. WSL2 uses cgroup v2 by default. Check with `docker info --format '{{.CgroupVersion}}'`, which must print `2`.
 
+### Kubernetes (v2)
+
+The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser.
+
+1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo.
+2. Run `scripts/k8s-up.sh`. It creates the cluster (1 server, 2 agents), builds and imports the images, generates the database password into git-ignored `deploy/k8s/.env`, and deploys everything. Re-run it after a change.
+3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page.
+4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard (it counts in the cluster's statistics).
+5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob.
+
+`RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3).
+
 ## Tests
 
 Unit tests need nothing: `go test ./...`. Integration tests need a database whose name ends in `_test`, and Docker with userns-remap:
@@ -43,6 +55,7 @@ go test -count=1 ./...
 - Tests that run dbt create and drop their own database (for example `rootcause_selector_test`), because a dbt build reads every row in its database.
 - `RC_DBT_STRESS=1 go test -run TestBuildStress ./internal/dbt/` kills dbt builds at random points and checks that the next build succeeds. `RC_DBT_STRESS_ROUNDS` sets the rounds (default 10).
 - A failed dbt build saves its full `dbt.log` under `~/.cache/rootcause/dbt-logs/` and prints the path. The newest 20 are kept.
+- `TestClusterSession` plays a session against the deployed cluster and runs only with `RC_CLUSTER_URL` set; `scripts/k8s-prove.sh` sets it.
 
 ## CI
 

cmd/rootcause/main.go

@@ -2,7 +2,7 @@
 //
 // The play commands are init, next, start, shell, ticket, hint, check, reset,
 // giveup, status, list, stats, serve (the dashboard) and lang. The developer
-// commands validate, vars and selftest need no database.
+// commands validate, vars, selftest and dbt-project need no database.
 package main
 
 import (
@@ -12,8 +12,8 @@ import (
 	"strings"
 
 	"github.com/DIZAMU/RootCause/internal/cli"
+	"github.com/DIZAMU/RootCause/internal/dbt"
 	"github.com/DIZAMU/RootCause/internal/exercise"
-	"github.com/DIZAMU/RootCause/internal/runtime/docker"
 	"github.com/DIZAMU/RootCause/internal/session"
 )
 
@@ -33,6 +33,8 @@ func main() {
 		os.Exit(cmdVars(args))
 	case "selftest":
 		os.Exit(cmdSelfTest(ctx, args))
+	case "dbt-project":
+		os.Exit(cmdDBTProject(args))
 	case "init":
 		os.Exit(cmdInit(ctx))
 	default:
@@ -115,20 +117,35 @@ func cmdValidate(args []string) int {
 }
 
 // cmdSelfTest runs an exercise's unsolved and solved scenarios in real
-// sandboxes, with the same flags a session uses. It needs Docker, not the
-// database. scripts/validate-exercise.sh calls it.
+// sandboxes, with the same flags a session uses, on the runtime RC_RUNTIME
+// selects. It needs that runtime, not the database.
+// scripts/validate-exercise.sh calls it.
 func cmdSelfTest(ctx context.Context, args []string) int {
 	seed, dir, ok := seedAndDir("selftest", args)
 	if !ok {
 		return 2
 	}
-	rt, err := docker.New()
+	be, err := cli.NewBackend(ctx)
 	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, cli.English(err))
+		return 1
+	}
+	if err := session.SelfTest(ctx, be.Runtime, be.Builder, dir, seed, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "FAIL:", cli.English(err))
 		return 1
 	}
-	if err := session.SelfTest(ctx, rt, rt, dir, seed, os.Stdout); err != nil {
-		fmt.Fprintln(os.Stderr, "FAIL:", err)
+	return 0
+}
+
+// cmdDBTProject writes the embedded dbt project into an existing directory.
+// The cluster's dbt CronJob runs it as an init container (spec section 8.2).
+func cmdDBTProject(args []string) int {
+	if len(args) != 1 {
+		fmt.Fprintln(os.Stderr, "dbt-project: expected exactly one directory")
+		return 2
+	}
+	if err := dbt.WriteProject(args[0]); err != nil {
+		fmt.Fprintln(os.Stderr, "dbt-project:", err)
 		return 1
 	}
 	return 0

dbt/models/marts/mart_build_info.sql

@@ -0,0 +1,9 @@
+-- One row: when the latest dbt build made the marts. The dashboard shows it,
+-- so statistics that lag behind (the v2 CronJob builds every 10 minutes) are
+-- visibly older than the latest session.
+-- depends_on: {{ ref('mart_topic_levels') }}
+-- depends_on: {{ ref('mart_review_queue') }}
+-- depends_on: {{ ref('mart_exercise_stats') }}
+-- depends_on: {{ ref('mart_progress') }}
+
+select now() as built_at

dbt/models/marts/schema.yml

@@ -49,3 +49,8 @@ models:
           - accepted_values:
               values: [1, 2, 3]
               quote: false
+
+  - name: mart_build_info
+    columns:
+      - name: built_at
+        tests: [not_null]

deploy/k8s/api.yaml

@@ -0,0 +1,92 @@
+# The API and dashboard (spec section 8.2): rootcause serve with the
+# kubernetes runtime. It runs as the rootcause-api service account, which may
+# only manage pods in sandboxes (rbac.yaml). The dashboard is reached through
+# the NodePort, which k3d publishes on the host's 127.0.0.1:7300 only
+# (k3d.yaml); the server's Host and token checks apply as on the laptop.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rootcause-api
+  namespace: rootcause-system
+spec:
+  type: NodePort
+  selector:
+    app: rootcause-api
+  ports:
+    - port: 7300
+      targetPort: 7300
+      nodePort: 30730
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rootcause-api
+  namespace: rootcause-system
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rootcause-api
+  template:
+    metadata:
+      labels:
+        app: rootcause-api
+    spec:
+      serviceAccountName: rootcause-api
+      # The kubernetes runtime needs the service account's token; the
+      # ServiceAccount itself does not mount it anywhere else.
+      automountServiceAccountToken: true
+      enableServiceLinks: false
+      hostUsers: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: api
+          image: rootcause/api:local
+          imagePullPolicy: Never
+          args: [serve, --listen-all, --port, "7300"]
+          env:
+            - name: RC_RUNTIME
+              value: kubernetes
+            - name: RC_DBT_MODE
+              value: cronjob
+            # A new pod's network policy is in place a moment after the pod
+            # starts; until then Postgres refuses it. Wait up to a minute.
+            - name: RC_DB_WAIT_SECONDS
+              value: "60"
+            - name: RC_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: rootcause-db
+                  key: database-url
+          ports:
+            - containerPort: 7300
+          readinessProbe:
+            tcpSocket:
+              port: 7300
+            periodSeconds: 5
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi

deploy/k8s/api/Dockerfile

@@ -0,0 +1,12 @@
+# The API and dashboard image for the cluster (spec section 8.2). The context
+# is a directory scripts/k8s-up.sh assembles: the rootcause binary, built with
+# -tags webdist so the dashboard is embedded, and the exercises folder, whose
+# scripts the engine copies into sandboxes. Nothing else: no shell tools are
+# needed, and the process runs as an unprivileged user.
+FROM debian:13-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
+COPY rootcause /opt/rootcause/rootcause
+COPY exercises /opt/rootcause/exercises
+USER 10001:10001
+ENV RC_ROOT=/opt/rootcause HOME=/tmp
+ENTRYPOINT ["/opt/rootcause/rootcause"]
+CMD ["serve", "--listen-all"]

deploy/k8s/dbt.yaml

@@ -0,0 +1,110 @@
+# dbt for the cluster (spec sections 8.2 and 9.3): a CronJob builds the marts
+# every 10 minutes. The API has no rights to start Jobs (spec section 13.2),
+# so the dashboard shows when the marts were last built instead.
+# scripts/k8s-dbt.sh starts a build at once.
+#
+# The init container writes the project embedded in the rootcause binary
+# (rootcause dbt-project), so the cluster builds exactly what the binary
+# ships, and dbt reads it read-only.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: dbt
+  namespace: rootcause-system
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 300
+      ttlSecondsAfterFinished: 3600
+      template:
+        metadata:
+          labels:
+            app: rootcause-dbt
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          enableServiceLinks: false
+          hostUsers: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+            seccompProfile:
+              type: RuntimeDefault
+          initContainers:
+            - name: project
+              image: rootcause/api:local
+              imagePullPolicy: Never
+              args: [dbt-project, /project]
+              resources:
+                limits:
+                  cpu: 200m
+                  memory: 64Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: [ALL]
+              volumeMounts:
+                - name: project
+                  mountPath: /project
+          containers:
+            - name: dbt
+              image: ghcr.io/dbt-labs/dbt-postgres:1.9.0@sha256:37624cce09877ff03fd5f6b01355fdf5a3462a1b651d40d8b8edc1cf056eedf0
+              args: [build]
+              env:
+                - name: DBT_HOST
+                  value: postgres
+                - name: DBT_PORT
+                  value: "5432"
+                - name: DBT_USER
+                  value: rootcause
+                - name: DBT_DBNAME
+                  value: rootcause
+                - name: DBT_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: rootcause-db
+                      key: password
+                - name: DBT_PROFILES_DIR
+                  value: /usr/app/dbt
+                - name: DBT_TARGET_PATH
+                  value: /tmp/target
+                - name: DBT_LOG_PATH
+                  value: /tmp/logs
+                - name: HOME
+                  value: /tmp
+                - name: DBT_SEND_ANONYMOUS_USAGE_STATS
+                  value: "false"
+                - name: DBT_VERSION_CHECK
+                  value: "false"
+              resources:
+                requests:
+                  cpu: 200m
+                  memory: 256Mi
+                limits:
+                  cpu: "1"
+                  memory: 512Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: [ALL]
+              volumeMounts:
+                - name: project
+                  mountPath: /usr/app/dbt
+                  readOnly: true
+                - name: tmp
+                  mountPath: /tmp
+          volumes:
+            - name: project
+              emptyDir:
+                sizeLimit: 16Mi
+            - name: tmp
+              emptyDir:
+                sizeLimit: 256Mi

deploy/k8s/k3d.yaml

@@ -0,0 +1,36 @@
+# The local v2 cluster (spec section 8.2): 1 server and 2 agents. Created by
+# scripts/k8s-up.sh with `k3d cluster create --config`.
+#
+# The dashboard is published on the host's 127.0.0.1:7300 only, through the
+# NodePort of the rootcause-api Service. traefik, metrics-server and servicelb
+# are disabled: nothing uses them. local-path storage stays for Postgres.
+apiVersion: k3d.io/v1alpha5
+kind: Simple
+metadata:
+  name: rootcause
+servers: 1
+agents: 2
+image: rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d
+ports:
+  - port: 127.0.0.1:7300:30730
+    nodeFilters:
+      - server:0:direct
+options:
+  k3d:
+    wait: true
+    timeout: "180s"
+    disableLoadbalancer: true
+  k3s:
+    extraArgs:
+      - arg: --disable=traefik,metrics-server,servicelb
+        nodeFilters:
+          - server:*
+      # Kubernetes has no per-pod process limit; the kubelet applies this one
+      # to every pod (spec section 13.1's limit of 256).
+      - arg: --kubelet-arg=pod-max-pids=256
+        nodeFilters:
+          - server:*
+          - agent:*
+  kubeconfig:
+    updateDefaultKubeconfig: true
+    switchCurrentContext: true

deploy/k8s/kustomization.yaml

@@ -0,0 +1,14 @@
+# Everything in the cluster except the rootcause-db Secret and the
+# api-to-apiserver NetworkPolicy, which scripts/k8s-up.sh creates (the first
+# holds the generated password, the second the API server's address).
+# k3d.yaml is the cluster itself, not a resource.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespaces.yaml
+  - rbac.yaml
+  - sandboxes.yaml
+  - system-network.yaml
+  - postgres.yaml
+  - api.yaml
+  - dbt.yaml