diff --git a/README.md b/README.md index fbead57..37f5797 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ The server listens on 127.0.0.1 only. WSL2 forwards `localhost` from Windows to Exercises that boot systemd (such as svc-02) need a host with cgroup v2. WSL2 uses cgroup v2 by default. Check with `docker info --format '{{.CgroupVersion}}'`, which must print `2`. +### Kubernetes (v2) + +The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser. + +1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo. +2. Run `scripts/k8s-up.sh`. It creates the cluster (1 server, 2 agents), builds and imports the images, generates the database password into git-ignored `deploy/k8s/.env`, and deploys everything. Re-run it after a change. +3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page. +4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard (it counts in the cluster's statistics). +5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. + +`RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3). + ## Tests Unit tests need nothing: `go test ./...`. Integration tests need a database whose name ends in `_test`, and Docker with userns-remap: @@ -43,6 +55,7 @@ go test -count=1 ./... - Tests that run dbt create and drop their own database (for example `rootcause_selector_test`), because a dbt build reads every row in its database. - `RC_DBT_STRESS=1 go test -run TestBuildStress ./internal/dbt/` kills dbt builds at random points and checks that the next build succeeds. `RC_DBT_STRESS_ROUNDS` sets the rounds (default 10). - A failed dbt build saves its full `dbt.log` under `~/.cache/rootcause/dbt-logs/` and prints the path. The newest 20 are kept. +- `TestClusterSession` plays a session against the deployed cluster and runs only with `RC_CLUSTER_URL` set; `scripts/k8s-prove.sh` sets it. ## CI diff --git a/cmd/rootcause/main.go b/cmd/rootcause/main.go index 1757958..7a68215 100644 --- a/cmd/rootcause/main.go +++ b/cmd/rootcause/main.go @@ -2,7 +2,7 @@ // // The play commands are init, next, start, shell, ticket, hint, check, reset, // giveup, status, list, stats, serve (the dashboard) and lang. The developer -// commands validate, vars and selftest need no database. +// commands validate, vars, selftest and dbt-project need no database. package main import ( @@ -12,8 +12,8 @@ import ( "strings" "github.com/DIZAMU/RootCause/internal/cli" + "github.com/DIZAMU/RootCause/internal/dbt" "github.com/DIZAMU/RootCause/internal/exercise" - "github.com/DIZAMU/RootCause/internal/runtime/docker" "github.com/DIZAMU/RootCause/internal/session" ) @@ -33,6 +33,8 @@ func main() { os.Exit(cmdVars(args)) case "selftest": os.Exit(cmdSelfTest(ctx, args)) + case "dbt-project": + os.Exit(cmdDBTProject(args)) case "init": os.Exit(cmdInit(ctx)) default: @@ -115,20 +117,35 @@ func cmdValidate(args []string) int { } // cmdSelfTest runs an exercise's unsolved and solved scenarios in real -// sandboxes, with the same flags a session uses. It needs Docker, not the -// database. scripts/validate-exercise.sh calls it. +// sandboxes, with the same flags a session uses, on the runtime RC_RUNTIME +// selects. It needs that runtime, not the database. +// scripts/validate-exercise.sh calls it. func cmdSelfTest(ctx context.Context, args []string) int { seed, dir, ok := seedAndDir("selftest", args) if !ok { return 2 } - rt, err := docker.New() + be, err := cli.NewBackend(ctx) if err != nil { - fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, cli.English(err)) + return 1 + } + if err := session.SelfTest(ctx, be.Runtime, be.Builder, dir, seed, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "FAIL:", cli.English(err)) return 1 } - if err := session.SelfTest(ctx, rt, rt, dir, seed, os.Stdout); err != nil { - fmt.Fprintln(os.Stderr, "FAIL:", err) + return 0 +} + +// cmdDBTProject writes the embedded dbt project into an existing directory. +// The cluster's dbt CronJob runs it as an init container (spec section 8.2). +func cmdDBTProject(args []string) int { + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "dbt-project: expected exactly one directory") + return 2 + } + if err := dbt.WriteProject(args[0]); err != nil { + fmt.Fprintln(os.Stderr, "dbt-project:", err) return 1 } return 0 diff --git a/dbt/models/marts/mart_build_info.sql b/dbt/models/marts/mart_build_info.sql new file mode 100644 index 0000000..11ecb88 --- /dev/null +++ b/dbt/models/marts/mart_build_info.sql @@ -0,0 +1,9 @@ +-- One row: when the latest dbt build made the marts. The dashboard shows it, +-- so statistics that lag behind (the v2 CronJob builds every 10 minutes) are +-- visibly older than the latest session. +-- depends_on: {{ ref('mart_topic_levels') }} +-- depends_on: {{ ref('mart_review_queue') }} +-- depends_on: {{ ref('mart_exercise_stats') }} +-- depends_on: {{ ref('mart_progress') }} + +select now() as built_at diff --git a/dbt/models/marts/schema.yml b/dbt/models/marts/schema.yml index 501f172..3861327 100644 --- a/dbt/models/marts/schema.yml +++ b/dbt/models/marts/schema.yml @@ -49,3 +49,8 @@ models: - accepted_values: values: [1, 2, 3] quote: false + + - name: mart_build_info + columns: + - name: built_at + tests: [not_null] diff --git a/deploy/k8s/api.yaml b/deploy/k8s/api.yaml new file mode 100644 index 0000000..ed10beb --- /dev/null +++ b/deploy/k8s/api.yaml @@ -0,0 +1,92 @@ +# The API and dashboard (spec section 8.2): rootcause serve with the +# kubernetes runtime. It runs as the rootcause-api service account, which may +# only manage pods in sandboxes (rbac.yaml). The dashboard is reached through +# the NodePort, which k3d publishes on the host's 127.0.0.1:7300 only +# (k3d.yaml); the server's Host and token checks apply as on the laptop. +apiVersion: v1 +kind: Service +metadata: + name: rootcause-api + namespace: rootcause-system +spec: + type: NodePort + selector: + app: rootcause-api + ports: + - port: 7300 + targetPort: 7300 + nodePort: 30730 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rootcause-api + namespace: rootcause-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: rootcause-api + template: + metadata: + labels: + app: rootcause-api + spec: + serviceAccountName: rootcause-api + # The kubernetes runtime needs the service account's token; the + # ServiceAccount itself does not mount it anywhere else. + automountServiceAccountToken: true + enableServiceLinks: false + hostUsers: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: rootcause/api:local + imagePullPolicy: Never + args: [serve, --listen-all, --port, "7300"] + env: + - name: RC_RUNTIME + value: kubernetes + - name: RC_DBT_MODE + value: cronjob + # A new pod's network policy is in place a moment after the pod + # starts; until then Postgres refuses it. Wait up to a minute. + - name: RC_DB_WAIT_SECONDS + value: "60" + - name: RC_DATABASE_URL + valueFrom: + secretKeyRef: + name: rootcause-db + key: database-url + ports: + - containerPort: 7300 + readinessProbe: + tcpSocket: + port: 7300 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/deploy/k8s/api/Dockerfile b/deploy/k8s/api/Dockerfile new file mode 100644 index 0000000..de21017 --- /dev/null +++ b/deploy/k8s/api/Dockerfile @@ -0,0 +1,12 @@ +# The API and dashboard image for the cluster (spec section 8.2). The context +# is a directory scripts/k8s-up.sh assembles: the rootcause binary, built with +# -tags webdist so the dashboard is embedded, and the exercises folder, whose +# scripts the engine copies into sandboxes. Nothing else: no shell tools are +# needed, and the process runs as an unprivileged user. +FROM debian:13-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad +COPY rootcause /opt/rootcause/rootcause +COPY exercises /opt/rootcause/exercises +USER 10001:10001 +ENV RC_ROOT=/opt/rootcause HOME=/tmp +ENTRYPOINT ["/opt/rootcause/rootcause"] +CMD ["serve", "--listen-all"] diff --git a/deploy/k8s/dbt.yaml b/deploy/k8s/dbt.yaml new file mode 100644 index 0000000..88a8684 --- /dev/null +++ b/deploy/k8s/dbt.yaml @@ -0,0 +1,110 @@ +# dbt for the cluster (spec sections 8.2 and 9.3): a CronJob builds the marts +# every 10 minutes. The API has no rights to start Jobs (spec section 13.2), +# so the dashboard shows when the marts were last built instead. +# scripts/k8s-dbt.sh starts a build at once. +# +# The init container writes the project embedded in the rootcause binary +# (rootcause dbt-project), so the cluster builds exactly what the binary +# ships, and dbt reads it read-only. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dbt + namespace: rootcause-system +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app: rootcause-dbt + spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + hostUsers: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: project + image: rootcause/api:local + imagePullPolicy: Never + args: [dbt-project, /project] + resources: + limits: + cpu: 200m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: project + mountPath: /project + containers: + - name: dbt + image: ghcr.io/dbt-labs/dbt-postgres:1.9.0@sha256:37624cce09877ff03fd5f6b01355fdf5a3462a1b651d40d8b8edc1cf056eedf0 + args: [build] + env: + - name: DBT_HOST + value: postgres + - name: DBT_PORT + value: "5432" + - name: DBT_USER + value: rootcause + - name: DBT_DBNAME + value: rootcause + - name: DBT_PASSWORD + valueFrom: + secretKeyRef: + name: rootcause-db + key: password + - name: DBT_PROFILES_DIR + value: /usr/app/dbt + - name: DBT_TARGET_PATH + value: /tmp/target + - name: DBT_LOG_PATH + value: /tmp/logs + - name: HOME + value: /tmp + - name: DBT_SEND_ANONYMOUS_USAGE_STATS + value: "false" + - name: DBT_VERSION_CHECK + value: "false" + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: project + mountPath: /usr/app/dbt + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: project + emptyDir: + sizeLimit: 16Mi + - name: tmp + emptyDir: + sizeLimit: 256Mi diff --git a/deploy/k8s/k3d.yaml b/deploy/k8s/k3d.yaml new file mode 100644 index 0000000..c1deb7a --- /dev/null +++ b/deploy/k8s/k3d.yaml @@ -0,0 +1,36 @@ +# The local v2 cluster (spec section 8.2): 1 server and 2 agents. Created by +# scripts/k8s-up.sh with `k3d cluster create --config`. +# +# The dashboard is published on the host's 127.0.0.1:7300 only, through the +# NodePort of the rootcause-api Service. traefik, metrics-server and servicelb +# are disabled: nothing uses them. local-path storage stays for Postgres. +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: rootcause +servers: 1 +agents: 2 +image: rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d +ports: + - port: 127.0.0.1:7300:30730 + nodeFilters: + - server:0:direct +options: + k3d: + wait: true + timeout: "180s" + disableLoadbalancer: true + k3s: + extraArgs: + - arg: --disable=traefik,metrics-server,servicelb + nodeFilters: + - server:* + # Kubernetes has no per-pod process limit; the kubelet applies this one + # to every pod (spec section 13.1's limit of 256). + - arg: --kubelet-arg=pod-max-pids=256 + nodeFilters: + - server:* + - agent:* + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..435b317 --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,14 @@ +# Everything in the cluster except the rootcause-db Secret and the +# api-to-apiserver NetworkPolicy, which scripts/k8s-up.sh creates (the first +# holds the generated password, the second the API server's address). +# k3d.yaml is the cluster itself, not a resource. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespaces.yaml + - rbac.yaml + - sandboxes.yaml + - system-network.yaml + - postgres.yaml + - api.yaml + - dbt.yaml diff --git a/deploy/k8s/namespaces.yaml b/deploy/k8s/namespaces.yaml new file mode 100644 index 0000000..322335d --- /dev/null +++ b/deploy/k8s/namespaces.yaml @@ -0,0 +1,19 @@ +# The two namespaces of spec section 8.2. Pod Security Admission enforces the +# restricted profile on the system workloads and baseline on sandboxes: the +# learner is root in a sandbox pod (in its own user namespace), which the +# restricted profile forbids. +apiVersion: v1 +kind: Namespace +metadata: + name: rootcause-system + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sandboxes + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest diff --git a/deploy/k8s/postgres.yaml b/deploy/k8s/postgres.yaml new file mode 100644 index 0000000..4626666 --- /dev/null +++ b/deploy/k8s/postgres.yaml @@ -0,0 +1,94 @@ +# Postgres for the cluster (spec section 8.2), on a local-path volume. The +# password comes from the rootcause-db Secret, which scripts/k8s-up.sh +# creates outside git (spec section 13.2). +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: rootcause-system +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: rootcause-system +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + hostUsers: false + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: postgres + image: postgres:17@sha256:a63c1ea3f6a3a5a1621f97d2aba3b30faecfab4224c1b0000cc85ced57bb1939 + env: + - name: POSTGRES_USER + value: rootcause + - name: POSTGRES_DB + value: rootcause + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: rootcause-db + key: password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: [pg_isready, -U, rootcause, -d, rootcause] + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: run + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + volumes: + - name: run + emptyDir: {} + - name: tmp + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/deploy/k8s/rbac.yaml b/deploy/k8s/rbac.yaml new file mode 100644 index 0000000..6445b24 --- /dev/null +++ b/deploy/k8s/rbac.yaml @@ -0,0 +1,40 @@ +# The API's identity (spec section 13.2): it may create, read, exec into and +# delete pods in sandboxes. Nothing else, and nowhere else: a Role, not a +# ClusterRole. scripts/k8s-prove.sh checks this with kubectl auth can-i. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rootcause-api + namespace: rootcause-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rootcause-api + namespace: sandboxes +rules: + - apiGroups: [""] + resources: [pods] + verbs: [create, get, list, delete] + # Exec over WebSocket is a GET that also needs create, so exec takes both. + - apiGroups: [""] + resources: [pods/exec] + verbs: [create, get] + - apiGroups: [""] + resources: [pods/log] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rootcause-api + namespace: sandboxes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rootcause-api +subjects: + - kind: ServiceAccount + name: rootcause-api + namespace: rootcause-system diff --git a/deploy/k8s/sandboxes.yaml b/deploy/k8s/sandboxes.yaml new file mode 100644 index 0000000..ba33c2f --- /dev/null +++ b/deploy/k8s/sandboxes.yaml @@ -0,0 +1,90 @@ +# Limits and network isolation for the sandboxes namespace (spec section +# 13.2). The runtime in internal/runtime/kubernetes labels every pod with +# rootcause.network ("none" or "internal"); the engine labels check runners +# rootcause.runner=true. +# +# Nothing in or out by default: no internet, no cluster DNS, no +# rootcause-system, no other sandbox. The one exception is a check runner, +# which may connect to sandboxes on the internal network (spec section 13.3 +# names the limit: with two sessions, a runner could reach the other one). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: sandboxes +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: runner-egress + namespace: sandboxes +spec: + podSelector: + matchLabels: + rootcause.runner: "true" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + rootcause.network: internal + matchExpressions: + - {key: rootcause.runner, operator: DoesNotExist} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: sandbox-from-runner + namespace: sandboxes +spec: + podSelector: + matchLabels: + rootcause.network: internal + matchExpressions: + - {key: rootcause.runner, operator: DoesNotExist} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + rootcause.runner: "true" +--- +# At most two sessions and a check runner at a time (spec section 8.2's +# memory budget on an 8 GB WSL2 host). +apiVersion: v1 +kind: ResourceQuota +metadata: + name: sandboxes + namespace: sandboxes +spec: + hard: + pods: "4" + requests.cpu: "2" + limits.cpu: "2" + requests.memory: 1Gi + limits.memory: 1Gi + services: "0" + configmaps: "1" + secrets: "0" + persistentvolumeclaims: "0" +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: sandboxes + namespace: sandboxes +spec: + limits: + - type: Container + default: + cpu: 500m + memory: 256Mi + defaultRequest: + cpu: 500m + memory: 256Mi + max: + cpu: "1" + memory: 512Mi diff --git a/deploy/k8s/system-network.yaml b/deploy/k8s/system-network.yaml new file mode 100644 index 0000000..bad1368 --- /dev/null +++ b/deploy/k8s/system-network.yaml @@ -0,0 +1,82 @@ +# Network isolation for rootcause-system. Nothing in or out by default; then +# exactly what each workload needs: +# - the API accepts the dashboard on 7300, and nothing else; +# - the API reaches Postgres and cluster DNS (its connection to the +# Kubernetes API server is the api-to-apiserver policy, which +# scripts/k8s-up.sh writes with the server's address); +# - dbt reaches Postgres and cluster DNS; +# - Postgres accepts the API and dbt on 5432. +# Sandboxes get no rule here, so nothing in sandboxes reaches this namespace. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: rootcause-system +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: api-ingress + namespace: rootcause-system +spec: + podSelector: + matchLabels: + app: rootcause-api + policyTypes: [Ingress] + ingress: + - ports: + - port: 7300 + protocol: TCP +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: to-postgres-and-dns + namespace: rootcause-system +spec: + podSelector: + matchExpressions: + - {key: app, operator: In, values: [rootcause-api, rootcause-dbt]} + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app: postgres + ports: + - port: 5432 + protocol: TCP + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgres-ingress + namespace: rootcause-system +spec: + podSelector: + matchLabels: + app: postgres + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchExpressions: + - {key: app, operator: In, values: [rootcause-api, rootcause-dbt]} + ports: + - port: 5432 + protocol: TCP diff --git a/docs/SPEC.md b/docs/SPEC.md index 7460fc3..5fb447c 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -246,14 +246,15 @@ WSL2 ├── namespace rootcause-system │ ├── api + dashboard (Deployment) │ ├── postgres (StatefulSet) - │ ├── dbt (on-demand Job + CronJob) - │ └── argocd + │ ├── dbt (CronJob every 10 minutes; a Job on demand by script) + │ └── argocd (Phase 8) └── namespace sandboxes - └── one pod per session: exercise image + ttyd web terminal + └── one pod per session: the exercise image, nothing added ``` -- Reached from Windows through k3d port mapping to localhost. Tailscale only if remote access is needed. -- The browser terminal is ttyd inside the session pod, proxied through the API. Sandboxes are never exposed directly. +- Reached from Windows through k3d port mapping to `127.0.0.1:7300` on the host only. Tailscale only if remote access is needed. +- The browser terminal is xterm.js in the dashboard, on a WebSocket to the API. The API runs `bash` in the session pod through the Kubernetes exec API (the v1 `docker exec` model), so nothing in a sandbox listens on the network and sandboxes stay closed to ingress. Phase 7 chose this over ttyd in the pod, which would have needed a listening port and a NetworkPolicy opening every sandbox to `rootcause-system`, plus an extra binary in every exercise image. +- `scripts/k8s-up.sh` builds the images, imports them into the cluster's nodes (`k3d image import`), creates the database Secret and applies `deploy/k8s`. `scripts/k8s-prove.sh` proves the Phase 7 criteria. The CLI uses the cluster with `RC_RUNTIME=kubernetes`; `rootcause selftest` then validates exercises in sandbox pods. ### 8.3 Runtime abstraction @@ -270,7 +271,9 @@ type Runtime interface { } ``` -Implementations: `internal/runtime/docker` (v1) and `internal/runtime/kubernetes` (v2, client-go). +Implementations: `internal/runtime/docker` (v1) and `internal/runtime/kubernetes` (v2, client-go). `RC_RUNTIME` (`docker`, the default, or `kubernetes`) selects one. Optional capabilities sit beside the interface: `Sheller` (v1 `rootcause shell`), `Terminal` (v2 web terminal), `Diagnoser` and `SystemdChecker`. + +The Kubernetes runtime makes each sandbox a pod in `sandboxes`. `Create` returns once the container runs, as `docker run -d` does. `Exec` and `CopyIn` go through the exec API (`CopyIn` streams the file to `cat`). `Restart` is not supported (a restarted container starts again from its image); the only exercise that restarts before its check boots systemd, which the runtime refuses first. Pods on the internal network get a hosts entry for every other internal pod, so a check runner reaches its sandbox by name as on docker's internal network. ### 8.4 Session lifecycle @@ -306,6 +309,7 @@ Migrations are versioned SQL files in `internal/store/migrations`, applied on st | marts | mart_review_queue | Review candidates per learner (rule 1 of section 7.2), with the time each is due | | marts | mart_exercise_stats | Solve rate, hint use, duration, calibration flag | | marts | mart_progress | Per learner, topic and day: sessions, solves, average stars, level at the end of the day | +| marts | mart_build_info | One row: when the latest build ran. The dashboard shows it, so statistics that lag behind are visibly older | dbt reads the source tables in schema `public` and writes every model to schema `analytics`, so a build never touches the engine's tables. Outcomes in `int_session_outcomes` are `solved`, `gave_up`, `expired_fail` (expired after a check), `expired_skip` (expired with no check) and `voided`. Only the first three count in the statistics and the review queue. The engine stays the authority for levels (section 7.1). `mart_topic_levels` replays its `level_changed` events, and a test asserts that the two agree. @@ -318,7 +322,7 @@ Every dbt container is labelled with its database. Before a build, containers le ### 9.3 When dbt runs - v1: after each solve and give up (after the result is shown, with a 120-second limit), on `rootcause stats --refresh`, and in `rootcause next` when the marts are stale. Expiry by the reaper does not run dbt; the next `rootcause next` catches up. In the dashboard, a solve or give up returns at once and dbt runs in the background, as does a rebuild when the home view finds the marts stale; starting the recommended exercise from the dashboard rebuilds stale marts first, like `rootcause next`. A Postgres advisory lock lets only one build run at a time, across the CLI and the dashboard. -- v2: an on-demand Job after each session, plus a CronJob every 15 minutes as backstop. +- v2: a CronJob every 10 minutes (`concurrencyPolicy: Forbid`). The API's service account may not start Jobs (section 13.2), so the dashboard shows when the marts were last built instead. `scripts/k8s-dbt.sh` runs a build at once. An init container writes the project embedded in the binary (`rootcause dbt-project`), so the cluster builds what the binary ships. The API runs with `RC_DBT_MODE=cronjob` and never starts dbt itself. - The marts are stale when they are missing, or when the learner's latest finished session is newer than the latest one in `int_session_outcomes`. `rootcause next` then rebuilds them once. If they are still stale, or dbt fails, the selector falls back to topic levels alone: rules 2 to 4 of section 7.2 from the engine's levels and the source tables, without the review queue. A failed build never changes a session's result. ## 10. CLI (v1) @@ -337,13 +341,14 @@ Every dbt container is labelled with its database. Before a build, containers le | `rootcause status` | Topic levels, active session, review queue | | `rootcause list` | All exercises with best stars | | `rootcause stats [--refresh]` | Statistics per exercise from the marts, with calibration flags. `--refresh` rebuilds the marts first | -| `rootcause serve [--port N]` | Starts the local dashboard on 127.0.0.1, port 7300 by default, until Ctrl-C. Reaps expired sessions every minute while it runs | +| `rootcause serve [--port N] [--listen-all]` | Starts the local dashboard on 127.0.0.1, port 7300 by default, until Ctrl-C. Reaps expired sessions every minute while it runs. `--listen-all` is for the cluster deployment only and is refused unless `RC_RUNTIME=kubernetes` | | `rootcause lang en\|no` | Sets the interface language | ## 11. Dashboard (v1) - Home: recommended next exercise, topic levels, review queue. -- Exercise view: ticket, hint buttons, check, reset, give up, result and walkthrough. Shows the `rootcause shell` command (v2 embeds the terminal). +- Exercise view: ticket, hint buttons, check, reset, give up, result and walkthrough. Shows the `rootcause shell` command, or, when the runtime has a web terminal (v2), the terminal itself. The terminal closes when its session ends. +- Home shows when the statistics were last built. - History: past sessions with stars and outcome. - Exercise list with best stars per exercise. - EN/NO toggle. It sets the profile language, the same one `rootcause lang` sets. @@ -353,7 +358,7 @@ Implementation: - The app (Vite, React, TypeScript, plain CSS, hash routing) lives in `web/`. `scripts/build-web.sh` builds it into `web/dist` with `npm ci` from the committed lockfile. `go build -tags webdist` embeds it. Without the tag, `/` serves a translated page that says how to build it. - `internal/server` is a JSON API over the same engine, selector and store as the CLI. Both front ends call the shared steps in `internal/play` (next pick and seed, exercise texts, statistics rebuild). Exercise texts are filled server-side in the learner's language. UI strings come from `locales/*.json` through `/api/i18n/{lang}`, so there is one copy of every string. -- Endpoints: `GET /api/overview`, `/api/exercises`, `/api/history`, `/api/session` (the active session, 204 if none), `/api/sessions/{id}`, `/api/i18n/{lang}`; `POST /api/start` (`{"exercise": ""}` starts the recommended one), `/api/hint`, `/api/check`, `/api/reset`, `/api/giveup`; `PUT /api/lang`. Actions name the session the page shows and are refused if it is no longer the active one. +- Endpoints: `GET /api/overview`, `/api/exercises`, `/api/history`, `/api/session` (the active session, 204 if none), `/api/sessions/{id}`, `/api/i18n/{lang}`; `POST /api/start` (`{"exercise": ""}` starts the recommended one), `/api/hint`, `/api/check`, `/api/reset`, `/api/giveup`; `PUT /api/lang`; `GET /api/terminal` (WebSocket, v2). Actions name the session the page shows and are refused if it is no longer the active one. - The dashboard follows the CLI by polling: the open session every 2 seconds, the overview every few seconds and on focus. The two front ends share only the database. - The session view never contains the seed, variant values or derived values. Hints appear once revealed, and the walkthrough once the session is solved or given up. @@ -376,13 +381,17 @@ Implementation: - The local server binds to 127.0.0.1 only. - It refuses any request whose Host header is not `127.0.0.1:PORT` or `localhost:PORT`, which blocks DNS rebinding. - Every `/api` request must carry the header `X-RootCause-Token`, a random token made per server process and written only into the served `index.html`. Another website cannot read that page, so it cannot call the API. The server sends no CORS headers, and every response forbids framing. +- The web terminal's WebSocket (v2) cannot carry headers, so the browser offers the token as a subprotocol (`rootcause.token.`) and the server answers with `rootcause.terminal` only. The request's `Origin` must be the dashboard itself, and the Host check applies as above. +- The Content-Security-Policy allows scripts from the dashboard only. Styles may be inline, because xterm.js writes its layout into `