From 395f7d625995ef88797a8f13f2828a75259266d5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 11 Sep 2026 00:03:37 +0900 Subject: [PATCH 1/2] =?UTF-8?q?jepsen:=20run=20the=20existing=20suites=20a?= =?UTF-8?q?gainst=20an=20encrypted=20cluster=20(=C2=A78.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encryption design's Jepsen gate is explicit that NO new workload is required: encryption is consistency-transparent, so the acceptance gate is the existing Redis and DynamoDB workloads passing against an encrypted cluster. What was missing was the ability to stand that cluster up. --encryption provisions the §5.1 KEK file on each node with owner-only permissions and starts the server with --encryption-enabled, --kekFile and --encryptionSidecarPath. The three travel together because a sidecar path alone only enables read-only capability probing, while the mutating RPCs a bootstrap needs require the flag and a KEK source too. The KEK is a fixed test value rather than a generated one: every node must unwrap the same sidecar, and a per-node random KEK would fail startup with ErrKEKMismatch — a far more confusing failure than a hardcoded test key. The flag lives in common-cli-opts, so a future workload inherits the gate without opting in, and defaults off so existing runs measure the same thing they did before. server-args is extracted from start-node! as a pure function purely so the flag set is testable without SSH. That matters more than it looks: a --encryption run that silently produced an UNENCRYPTED cluster would report PASS and be recorded as evidence for this gate, which is worse than having no gate. The first version of these tests did not pin it — dropping the flags from db.clj left them all green. Full suite: 157 tests, 0 failures. Claude-Session: https://claude.ai/code/session_013rNHooj7NF3giihWVba8QE --- ...6_04_29_partial_data_at_rest_encryption.md | 9 ++ jepsen/src/elastickv/cli.clj | 5 ++ jepsen/src/elastickv/db.clj | 86 +++++++++++++++---- jepsen/src/elastickv/dynamodb_workload.clj | 3 +- jepsen/src/elastickv/redis_workload.clj | 3 +- .../test/elastickv/encrypted_cluster_test.clj | 86 +++++++++++++++++++ 6 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 jepsen/test/elastickv/encrypted_cluster_test.clj diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0588b2805..55afe3ce0 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -2445,6 +2445,15 @@ different output bytes; FSM apply still deterministic), so no new Jepsen workload is required. A pass under the existing suite is the acceptance gate. +**Implemented.** `lein test :only ...` aside, any workload accepts +`--encryption`, which provisions the §5.1 KEK file on each node and +starts the server with `--encryption-enabled`, `--kekFile` and +`--encryptionSidecarPath`. The switch defaults off so the existing +unencrypted runs are unchanged, and `server-args` is a pure function so +a test can assert the flags actually reach the server — a `--encryption` +run that silently produced an unencrypted cluster would report PASS and +be recorded as evidence for this gate. + --- ## 9. Operational concerns diff --git a/jepsen/src/elastickv/cli.clj b/jepsen/src/elastickv/cli.clj index a802e9ab0..95cc2883c 100644 --- a/jepsen/src/elastickv/cli.clj +++ b/jepsen/src/elastickv/cli.clj @@ -14,6 +14,11 @@ :default default-nodes-str] [nil "--local" "Run locally without SSH or nemesis." :default false] + ;; §8.4: the encrypted acceptance gate is the EXISTING suites run + ;; against an encrypted cluster, so this is a cluster-setup switch + ;; rather than a workload selector — every workload honours it. + [nil "--encryption" "Run against a cluster with data-at-rest encryption enabled." + :default false] [nil "--host HOST" "Host override for clients." :default nil] [nil "--grpc-port PORT" "gRPC/Raft port." diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 369d33bc9..e804bcc73 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -15,6 +15,22 @@ (def ^:private transport-metrics-file "/var/log/elastickv-transport-metrics.prom") (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) + +;; §8.4 acceptance gate: the encrypted-cluster run reuses the EXISTING +;; Redis and DynamoDB workloads rather than adding a new one — +;; encryption is consistency-transparent (same input bytes, different +;; output bytes, apply still deterministic), so what has to be built is +;; the ability to stand the cluster up encrypted, not a new checker. +(def ^:private kek-file (str data-dir "/kek.bin")) +(def ^:private sidecar-file (str data-dir "/keys.json")) + +;; kek-bytes is the §5.1 KEK: exactly 32 raw bytes, owner-only mode. +;; A fixed test value, not a generated one — every node in the cluster +;; must unwrap the same sidecar, and a per-node random KEK would make +;; the cluster refuse to start with ErrKEKMismatch, which is a far more +;; confusing failure than a hardcoded test key. +(def ^:private kek-test-bytes + (apply str (repeat 32 "k"))) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) (def ^:private build-dir @@ -62,6 +78,19 @@ (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) +(defn- provision-kek! + "Writes the §5.1 KEK file with owner-only permissions. + + Runs before start-node! because --encryption-enabled refuses to start + without a readable KEK source, and the refusal happens during startup + guards — well before anything the workload could observe." + [node] + (c/on node + (c/su + (c/exec :mkdir :-p data-dir) + (c/exec :bash :-c (str "printf '%s' '" kek-test-bytes "' > " kek-file)) + (c/exec :chmod "600" kek-file)))) + (defn- node-addr "Returns host:port for the node and port." [node port] @@ -103,8 +132,41 @@ (defn- build-raft-dynamo-map [nodes grpc-port dynamo-port raft-groups] (build-raft-service-map nodes grpc-port dynamo-port raft-groups)) +(defn server-args + "Builds the elastickv server argv for one node. + + Extracted from start-node! as a pure function so the flag set — and + in particular whether encryption is actually switched on — is + testable without SSH. A --encryption run that silently produced an + UNENCRYPTED cluster would report PASS and be recorded as evidence for + the §8.4 acceptance gate, which is worse than having no gate." + [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine + raft-redis-map raft-dynamo-map raft-groups shard-ranges + encryption bootstrap?]}] + (cond-> ["--address" grpc + "--redisAddress" redis + "--raftId" (name node) + "--raftDataDir" data-dir + "--raftEngine" (or raft-engine "etcd") + "--raftRedisMap" raft-redis-map] + dynamo (conj "--dynamoAddress" dynamo + "--raftDynamoMap" raft-dynamo-map) + s3 (conj "--s3Address" s3) + sqs (conj "--sqsAddress" sqs) + (and sqs sqs-region) (conj "--sqsRegion" sqs-region) + (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) + (seq shard-ranges) (conj "--shardRanges" shard-ranges) + ;; Sidecar path alone only enables read-only capability probing; the + ;; mutating RPCs the bootstrap needs also require + ;; --encryption-enabled AND a KEK source, so the three travel + ;; together or not at all. + encryption (conj "--encryptionSidecarPath" sidecar-file + "--encryption-enabled" + "--kekFile" kek-file) + bootstrap? (conj "--raftBootstrap"))) + (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env encryption]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -123,20 +185,12 @@ raft-dynamo-map (when dynamo (build-raft-dynamo-map (:nodes test) grpc-port dynamo-port raft-groups)) bootstrap? (= node bootstrap-node) - args (cond-> ["--address" grpc - "--redisAddress" redis - "--raftId" (name node) - "--raftDataDir" data-dir - "--raftEngine" (or raft-engine "etcd") - "--raftRedisMap" raft-redis-map] - dynamo (conj "--dynamoAddress" dynamo - "--raftDynamoMap" raft-dynamo-map) - s3 (conj "--s3Address" s3) - sqs (conj "--sqsAddress" sqs) - (and sqs sqs-region) (conj "--sqsRegion" sqs-region) - (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) - (seq shard-ranges) (conj "--shardRanges" shard-ranges) - bootstrap? (conj "--raftBootstrap")) + args (server-args + {:node node :grpc grpc :redis redis :dynamo dynamo :s3 s3 :sqs sqs + :sqs-region sqs-region :data-dir data-dir :raft-engine raft-engine + :raft-redis-map raft-redis-map :raft-dynamo-map raft-dynamo-map + :raft-groups raft-groups :shard-ranges shard-ranges + :encryption encryption :bootstrap? bootstrap?}) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file @@ -194,6 +248,8 @@ (c/su (c/exec :mkdir :-p data-dir) (c/exec :rm :-f log-file transport-metrics-file))) + (when (:encryption opts) + (provision-kek! node)) (start-node! test node (merge {:data-dir data-dir :grpc-port (or (:grpc-port opts) 50051) :redis-port (or (:redis-port opts) 6379) diff --git a/jepsen/src/elastickv/dynamodb_workload.clj b/jepsen/src/elastickv/dynamodb_workload.clj index c5d586a43..87a0f7ad8 100644 --- a/jepsen/src/elastickv/dynamodb_workload.clj +++ b/jepsen/src/elastickv/dynamodb_workload.clj @@ -329,7 +329,8 @@ :redis-port (or (:redis-port opts) 6379) :dynamo-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/redis_workload.clj b/jepsen/src/elastickv/redis_workload.clj index 20283a64c..ac4f5d3f2 100644 --- a/jepsen/src/elastickv/redis_workload.clj +++ b/jepsen/src/elastickv/redis_workload.clj @@ -125,7 +125,8 @@ (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) :redis-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/test/elastickv/encrypted_cluster_test.clj b/jepsen/test/elastickv/encrypted_cluster_test.clj new file mode 100644 index 000000000..6e9d4585b --- /dev/null +++ b/jepsen/test/elastickv/encrypted_cluster_test.clj @@ -0,0 +1,86 @@ +(ns elastickv.encrypted-cluster-test + "Pins the §8.4 encrypted acceptance gate: the EXISTING Redis and + DynamoDB workloads must be runnable against a cluster with + data-at-rest encryption enabled. + + The design is explicit that no new workload is required — encryption + is consistency-transparent, so the gate is a cluster-setup switch. The + risk that switch carries is that it silently does nothing: a run that + reported PASS while the cluster was never actually encrypted would be + worse than no gate at all, because it would be recorded as evidence." + (:require [clojure.test :refer :all] + [elastickv.cli :as cli] + [elastickv.db :as ekdb] + [elastickv.dynamodb-workload :as dynamo] + [elastickv.redis-workload :as redis])) + +(deftest encryption-flag-is-available-to-every-workload + ;; It lives in common-cli-opts rather than per-workload, so a future + ;; workload gets the gate without opting in. + (let [names (set (map second cli/common-cli-opts))] + (is (contains? names "--encryption")))) + +(deftest encryption-defaults-off + ;; The unencrypted suites are the existing baseline; turning this on + ;; by default would silently change what every current run measures. + (let [spec (first (filter #(= "--encryption" (second %)) cli/common-cli-opts))] + (is (false? (:default (apply hash-map (drop 3 spec))))))) + +(deftest redis-workload-propagates-encryption-to-the-db + (let [test-map (redis/elastickv-redis-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest dynamodb-workload-propagates-encryption-to-the-db + (let [test-map (dynamo/elastickv-dynamodb-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest encryption-is-absent-from-the-db-when-not-requested + ;; The flag must not leak a truthy value into an ordinary run. + (let [test-map (redis/elastickv-redis-test {})] + (is (not (true? (get-in test-map [:db :opts :encryption])))))) + +(deftest db-accepts-the-encryption-option + ;; ekdb/db carries opts verbatim; this pins that the key survives + ;; construction rather than being dropped by a destructuring form. + (let [db (ekdb/db {:grpc-port 50051 :encryption true})] + (is (true? (get-in db [:opts :encryption]))))) + +;; --------------------------------------------------------------------------- +;; The load-bearing property: the switch must actually reach the server +;; --------------------------------------------------------------------------- + +(defn- args-for [over] + (ekdb/server-args (merge {:node "n1" :grpc "n1:50051" :redis "n1:6379" + :data-dir "/var/lib/elastickv" + :raft-redis-map "n1=n1:6379"} + over))) + +(deftest encryption-emits-all-three-server-flags + ;; A sidecar path alone only enables read-only capability probing. + ;; The mutating RPCs the bootstrap needs require --encryption-enabled + ;; AND a KEK source, so all three must travel together — two of the + ;; three would produce a cluster that refuses to start, or worse, one + ;; that starts unencrypted. + (let [args (set (args-for {:encryption true}))] + (is (contains? args "--encryption-enabled")) + (is (contains? args "--encryptionSidecarPath")) + (is (contains? args "--kekFile")))) + +(deftest without-encryption-no-encryption-flag-is-emitted + ;; The unencrypted suites must be byte-identical to before, or the + ;; baseline every existing run measures has silently changed. + (let [args (set (args-for {}))] + (is (not (contains? args "--encryption-enabled"))) + (is (not (contains? args "--encryptionSidecarPath"))) + (is (not (contains? args "--kekFile"))))) + +(deftest encryption-does-not-disturb-the-other-flags + (let [plain (remove #{"--encryptionSidecarPath" "--encryption-enabled" "--kekFile"} + (args-for {:encryption true})) + encrypted (args-for {})] + ;; Removing the encryption flags and their values must leave the + ;; same argv an unencrypted run would produce. + (is (= (set encrypted) + (set (remove #(or (= % "/var/lib/elastickv/keys.json") + (= % "/var/lib/elastickv/kek.bin")) + plain)))))) From 2d280f241794b4380fd3e66ead4971095c700a9d Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 12 Sep 2026 14:24:38 +0900 Subject: [PATCH 2/2] jepsen: actually activate encryption before running the workloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both of which made the §8.4 gate report success on a cleartext cluster — the exact failure this suite's own docstring calls "worse than no gate at all, because it would be recorded as evidence". P1 — setup! provisioned a KEK and started the nodes, and nothing else. --encryption-enabled only opens the EncryptionAdmin mutator RPCs; buildEncryptionWriteWiring deliberately keeps the store's envelope gate closed until BOTH BootstrapEncryption and EnableStorageEnvelope have applied (the gate reads cache.StorageEnvelopeActive). Every "encrypted" Redis and DynamoDB run therefore stored cleartext and could still PASS. setup! now bootstraps the DEKs and performs the §7.1 Phase-1 storage cutover after membership is established — after, because the writer batch needs a registry entry per member (§5.6 step 1a) and the cutover's capability gate requires every voter to report capable. It then polls every node until it reports storage_envelope_active and FAILS SETUP if any node does not, so the workload can never start against a cluster that is still writing cleartext. Bootstrap takes the KEK-wrapped DEK bytes as an argument, which an operator gets from their KMS and the harness has no way to produce. The new cmd/jepsen-encryption-setup generates a DEK, wraps it under the harness KEK file, and prints only the wrapped form — the plaintext never leaves the process. It is harness tooling on purpose: putting DEK generation into elastickv-admin would add operator-facing handling of raw key material, which needs its own design and review. The orchestration itself uses the shipped elastickv-admin subcommands, so the run exercises the real operator path. P2 — --encryption lives in common-cli-opts so that, as this suite already claimed, "a future workload gets the gate without opting in". Five entrypoints (s3, sqs-htfifo, dynamodb-types, dynamodb-multi-table, redis-zset-safety) accepted the flag and then dropped it when building ekdb/db, so the CLI advertised a guarantee it did not provide. All five now thread it, and one table-driven test drives every workload constructor so the next one fails here instead of shipping a silent lie. Claude-Session: https://claude.ai/code/session_013rNHooj7NF3giihWVba8QE --- cmd/jepsen-encryption-setup/main.go | 72 ++++++++++ cmd/jepsen-encryption-setup/main_test.go | 75 +++++++++++ jepsen/src/elastickv/db.clj | 126 +++++++++++++++++- .../dynamodb_multi_table_workload.clj | 3 +- .../src/elastickv/dynamodb_types_workload.clj | 3 +- .../elastickv/redis_zset_safety_workload.clj | 3 +- jepsen/src/elastickv/s3_workload.clj | 3 +- jepsen/src/elastickv/sqs_htfifo_workload.clj | 3 +- .../test/elastickv/encrypted_cluster_test.clj | 118 +++++++++++++++- 9 files changed, 397 insertions(+), 9 deletions(-) create mode 100644 cmd/jepsen-encryption-setup/main.go create mode 100644 cmd/jepsen-encryption-setup/main_test.go diff --git a/cmd/jepsen-encryption-setup/main.go b/cmd/jepsen-encryption-setup/main.go new file mode 100644 index 000000000..b279b0c41 --- /dev/null +++ b/cmd/jepsen-encryption-setup/main.go @@ -0,0 +1,72 @@ +// Command jepsen-encryption-setup produces a KEK-wrapped DEK for the Jepsen +// harness, which needs one to call `elastickv-admin encryption bootstrap`. +// +// This is TEST HARNESS TOOLING, not an operator tool. It exists because +// bootstrap takes the wrapped DEK bytes as an argument -- an operator gets them +// from their KMS -- and the Jepsen harness has only a local KEK file. Keeping it +// here rather than adding an `elastickv-admin` subcommand avoids putting DEK +// generation into operator-facing tooling, where handling raw key material +// needs its own design and review. +// +// The plaintext DEK never leaves this process: it is generated, wrapped under +// the KEK, and only the wrapped form is printed. The wrapped DEK is safe to pass +// on a command line, which is the whole point of the envelope scheme. +package main + +import ( + "crypto/rand" + "encoding/base64" + "flag" + "fmt" + "os" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/cockroachdb/errors" +) + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "jepsen-encryption-setup: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, out *os.File) error { + fs := flag.NewFlagSet("jepsen-encryption-setup", flag.ContinueOnError) + kekFile := fs.String("kek-file", "", "path to the §5.1 KEK file (32 raw bytes, owner-only mode)") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return errors.Wrap(err, "parse flags") + } + if *kekFile == "" { + return errors.New("--kek-file is required") + } + wrapped, err := wrapFreshDEK(*kekFile) + if err != nil { + return err + } + if _, err := fmt.Fprintln(out, wrapped); err != nil { + return errors.Wrap(err, "write wrapped dek") + } + return nil +} + +// wrapFreshDEK generates one AES-256 DEK and returns its base64 wrapped form. +func wrapFreshDEK(kekFile string) (string, error) { + wrapper, err := kek.NewFileWrapper(kekFile) + if err != nil { + return "", errors.Wrap(err, "open kek file") + } + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + return "", errors.Wrap(err, "generate dek") + } + wrapped, err := wrapper.Wrap(dek) + if err != nil { + return "", errors.Wrap(err, "wrap dek") + } + return base64.StdEncoding.EncodeToString(wrapped), nil +} diff --git a/cmd/jepsen-encryption-setup/main_test.go b/cmd/jepsen-encryption-setup/main_test.go new file mode 100644 index 000000000..ed3fd2e46 --- /dev/null +++ b/cmd/jepsen-encryption-setup/main_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/stretchr/testify/require" +) + +func writeTestKEK(t *testing.T) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "kek") + kekBytes := make([]byte, encryption.KeySize) + for i := range kekBytes { + kekBytes[i] = byte(i + 1) + } + require.NoError(t, os.WriteFile(path, kekBytes, 0o600)) + return path +} + +// The wrapped DEK must round-trip through the same wrapper the server uses, or +// bootstrap would be handed bytes the node cannot unwrap -- and the Jepsen suite +// would fail at bootstrap instead of running encrypted. +func TestWrapFreshDEKProducesAnUnwrappableDEK(t *testing.T) { + t.Parallel() + + path := writeTestKEK(t) + encoded, err := wrapFreshDEK(path) + require.NoError(t, err) + + raw, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + wrapper, err := kek.NewFileWrapper(path) + require.NoError(t, err) + dek, err := wrapper.Unwrap(raw) + require.NoError(t, err) + require.Len(t, dek, encryption.KeySize, + "bootstrap rejects a DEK that is not AES-256") +} + +// Two invocations must not produce the same DEK: the harness calls this once for +// the storage DEK and once for the raft DEK, and bootstrap requires them to +// differ. +func TestWrapFreshDEKGeneratesADistinctDEKEachCall(t *testing.T) { + t.Parallel() + + path := writeTestKEK(t) + wrapper, err := kek.NewFileWrapper(path) + require.NoError(t, err) + + seen := make(map[string]struct{}, 8) + for range 8 { + encoded, err := wrapFreshDEK(path) + require.NoError(t, err) + raw, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + dek, err := wrapper.Unwrap(raw) + require.NoError(t, err) + _, dup := seen[string(dek)] + require.False(t, dup, "each call must generate a fresh DEK") + seen[string(dek)] = struct{}{} + } +} + +func TestRunRequiresAKEKFile(t *testing.T) { + t.Parallel() + + require.Error(t, run(nil, os.Stdout)) +} diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index e804bcc73..e0c7aeac4 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -32,6 +32,13 @@ (def ^:private kek-test-bytes (apply str (repeat 32 "k"))) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) +(def ^:private admin-bin (str bin-dir "/elastickv-admin")) +(def ^:private encryption-setup-bin (str bin-dir "/jepsen-encryption-setup")) + +;; §5.2 DEK ids for the bootstrap. Any non-zero pair that differs is valid; +;; bootstrap rejects zero and rejects the two being equal. +(def ^:private storage-dek-id 1) +(def ^:private raft-dek-id 2) (def ^:private build-dir ;; local (control node) directory for built binaries @@ -54,7 +61,9 @@ "GOPATH" "/home/vagrant/go" "GOCACHE" "/home/vagrant/.cache/go-build"})] (doseq [[out-cmd args] [["elastickv" ["go" "build" "-o" (str build-dir "/elastickv") "./cmd/server"]] - ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]]]] + ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]] + ["elastickv-admin" ["go" "build" "-o" (str build-dir "/elastickv-admin") "./cmd/elastickv-admin"]] + ["jepsen-encryption-setup" ["go" "build" "-o" (str build-dir "/jepsen-encryption-setup") "./cmd/jepsen-encryption-setup"]]]] (let [{:keys [exit err]} (apply sh/sh (concat args [:env env :dir root]))] (when-not (zero? exit) (throw (ex-info (str "failed to build " out-cmd) {:err err}))))))) @@ -74,7 +83,7 @@ (c/on node (c/su (c/exec :mkdir :-p bin-dir) - (doseq [bin ["elastickv" "raftadmin"]] + (doseq [bin ["elastickv" "raftadmin" "elastickv-admin" "jepsen-encryption-setup"]] (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) @@ -239,6 +248,110 @@ (c/exec :env "RAFTADMIN_ALLOW_INSECURE=true" raftadmin-bin leader-addr "add_voter" peer-id peer-addr "0")))) +(defn encryption-endpoint + "Returns the gRPC address the EncryptionAdmin RPCs must be sent to. + + The bootstrap and cutover entries are proposed through the DEFAULT Raft + group, so a multi-group deployment must be addressed on that group's port + rather than on whichever port happens to be first in the map. group-ids is + sorted, so the lowest group id is the default one." + [node grpc-port raft-groups] + (if (seq raft-groups) + (group-addr node raft-groups (first (group-ids raft-groups))) + (node-addr node grpc-port))) + +(defn bootstrap-args + "argv for `elastickv-admin encryption bootstrap`. + + The writer batch comes from --discover-from rather than hand-written + --writer entries: §5.6 step 1a requires one registry entry per member, and + polling GetCapability is the only way to learn each node's real full_node_id + and local_epoch. A hand-written batch would go stale the moment a node + restarted and bumped its epoch." + [endpoint peer-endpoints wrapped-storage wrapped-raft] + (concat [admin-bin "encryption" "bootstrap" + (str "--endpoint=" endpoint) + (str "--storage-dek-id=" storage-dek-id) + (str "--raft-dek-id=" raft-dek-id) + (str "--wrapped-storage-dek=" wrapped-storage) + (str "--wrapped-raft-dek=" wrapped-raft)] + (map #(str "--discover-from=" %) peer-endpoints))) + +(defn enable-storage-envelope-args + "argv for `elastickv-admin encryption enable-storage-envelope`. + + This is the step that actually makes writes ciphertext: --encryption-enabled + only opens the mutator RPCs, and buildEncryptionWriteWiring keeps the store + gate closed until this entry applies." + [endpoint full-node-id local-epoch] + [admin-bin "encryption" "enable-storage-envelope" + (str "--endpoint=" endpoint) + (str "--proposer-node-id=" full-node-id) + (str "--proposer-local-epoch=" local-epoch)]) + +(defn parse-encryption-status + "Parses `elastickv-admin encryption status` output into a map. + + Returns :full-node-id and :local-epoch (needed as the proposer identity for + the cutover) and :storage-envelope-active (the only thing that proves the + cluster is storing ciphertext)." + [out] + (let [field (fn [k] (second (re-find (re-pattern (str "(?m)^\\s*" k ":\\s*(\\S+)\\s*$")) out))) + num (fn [k] (some-> (field k) Long/parseLong))] + {:full-node-id (num "full_node_id") + :local-epoch (num "local_epoch") + :storage-envelope-active (= "true" (field "storage_envelope_active"))})) + +(defn- encryption-status + [node endpoint] + (parse-encryption-status + (c/on node (c/su (c/exec admin-bin "encryption" "status" (str "--endpoint=" endpoint)))))) + +(defn- wrap-fresh-dek! + "Returns a base64 KEK-wrapped DEK, generated on the node." + [node] + (clojure.string/trim + (c/on node (c/su (c/exec encryption-setup-bin (str "--kek-file=" kek-file)))))) + +(defn- activate-encryption! + "Bootstraps the DEKs and performs the §7.1 Phase-1 storage cutover, then + VERIFIES it applied. + + The verification is the point. --encryption-enabled only enables the + EncryptionAdmin mutator RPCs; buildEncryptionWriteWiring deliberately keeps + the store's envelope gate closed until BOTH BootstrapEncryption and + EnableStorageEnvelope have applied. Without these calls every workload ran + against a cleartext cluster and could still report PASS -- which, as the §8.4 + gate's own rationale says, is worse than having no gate, because the run gets + recorded as encryption evidence. So a cluster that does not report + storage_envelope_active here must fail setup rather than proceed." + [test node grpc-port raft-groups] + (let [endpoint (encryption-endpoint node grpc-port raft-groups) + peers (map #(encryption-endpoint % grpc-port raft-groups) (:nodes test)) + wrapped-s (wrap-fresh-dek! node) + wrapped-r (wrap-fresh-dek! node)] + (info "bootstrapping encryption" endpoint) + (c/on node (c/su (apply c/exec (bootstrap-args endpoint peers wrapped-s wrapped-r)))) + (let [{:keys [full-node-id local-epoch]} (encryption-status node endpoint)] + (when-not full-node-id + (throw (ex-info "encryption status did not report a full_node_id" + {:endpoint endpoint}))) + (info "enabling storage envelope" endpoint full-node-id local-epoch) + (c/on node (c/su (apply c/exec (enable-storage-envelope-args + endpoint full-node-id (or local-epoch 0)))))) + ;; Every node must report the cutover, not just the proposer: a node that + ;; has not applied it is still writing cleartext, and the workload would be + ;; measuring a half-encrypted cluster. + (doseq [peer (:nodes test)] + (let [peer-endpoint (encryption-endpoint peer grpc-port raft-groups)] + (util/await-fn + (fn [] + (when (:storage-envelope-active (encryption-status node peer-endpoint)) + true)) + {:timeout 60000 + :log-message (str "waiting for storage envelope cutover on " peer)}))) + (info "encryption active on every node"))) + (defrecord ElastickvDB [opts] db/DB (setup! [_ test node] @@ -281,7 +394,14 @@ (warn t "retrying join for" peer) nil))) {:timeout 120000 - :log-message (str "joining " peer)})))) + :log-message (str "joining " peer)})) + ;; After membership, not before: the bootstrap's writer batch needs a + ;; registry entry for every member (§5.6 step 1a), and the cutover's + ;; capability gate requires every voter to report encryption-capable. + ;; Running this against a single-node cluster would register one writer + ;; and then refuse the cutover once the peers joined. + (when (:encryption opts) + (activate-encryption! test node grpc-port raft-groups)))) (info "node started" node)) (teardown! [_ _test node] diff --git a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj index 7848d051a..e94e81fdd 100644 --- a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj +++ b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj @@ -521,7 +521,8 @@ :dynamo-port node->port :raft-groups (:raft-groups opts) :shard-ranges (:shard-ranges opts) - :server-env (:server-env opts)})) + :server-env (:server-env opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/dynamodb_types_workload.clj b/jepsen/src/elastickv/dynamodb_types_workload.clj index fc3e6a4be..3105c69f3 100644 --- a/jepsen/src/elastickv/dynamodb_types_workload.clj +++ b/jepsen/src/elastickv/dynamodb_types_workload.clj @@ -427,7 +427,8 @@ :redis-port (or (:redis-port opts) 6379) :dynamo-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/redis_zset_safety_workload.clj b/jepsen/src/elastickv/redis_zset_safety_workload.clj index a214da5fd..0b499756b 100644 --- a/jepsen/src/elastickv/redis_zset_safety_workload.clj +++ b/jepsen/src/elastickv/redis_zset_safety_workload.clj @@ -1415,7 +1415,8 @@ (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) :redis-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 10)) time-limit (or (:time-limit opts) 60) faults (if local? diff --git a/jepsen/src/elastickv/s3_workload.clj b/jepsen/src/elastickv/s3_workload.clj index c25655cc6..317b2aa1d 100644 --- a/jepsen/src/elastickv/s3_workload.clj +++ b/jepsen/src/elastickv/s3_workload.clj @@ -187,7 +187,8 @@ :redis-port (or (:redis-port opts) 6379) :s3-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/sqs_htfifo_workload.clj b/jepsen/src/elastickv/sqs_htfifo_workload.clj index 9942972a5..32ecaeb9d 100644 --- a/jepsen/src/elastickv/sqs_htfifo_workload.clj +++ b/jepsen/src/elastickv/sqs_htfifo_workload.clj @@ -495,7 +495,8 @@ :sqs-port node->port :sqs-region sqs-region :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) ;; Drain must outlast the visibility-timeout window plus a diff --git a/jepsen/test/elastickv/encrypted_cluster_test.clj b/jepsen/test/elastickv/encrypted_cluster_test.clj index 6e9d4585b..8796b6f53 100644 --- a/jepsen/test/elastickv/encrypted_cluster_test.clj +++ b/jepsen/test/elastickv/encrypted_cluster_test.clj @@ -11,8 +11,13 @@ (:require [clojure.test :refer :all] [elastickv.cli :as cli] [elastickv.db :as ekdb] + [elastickv.dynamodb-multi-table-workload :as multi] + [elastickv.dynamodb-types-workload :as types] [elastickv.dynamodb-workload :as dynamo] - [elastickv.redis-workload :as redis])) + [elastickv.redis-workload :as redis] + [elastickv.redis-zset-safety-workload :as zset] + [elastickv.s3-workload :as s3] + [elastickv.sqs-htfifo-workload :as sqs])) (deftest encryption-flag-is-available-to-every-workload ;; It lives in common-cli-opts rather than per-workload, so a future @@ -84,3 +89,114 @@ (set (remove #(or (= % "/var/lib/elastickv/keys.json") (= % "/var/lib/elastickv/kek.bin")) plain)))))) + +;; --------------------------------------------------------------------------- +;; The switch must ACTIVATE encryption, not just reach the server +;; --------------------------------------------------------------------------- +;; +;; The flag tests above pass whether or not the cluster ends up encrypted: +;; --encryption-enabled only opens the EncryptionAdmin mutator RPCs, and +;; buildEncryptionWriteWiring keeps the store's envelope gate closed until BOTH +;; BootstrapEncryption and EnableStorageEnvelope have applied. Without those +;; calls every workload ran against a cleartext cluster and still reported PASS +;; -- the exact failure this namespace's docstring names as worse than no gate. + +(deftest bootstrap-discovers-the-writer-batch-from-every-member + ;; §5.6 step 1a needs one registry entry per member, with each node's real + ;; full_node_id and local_epoch. Hand-written --writer entries would go stale + ;; as soon as a node restarted and bumped its epoch, so the batch must be + ;; discovered from every node. + (let [args (ekdb/bootstrap-args "n1:50051" ["n1:50051" "n2:50051" "n3:50051"] "WS" "WR")] + (is (= 3 (count (filter #(re-find #"^--discover-from=" %) args)))) + (is (some #{"--discover-from=n2:50051"} args)) + (is (some #{"--discover-from=n3:50051"} args)))) + +(deftest bootstrap-passes-distinct-non-zero-dek-ids + ;; Bootstrap rejects a zero id and rejects the two being equal. + (let [args (ekdb/bootstrap-args "n1:50051" ["n1:50051"] "WS" "WR") + id (fn [flag] (some->> args + (filter #(clojure.string/starts-with? % (str flag "="))) + first + (re-find #"\d+$") + Long/parseLong))] + (is (pos? (id "--storage-dek-id"))) + (is (pos? (id "--raft-dek-id"))) + (is (not= (id "--storage-dek-id") (id "--raft-dek-id"))))) + +(deftest bootstrap-carries-both-wrapped-deks + (let [args (set (ekdb/bootstrap-args "n1:50051" ["n1:50051"] "WRAPPED-S" "WRAPPED-R"))] + (is (contains? args "--wrapped-storage-dek=WRAPPED-S")) + (is (contains? args "--wrapped-raft-dek=WRAPPED-R")))) + +(deftest enable-storage-envelope-carries-the-proposer-identity + ;; §6.1 treats proposer-node-id 0 as the not-capable sentinel, so the real + ;; full_node_id read back from `encryption status` has to be threaded through. + (let [args (set (ekdb/enable-storage-envelope-args "n1:50051" 12345 7))] + (is (contains? args "--proposer-node-id=12345")) + (is (contains? args "--proposer-local-epoch=7")))) + +(deftest status-parsing-reads-the-fields-the-cutover-needs + (let [out (str "capability:\n" + " encryption_capable: true\n" + " sidecar_present: true\n" + " full_node_id: 8675309\n" + " local_epoch: 3\n" + "sidecar:\n" + " storage_envelope_active: true\n") + got (ekdb/parse-encryption-status out)] + (is (= 8675309 (:full-node-id got))) + (is (= 3 (:local-epoch got))) + (is (true? (:storage-envelope-active got))))) + +(deftest status-parsing-treats-an-inactive-envelope-as-inactive + ;; The polling loop waits on this value, so a false must never read as true: + ;; that would let the workload start against a cleartext cluster, which is the + ;; whole failure mode. + (let [out (str "capability:\n" + " full_node_id: 1\n" + " local_epoch: 0\n" + "sidecar:\n" + " storage_envelope_active: false\n") + got (ekdb/parse-encryption-status out)] + (is (false? (:storage-envelope-active got))))) + +(deftest status-parsing-does-not-invent-an-active-envelope + ;; A node with no sidecar omits the line entirely. Absent must not be active. + (let [got (ekdb/parse-encryption-status "capability:\n encryption_capable: false\n")] + (is (false? (:storage-envelope-active got))) + (is (nil? (:full-node-id got))))) + +(deftest encryption-admin-targets-the-default-raft-group + ;; Bootstrap and the cutover are proposed through the default group, so a + ;; multi-group deployment must be addressed on that group's port. group-ids is + ;; sorted, so the lowest id is the default one. + (is (= "n1:50051" (ekdb/encryption-endpoint "n1" 50051 nil))) + (is (= "n1:50061" (ekdb/encryption-endpoint "n1" 50051 {1 50061, 2 50062}))) + (is (= "n1:50061" (ekdb/encryption-endpoint "n1" 50051 {2 50062, 1 50061})))) + +;; --------------------------------------------------------------------------- +;; Every workload that accepts --encryption must honour it +;; --------------------------------------------------------------------------- + +(deftest every-workload-accepting-encryption-propagates-it + ;; --encryption lives in common-cli-opts so that, as the test above puts it, + ;; "a future workload gets the gate without opting in". That promise was not + ;; kept: five entrypoints accepted the flag from the common options and then + ;; dropped it when constructing ekdb/db, so `--encryption` on those commands + ;; silently launched a cleartext cluster -- the CLI advertising a guarantee it + ;; did not provide. + ;; + ;; Driving every constructor from one list is the point: a new workload that + ;; forgets to thread the option fails here instead of shipping a silent lie. + (doseq [[label ctor] [["redis" redis/elastickv-redis-test] + ["redis-zset-safety" zset/elastickv-zset-safety-test] + ["dynamodb" dynamo/elastickv-dynamodb-test] + ["dynamodb-types" types/elastickv-dynamodb-types-test] + ["dynamodb-multi-table" multi/elastickv-dynamodb-multi-table-test] + ["s3" s3/elastickv-s3-test] + ["sqs-htfifo" sqs/elastickv-sqs-htfifo-test]]] + (testing label + (is (true? (get-in (ctor {:encryption true}) [:db :opts :encryption])) + (str label " accepts --encryption from common-cli-opts but drops it")) + (is (not (true? (get-in (ctor {}) [:db :opts :encryption]))) + (str label " must not enable encryption when it was not requested")))))