jepsen: run the existing suites against an encrypted cluster (§8.4)

cmd/jepsen-encryption-setup/main.go

@@ -0,0 +1,72 @@
+// Command jepsen-encryption-setup produces a KEK-wrapped DEK for the Jepsen
+// harness, which needs one to call `elastickv-admin encryption bootstrap`.
+//
+// This is TEST HARNESS TOOLING, not an operator tool. It exists because
+// bootstrap takes the wrapped DEK bytes as an argument -- an operator gets them
+// from their KMS -- and the Jepsen harness has only a local KEK file. Keeping it
+// here rather than adding an `elastickv-admin` subcommand avoids putting DEK
+// generation into operator-facing tooling, where handling raw key material
+// needs its own design and review.
+//
+// The plaintext DEK never leaves this process: it is generated, wrapped under
+// the KEK, and only the wrapped form is printed. The wrapped DEK is safe to pass
+// on a command line, which is the whole point of the envelope scheme.
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/bootjp/elastickv/internal/encryption"
+	"github.com/bootjp/elastickv/internal/encryption/kek"
+	"github.com/cockroachdb/errors"
+)
+
+func main() {
+	if err := run(os.Args[1:], os.Stdout); err != nil {
+		fmt.Fprintf(os.Stderr, "jepsen-encryption-setup: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(args []string, out *os.File) error {
+	fs := flag.NewFlagSet("jepsen-encryption-setup", flag.ContinueOnError)
+	kekFile := fs.String("kek-file", "", "path to the §5.1 KEK file (32 raw bytes, owner-only mode)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
+		return errors.Wrap(err, "parse flags")
+	}
+	if *kekFile == "" {
+		return errors.New("--kek-file is required")
+	}
+	wrapped, err := wrapFreshDEK(*kekFile)
+	if err != nil {
+		return err
+	}
+	if _, err := fmt.Fprintln(out, wrapped); err != nil {
+		return errors.Wrap(err, "write wrapped dek")
+	}
+	return nil
+}
+
+// wrapFreshDEK generates one AES-256 DEK and returns its base64 wrapped form.
+func wrapFreshDEK(kekFile string) (string, error) {
+	wrapper, err := kek.NewFileWrapper(kekFile)
+	if err != nil {
+		return "", errors.Wrap(err, "open kek file")
+	}
+	dek := make([]byte, encryption.KeySize)
+	if _, err := rand.Read(dek); err != nil {
+		return "", errors.Wrap(err, "generate dek")
+	}
+	wrapped, err := wrapper.Wrap(dek)
+	if err != nil {
+		return "", errors.Wrap(err, "wrap dek")
+	}
+	return base64.StdEncoding.EncodeToString(wrapped), nil
+}

cmd/jepsen-encryption-setup/main_test.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"encoding/base64"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/bootjp/elastickv/internal/encryption"
+	"github.com/bootjp/elastickv/internal/encryption/kek"
+	"github.com/stretchr/testify/require"
+)
+
+func writeTestKEK(t *testing.T) string {
+	t.Helper()
+
+	path := filepath.Join(t.TempDir(), "kek")
+	kekBytes := make([]byte, encryption.KeySize)
+	for i := range kekBytes {
+		kekBytes[i] = byte(i + 1)
+	}
+	require.NoError(t, os.WriteFile(path, kekBytes, 0o600))
+	return path
+}
+
+// The wrapped DEK must round-trip through the same wrapper the server uses, or
+// bootstrap would be handed bytes the node cannot unwrap -- and the Jepsen suite
+// would fail at bootstrap instead of running encrypted.
+func TestWrapFreshDEKProducesAnUnwrappableDEK(t *testing.T) {
+	t.Parallel()
+
+	path := writeTestKEK(t)
+	encoded, err := wrapFreshDEK(path)
+	require.NoError(t, err)
+
+	raw, err := base64.StdEncoding.DecodeString(encoded)
+	require.NoError(t, err)
+
+	wrapper, err := kek.NewFileWrapper(path)
+	require.NoError(t, err)
+	dek, err := wrapper.Unwrap(raw)
+	require.NoError(t, err)
+	require.Len(t, dek, encryption.KeySize,
+		"bootstrap rejects a DEK that is not AES-256")
+}
+
+// Two invocations must not produce the same DEK: the harness calls this once for
+// the storage DEK and once for the raft DEK, and bootstrap requires them to
+// differ.
+func TestWrapFreshDEKGeneratesADistinctDEKEachCall(t *testing.T) {
+	t.Parallel()
+
+	path := writeTestKEK(t)
+	wrapper, err := kek.NewFileWrapper(path)
+	require.NoError(t, err)
+
+	seen := make(map[string]struct{}, 8)
+	for range 8 {
+		encoded, err := wrapFreshDEK(path)
+		require.NoError(t, err)
+		raw, err := base64.StdEncoding.DecodeString(encoded)
+		require.NoError(t, err)
+		dek, err := wrapper.Unwrap(raw)
+		require.NoError(t, err)
+		_, dup := seen[string(dek)]
+		require.False(t, dup, "each call must generate a fresh DEK")
+		seen[string(dek)] = struct{}{}
+	}
+}
+
+func TestRunRequiresAKEKFile(t *testing.T) {
+	t.Parallel()
+
+	require.Error(t, run(nil, os.Stdout))
+}

docs/design/2026_04_29_partial_data_at_rest_encryption.md

@@ -2445,6 +2445,15 @@ different output bytes; FSM apply still deterministic), so no new
 Jepsen workload is required. A pass under the existing suite is the
 acceptance gate.
 
+**Implemented.** `lein test :only ...` aside, any workload accepts
+`--encryption`, which provisions the §5.1 KEK file on each node and
+starts the server with `--encryption-enabled`, `--kekFile` and
+`--encryptionSidecarPath`. The switch defaults off so the existing
+unencrypted runs are unchanged, and `server-args` is a pure function so
+a test can assert the flags actually reach the server — a `--encryption`
+run that silently produced an unencrypted cluster would report PASS and
+be recorded as evidence for this gate.
+
 ---
 
 ## 9. Operational concerns

jepsen/src/elastickv/cli.clj

@@ -14,6 +14,11 @@
     :default default-nodes-str]
    [nil "--local" "Run locally without SSH or nemesis."
     :default false]
+   ;; §8.4: the encrypted acceptance gate is the EXISTING suites run
+   ;; against an encrypted cluster, so this is a cluster-setup switch
+   ;; rather than a workload selector — every workload honours it.
+   [nil "--encryption" "Run against a cluster with data-at-rest encryption enabled."
+    :default false]
    [nil "--host HOST" "Host override for clients."
     :default nil]
    [nil "--grpc-port PORT" "gRPC/Raft port."