diff --git a/docs/firecracker.md b/docs/firecracker.md index 28e8d9a..c623a86 100644 --- a/docs/firecracker.md +++ b/docs/firecracker.md @@ -232,6 +232,76 @@ lost there; the volume's journal keeps it consistent. A volume survives its VM's end, a daemon restart or `kill -9` and that destroy; only `DELETE /volumes/{id}` removes it. +### Controller-only cold backup and new-volume restore + +These operations use the control `X-API-Key`, not a guest token, and are available +only on a local Firecracker worker with volumes enabled. **Raw ext4 backups are +credential-bearing**, including possible deleted blocks. They are not sanitized +customer exports. Store artifacts privately with trusted provenance; SHA-256 +detects corruption, not an attacker replacing both manifest and payload. + +`POST /volumes/{volumeID}/backup` takes no body/query and returns +`application/vnd.sandboxd.volume-backup`. A detached check and per-volume +reservation atomically exclude guest attachment, deletion and another backup +(409) until hashing and streaming finish or are cancelled. Unrelated volumes +can progress; the driver does not hold its global mutex across bulk I/O. + +The binary envelope is: + +| Bytes | Content | +|---|---| +| 8 | ASCII `SBXVOL01` | +| 4 | Big-endian unsigned JSON manifest length, 1–4096 | +| manifest length | UTF-8 JSON fields below | +| exactly `sizeBytes` | Raw ext4 data; no trailing bytes | + +Manifest fields: `sourceVolumeID`, integer `sizeBytes`, lowercase hex `sha256` +of the disk payload, and `applicationConsistency:"not-asserted"`. Disk size must +be a whole MiB, from 64 MiB through 1 TiB. The server hashes the reserved stable +disk before emitting the manifest. A receiver must verify exact payload length, +source ID and digest before atomically publishing its immutable artifact. +HTTP 200 headers alone do not establish a completed backup. + +`POST /volumes/restore?name=&sourceVolumeID=` +accepts that media type and envelope. The explicitly expected source must match +the manifest. It returns 201 with the usual volume identity/token response only +after publishing a complete, synced disk under a **fresh random ID**. Existing +IDs/names are never overwritten; a duplicate/reserved name returns 409. +Unknown versions, malformed metadata, truncated/extra data and checksum mismatch +return 400. An unsupported native backup driver returns 501. + +At most two transfers per daemon run concurrently; each has a ten-minute total +deadline including hashing and transfer. Additional HTTP transfers return 429. +Native operation conflicts return 409. Imports reserve name and full logical +capacity before accepting disk bytes; existing budget and disk-floor checks +apply (400 on insufficient capacity). Disk headroom conservatively reserves the +full size of concurrent imports even when their staging files are partly written. +Transfer buffers are fixed-size; no disk-sized in-memory archive is built. + +Restore writes root-only staging, validates all bytes, syncs the file, then +publishes record-before-disk without replacing an existing file. Errors/cancel +remove staging; restart removes incomplete import leftovers only after obtaining +the exclusive instance claim. A competing daemon constructor cannot erase an +active restore before its conflicting claim is rejected. A crash after complete +durable publication but before +the response may leave a valid new volume: reconcile by its requested name +instead of retrying under another name. No untrusted filesystem is mounted on +the host. + +Detached does **not** mean application-quiesced: the guest flush above is +best-effort. Forge's integration policy stops admission/customer access and +customer services, verifies their exit, and requires a successful explicit sync; +failed/unknown quiescence aborts a normal backup. Forge records that separate +receipt, not a native application-consistency guarantee. + +Forge restores into quarantine: no internet, customer route or automatic +service replay; the original workspace stays stopped/revoked during cutover. +Capabilities, client enrollment and guest SSH host identity must be rotated. +Retained service/subscription credentials are cleared through their supported +interfaces and deliberately re-enrolled before release. Deleting local tokens +does not prove provider-side revocation. Customer file export remains a separate +explicit allowlist policy; this raw API makes no secret-free-content promise. + ## Memory return (`-fc-balloon`) A Firecracker VM keeps every page its guest has ever touched: after a diff --git a/internal/control/control.go b/internal/control/control.go index 47a53bc..366bb1a 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -105,7 +105,8 @@ type Service struct { // volumes is Config.Volumes: nil when volumes are off. volumes vm.Volumes // egress is Config.Egress: nil when egress policies are off. - egress vm.EgressPolicies + egress vm.EgressPolicies + volumeTransferSlots chan struct{} } // Open creates the durable ledger before accepting any VM allocations. The @@ -204,6 +205,7 @@ func Open(ctx context.Context, path string, driver vm.Driver, cfg Config) (*Serv s := &Service{ledger: ledger, cfg: cfg, templates: templates, tokenKey: slices.Clone(cfg.TokenSecret), workers: members, byID: make(map[string]*member, len(members)), busy: make(map[string]chan struct{}), apiHash: sha256.Sum256([]byte(cfg.APIKey)), stop: make(chan struct{}), done: make(chan struct{}), signers: make(map[string]string), volumes: cfg.Volumes, egress: cfg.Egress} + s.volumeTransferSlots = make(chan struct{}, vm.MaxVolumeTransfers) rows, err := ledger.Active(ctx) if err != nil { _ = ledger.Close() @@ -779,6 +781,8 @@ func (s *Service) launch(ctx context.Context, row *store.Row, place placement) e // The driver alone knows every VM trace holding a volume; the client // learns why, while the row is held exactly as for any failed start. switch { + case errors.Is(err, vm.ErrVolumeTransferBusy): + return &refusal{http.StatusConflict, "volume storage operation in progress"} case errors.Is(err, vm.ErrVolumeInUse): return &refusal{http.StatusConflict, "volume is attached to another sandbox"} case errors.Is(err, vm.ErrVolumeNotFound): diff --git a/internal/control/volume_backups.go b/internal/control/volume_backups.go new file mode 100644 index 0000000..4b347bd --- /dev/null +++ b/internal/control/volume_backups.go @@ -0,0 +1,111 @@ +package control + +import ( + "context" + "errors" + "io" + "log" + "mime" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gitmoot/sandboxd/internal/vm" +) + +// serveVolumeTransfers runs after the normal control API authentication. Raw +// block backups are operator-only credentials, never customer-safe file exports. +func (s *Service) serveVolumeTransfers(w http.ResponseWriter, r *http.Request) bool { + id, backup := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/volumes/"), "/backup") + restore := r.URL.Path == "/volumes/restore" + if !restore && (!strings.HasPrefix(r.URL.Path, "/volumes/") || !backup) { + return false + } + if r.Method != http.MethodPost { + s.fail(w, ProfileE2B, http.StatusMethodNotAllowed, "method not allowed") + return true + } + native, ok := s.volumes.(vm.VolumeBackups) + if !ok { + s.fail(w, ProfileE2B, http.StatusNotImplemented, "native volume backups unavailable") + return true + } + if !restore && (!vm.ValidVolumeID(id) || r.ContentLength != 0 || len(r.TransferEncoding) != 0 || r.URL.RawQuery != "") { + s.fail(w, ProfileE2B, http.StatusBadRequest, "backup requires a volume ID and no body or query") + return true + } + select { + case s.volumeTransferSlots <- struct{}{}: + defer func() { <-s.volumeTransferSlots }() + default: + s.fail(w, ProfileE2B, http.StatusTooManyRequests, "too many volume transfers") + return true + } + deadline := time.Now().Add(vm.VolumeTransferTimeout) + ctx, cancel := context.WithDeadline(r.Context(), deadline) + defer cancel() + controller := http.NewResponseController(w) + if err := controller.SetReadDeadline(deadline); err != nil && !errors.Is(err, http.ErrNotSupported) { + s.fail(w, ProfileE2B, 503, "transfer deadlines unavailable") + return true + } + if err := controller.SetWriteDeadline(deadline); err != nil && !errors.Is(err, http.ErrNotSupported) { + s.fail(w, ProfileE2B, 503, "transfer deadlines unavailable") + return true + } + stopped := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + _ = controller.SetReadDeadline(time.Now()) + _ = controller.SetWriteDeadline(time.Now()) + close(stopped) + }) + defer func() { + if !stop() { + <-stopped + } + _ = controller.SetReadDeadline(time.Time{}) + _ = controller.SetWriteDeadline(time.Time{}) + }() + if restore { + contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + query, queryErr := url.ParseQuery(r.URL.RawQuery) + if err != nil || queryErr != nil || contentType != vm.VolumeBackupMediaType || len(params) != 0 || len(query) != 2 || len(query["name"]) != 1 || !vm.ValidVolumeName(query.Get("name")) || len(query["sourceVolumeID"]) != 1 || !vm.ValidVolumeID(query.Get("sourceVolumeID")) { + s.fail(w, ProfileE2B, http.StatusBadRequest, "restore requires backup media type, new name and expected sourceVolumeID") + return true + } + manifest, err := vm.ReadVolumeManifest(r.Body) + if err != nil || manifest.SourceVolumeID != query.Get("sourceVolumeID") { + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume backup manifest") + return true + } + volume, err := native.RestoreVolume(ctx, query.Get("name"), manifest, r.Body) + if err != nil { + s.volumeFailure(w, "restore", err) + return true + } + jsonResponse(w, http.StatusCreated, s.withToken(volume)) + return true + } + snapshot, err := native.OpenVolumeBackup(ctx, id) + if err != nil { + s.volumeFailure(w, "backup", err) + return true + } + defer snapshot.Data.Close() + w.Header().Set("Content-Type", vm.VolumeBackupMediaType) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + if err := vm.WriteVolumeManifest(w, snapshot.Manifest); err != nil { + log.Printf("volume backup manifest transfer failed: %v", err) + panic(http.ErrAbortHandler) + } + // Disk size was checked before headers; copy exactly that many bytes. A + // failed stream aborts the response rather than appending an error to a backup. + count, err := io.CopyBuffer(w, io.LimitReader(snapshot.Data, snapshot.Manifest.SizeBytes), make([]byte, 64<<10)) + if err != nil || count != snapshot.Manifest.SizeBytes || ctx.Err() != nil { + log.Printf("volume backup transfer incomplete: bytes=%d error=%v", count, err) + panic(http.ErrAbortHandler) + } + return true +} diff --git a/internal/control/volumes.go b/internal/control/volumes.go index 3cafc8f..ec33d35 100644 --- a/internal/control/volumes.go +++ b/internal/control/volumes.go @@ -47,6 +47,9 @@ func (s *Service) withToken(volume vm.Volume) volumeAndToken { // serveVolumes answers E2B's volume API when volumes are configured, and // reports whether path was a volume route. func (s *Service) serveVolumes(w http.ResponseWriter, r *http.Request) bool { + if s.serveVolumeTransfers(w, r) { + return true + } path := r.URL.Path switch { case path == "/volumecontent" || strings.HasPrefix(path, "/volumecontent/"): @@ -91,6 +94,10 @@ func (s *Service) volumeFailure(w http.ResponseWriter, action string, err error) s.fail(w, ProfileE2B, http.StatusConflict, "a volume with that name already exists") case errors.Is(err, vm.ErrVolumeInUse): s.fail(w, ProfileE2B, http.StatusConflict, "volume is attached to a sandbox: kill the sandbox first") + case errors.Is(err, vm.ErrVolumeTransferBusy): + s.fail(w, ProfileE2B, http.StatusConflict, "volume storage operation in progress") + case errors.Is(err, vm.ErrVolumeBackupInvalid): + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume backup") case errors.Is(err, vm.ErrVolumeBudget): // 400 is the only client error the SDKs declare for create. s.fail(w, ProfileE2B, http.StatusBadRequest, "volume budget exhausted: delete a volume first") diff --git a/internal/vm/firecracker.go b/internal/vm/firecracker.go index 2ceb84a..e318024 100644 --- a/internal/vm/firecracker.go +++ b/internal/vm/firecracker.go @@ -144,7 +144,8 @@ type FirecrackerDriver struct { // volumeMu orders every volume create, delete, attach and release, so // an attach's in-use check and its record write are one step. - volumeMu sync.Mutex + volumeMu sync.Mutex + volumeTransfers map[string]fcVolumeTransfer // mkfs is the trusted host mke2fs, set when volumes are on. mkfs string diff --git a/internal/vm/firecracker_backup.go b/internal/vm/firecracker_backup.go new file mode 100644 index 0000000..b3159b3 --- /dev/null +++ b/internal/vm/firecracker_backup.go @@ -0,0 +1,324 @@ +//go:build linux + +package vm + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sync" + "syscall" +) + +// volumeTransfers is protected by volumeMu. A zero SizeMiB is a backup of an +// existing volume; a positive size reserves a new name and restore capacity. +// Reservations are process-local: the daemon's instance/ledger claim excludes +// other writers, and restart removes incomplete .ext4.tmp imports. +type fcVolumeTransfer struct { + Name string + SizeMiB int +} + +func (d *FirecrackerDriver) reserveVolumeTransfer(id string, transfer fcVolumeTransfer) error { + if _, exists := d.volumeTransfers[id]; exists { + return ErrVolumeTransferBusy + } + if len(d.volumeTransfers) >= MaxVolumeTransfers { + return ErrVolumeTransferBusy + } + if d.volumeTransfers == nil { + d.volumeTransfers = make(map[string]fcVolumeTransfer) + } + d.volumeTransfers[id] = transfer + return nil +} + +func (d *FirecrackerDriver) releaseVolumeTransfer(id string) { + d.volumeMu.Lock() + delete(d.volumeTransfers, id) + d.volumeMu.Unlock() +} + +// reservedVolumeMiB also rejects names reserved by an in-flight restore. Count +// full restore sizes against disk headroom conservatively, even if a stream has +// partially allocated its staging file; a slow import must never overcommit it. +func (d *FirecrackerDriver) reservedVolumeMiB(name string) (int, error) { + reserved := 0 + for _, transfer := range d.volumeTransfers { + if transfer.SizeMiB == 0 { + continue + } + if transfer.Name == name { + return 0, ErrVolumeExists + } + reserved += transfer.SizeMiB + } + return reserved, nil +} + +func (d *FirecrackerDriver) OpenVolumeBackup(ctx context.Context, id string) (VolumeBackup, error) { + if err := d.volumesOn(); err != nil { + return VolumeBackup{}, err + } + ctx, cancel := context.WithTimeout(ctx, VolumeTransferTimeout) + success := false + defer func() { + if !success { + cancel() + } + }() + d.volumeMu.Lock() + volume, err := d.readVolume(id) + if err == nil { + err = d.volumeInUse(id) + } + if err == nil { + err = d.reserveVolumeTransfer(id, fcVolumeTransfer{}) + } + d.volumeMu.Unlock() + if err != nil { + return VolumeBackup{}, err + } + defer func() { + if !success { + d.releaseVolumeTransfer(id) + } + }() + fd, err := syscall.Open(d.volumeDisk(id), syscall.O_RDONLY|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return VolumeBackup{}, err + } + file := os.NewFile(uintptr(fd), "volume-backup") + stop := context.AfterFunc(ctx, func() { _ = file.Close() }) + defer func() { + if !success { + stop() + _ = file.Close() + } + }() + info, err := file.Stat() + if err != nil { + return VolumeBackup{}, err + } + stat, ok := info.Sys().(*syscall.Stat_t) + size := int64(volume.SizeMiB) << 20 + if !ok || !info.Mode().IsRegular() || stat.Nlink != 1 || stat.Uid != uint32(os.Geteuid()) || info.Size() != size { + return VolumeBackup{}, errors.New("volume disk is not a detached native disk") + } + digest := sha256.New() + if err := copyVolumeBytes(ctx, digest, file, size); err != nil { + return VolumeBackup{}, err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return VolumeBackup{}, err + } + manifest := VolumeManifest{SourceVolumeID: id, SizeBytes: size, SHA256: hex.EncodeToString(digest.Sum(nil)), ApplicationConsistency: "not-asserted"} + if err := manifest.Validate(); err != nil { + return VolumeBackup{}, err + } + if err := ctx.Err(); err != nil { + return VolumeBackup{}, err + } + reader := &fcBackupReader{file: file, ctx: ctx, cancel: cancel, stop: stop, release: func() { d.releaseVolumeTransfer(id) }} + success = true + return VolumeBackup{Manifest: manifest, Data: reader}, nil +} + +type fcBackupReader struct { + file *os.File + ctx context.Context + cancel context.CancelFunc + stop func() bool + release func() + once sync.Once + err error +} + +func (r *fcBackupReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.file.Read(p) +} +func (r *fcBackupReader) Close() error { + r.once.Do(func() { r.stop(); r.cancel(); r.err = r.file.Close(); r.release() }) + return r.err +} + +func (d *FirecrackerDriver) RestoreVolume(ctx context.Context, name string, m VolumeManifest, source io.Reader) (result Volume, runErr error) { + if err := d.volumesOn(); err != nil { + return Volume{}, err + } + if !ValidVolumeName(name) { + return Volume{}, ErrVolumeBackupInvalid + } + if err := m.Validate(); err != nil { + return Volume{}, err + } + ctx, cancel := context.WithTimeout(ctx, VolumeTransferTimeout) + defer cancel() + d.volumeMu.Lock() + volume, err := d.reserveRestore(ctx, name, int(m.SizeBytes>>20)) + d.volumeMu.Unlock() + if err != nil { + return Volume{}, err + } + defer d.releaseVolumeTransfer(volume.ID) + path := d.volumeDisk(volume.ID) + tmp := path + ".tmp" + file, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return Volume{}, err + } + committed := false + defer func() { + _ = file.Close() + if !committed { + runErr = errors.Join(runErr, fcRemoveMissing(tmp), fcSyncPath(d.cfg.VolumeDir)) + } + }() + stop := context.AfterFunc(ctx, func() { _ = file.Close() }) + defer stop() + if err := file.Chmod(0o600); err != nil { + return Volume{}, err + } + digest := sha256.New() + sink := &volumeWriteError{Writer: file} + if err := copyVolumeBytes(ctx, io.MultiWriter(sink, digest), source, m.SizeBytes); err != nil { + if ctx.Err() != nil { + return Volume{}, ctx.Err() + } + if sink.err != nil { + return Volume{}, sink.err + } + return Volume{}, fmt.Errorf("%w: disk transfer: %v", ErrVolumeBackupInvalid, err) + } + // Exact envelope length is mandatory: no appended archive entries or bytes. + var extra [1]byte + n, err := io.ReadFull(source, extra[:]) + if ctx.Err() != nil { + return Volume{}, ctx.Err() + } + if n != 0 || err != io.EOF || hex.EncodeToString(digest.Sum(nil)) != m.SHA256 { + return Volume{}, ErrVolumeBackupInvalid + } + if err := file.Sync(); err != nil { + return Volume{}, err + } + if err := file.Close(); err != nil { + return Volume{}, err + } + d.volumeMu.Lock() + defer d.volumeMu.Unlock() + if err := ctx.Err(); err != nil { + return Volume{}, err + } + record, err := json.Marshal(volume) + if err != nil { + return Volume{}, err + } + // Record first and durable, disk last. Startup already removes a record with + // no disk. Once a final disk is visible, its complete synced record exists. + recordFile, err := os.OpenFile(d.volumeRecord(volume.ID), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return Volume{}, err + } + published := false + defer func() { + if !committed { + if published { + runErr = errors.Join(runErr, fcRemoveMissing(path)) + } + runErr = errors.Join(runErr, fcRemoveMissing(d.volumeRecord(volume.ID)), fcSyncPath(d.cfg.VolumeDir)) + } + }() + recordErr := writeVolumeBytes(recordFile, record) + if recordErr == nil { + recordErr = recordFile.Sync() + } + recordErr = errors.Join(recordErr, recordFile.Close()) + if recordErr != nil { + return Volume{}, recordErr + } + if err := fcSyncPath(d.cfg.VolumeDir); err != nil { + return Volume{}, err + } + if err := ctx.Err(); err != nil { + return Volume{}, err + } + if err := os.Link(tmp, path); err != nil { + return Volume{}, err + } + published = true + if err := os.Remove(tmp); err != nil { + return Volume{}, err + } + if err := fcSyncPath(d.cfg.VolumeDir); err != nil { + return Volume{}, err + } + committed = true + return Volume{ID: volume.ID, Name: volume.Name, SizeMiB: volume.SizeMiB}, nil +} + +// reserveRestore runs under volumeMu; no disk bytes are accepted until it wins. +func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, size int) (fcVolume, error) { + if err := ctx.Err(); err != nil { + return fcVolume{}, err + } + if len(d.volumeTransfers) >= MaxVolumeTransfers { + return fcVolume{}, ErrVolumeTransferBusy + } + reserved, err := d.reservedVolumeMiB(name) + if err != nil { + return fcVolume{}, err + } + volumes, err := d.readVolumes() + if err != nil { + return fcVolume{}, err + } + used := reserved + for _, volume := range volumes { + if volume.Name == name { + return fcVolume{}, ErrVolumeExists + } + used += volume.SizeMiB + } + if used+size > d.cfg.VolumeBudgetMiB { + return fcVolume{}, ErrVolumeBudget + } + free, err := d.host.FreeBytes(d.cfg.VolumeDir) + if err != nil { + return fcVolume{}, err + } + if free < uint64(d.cfg.DiskFloorMiB+size+reserved)<<20 { + return fcVolume{}, ErrVolumeBudget + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return fcVolume{}, err + } + volume := fcVolume{ID: "vol-" + hex.EncodeToString(random[:]), Name: name, SizeMiB: size} + if err := d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: name, SizeMiB: size}); err != nil { + return fcVolume{}, err + } + return volume, nil +} + +type volumeWriteError struct { + io.Writer + err error +} + +func (w *volumeWriteError) Write(p []byte) (int, error) { + n, err := w.Writer.Write(p) + if err != nil { + w.err = err + } + return n, err +} diff --git a/internal/vm/firecracker_backup_test.go b/internal/vm/firecracker_backup_test.go new file mode 100644 index 0000000..05552fb --- /dev/null +++ b/internal/vm/firecracker_backup_test.go @@ -0,0 +1,277 @@ +//go:build linux + +package vm + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func backupDriver(t *testing.T) (*FirecrackerDriver, *fakeFCHost, FirecrackerConfig) { + t.Helper() + return newTestFirecracker(t, func(c *FirecrackerConfig) { + c.VolumeBudgetMiB = 256 + c.VolumeMiB = 64 + c.VolumeDir = filepath.Join(c.Root, "volumes") + }) +} + +func TestFirecrackerColdBackupRoundTrip(t *testing.T) { + d, _, cfg := backupDriver(t) + ctx := context.Background() + original, err := d.CreateVolume(ctx, "original") + if err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(d.volumeDisk(original.ID), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + marker := []byte("preserve raw data, including credential-bearing and deleted blocks\x00\xff") + if _, err := file.WriteAt(marker, 12345); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + backup, err := d.OpenVolumeBackup(ctx, original.ID) + if err != nil { + t.Fatal(err) + } + defer backup.Data.Close() + if backup.Manifest.SourceVolumeID != original.ID || backup.Manifest.SizeBytes != 64<<20 || backup.Manifest.ApplicationConsistency != "not-asserted" { + t.Fatalf("manifest=%+v", backup.Manifest) + } + // A held backup excludes actual guest attachment, deletion and another backup. + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", original.ID)); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("attach during backup: %v", err) + } + if err := d.DeleteVolume(ctx, original.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("delete during backup: %v", err) + } + if _, err := d.OpenVolumeBackup(ctx, original.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("second backup: %v", err) + } + restored, err := d.RestoreVolume(ctx, "restored", backup.Manifest, backup.Data) + if err != nil { + t.Fatal(err) + } + if restored.ID == original.ID || restored.Name != "restored" || restored.SizeMiB != 64 { + t.Fatalf("restored identity=%+v", restored) + } + if err := backup.Data.Close(); err != nil { + t.Fatal(err) + } + check, err := d.OpenVolumeBackup(ctx, restored.ID) + if err != nil { + t.Fatal(err) + } + if check.Manifest.SHA256 != backup.Manifest.SHA256 { + t.Fatalf("restored checksum differs: %s != %s", check.Manifest.SHA256, backup.Manifest.SHA256) + } + _ = check.Data.Close() + restoredFile, err := os.Open(d.volumeDisk(restored.ID)) + if err != nil { + t.Fatal(err) + } + defer restoredFile.Close() + got := make([]byte, len(marker)) + _, err = restoredFile.ReadAt(got, 12345) + if err != nil || !bytes.Equal(got, marker) { + t.Fatalf("restored content: %q %v", got, err) + } + // Once the backup closes, the original can attach. Backup then rejects both + // the running ownership record and, independently, a leftover jail hard link. + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", original.ID)); err != nil { + t.Fatal(err) + } + if _, err := d.OpenVolumeBackup(ctx, original.ID); !errors.Is(err, ErrVolumeInUse) { + t.Fatalf("attached backup: %v", err) + } + if err := d.Destroy(ctx, fcTestID); err != nil { + t.Fatal(err) + } + link := filepath.Join(cfg.Root, "held-disk") + if err := os.Link(d.volumeDisk(original.ID), link); err != nil { + t.Fatal(err) + } + if _, err := d.OpenVolumeBackup(ctx, original.ID); !errors.Is(err, ErrVolumeInUse) { + t.Fatalf("hard-linked backup: %v", err) + } +} + +func TestFirecrackerRestoreFailureCleanup(t *testing.T) { + d, _, cfg := backupDriver(t) + ctx := context.Background() + source, err := d.CreateVolume(ctx, "source") + if err != nil { + t.Fatal(err) + } + backup, err := d.OpenVolumeBackup(ctx, source.ID) + if err != nil { + t.Fatal(err) + } + manifest := backup.Manifest + _ = backup.Data.Close() + for _, kind := range []string{"truncated", "digest", "extra", "cancelled"} { + t.Run(kind, func(t *testing.T) { + sourceFile, err := os.Open(d.volumeDisk(source.ID)) + if err != nil { + t.Fatal(err) + } + defer sourceFile.Close() + var reader io.Reader = sourceFile + m := manifest + child, cancel := context.WithCancel(ctx) + defer cancel() + switch kind { + case "truncated": + reader = io.LimitReader(sourceFile, 1024) + case "digest": + m.SHA256 = strings.Repeat("f", 64) + case "extra": + reader = io.MultiReader(sourceFile, strings.NewReader("extra")) + case "cancelled": + reader = &cancelVolumeReader{Reader: sourceFile, cancel: cancel} + } + _, err = d.RestoreVolume(child, "failure", m, reader) + if kind == "cancelled" { + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancel: %v", err) + } + } else if !errors.Is(err, ErrVolumeBackupInvalid) { + t.Fatalf("%s: %v", kind, err) + } + volumes, err := d.Volumes(ctx) + if err != nil { + t.Fatal(err) + } + if len(volumes) != 1 || volumes[0].ID != source.ID { + t.Fatalf("failed restore published volume: %+v", volumes) + } + names, err := os.ReadDir(cfg.VolumeDir) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if name.Name() != source.ID+".json" && name.Name() != source.ID+".ext4" { + t.Fatalf("failed restore left %s", name.Name()) + } + } + }) + } + // Failed imports release the name and budget, and a subsequent valid import succeeds. + sourceFile, err := os.Open(d.volumeDisk(source.ID)) + if err != nil { + t.Fatal(err) + } + defer sourceFile.Close() + if _, err := d.RestoreVolume(ctx, "failure", manifest, sourceFile); err != nil { + t.Fatal(err) + } +} + +type cancelVolumeReader struct { + io.Reader + cancel context.CancelFunc +} + +func (r *cancelVolumeReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + r.cancel() + return n, err +} + +type pausedVolumeReader struct { + io.Reader + entered chan struct{} + resume chan struct{} + once bool +} + +func (r *pausedVolumeReader) Read(p []byte) (int, error) { + if !r.once { + r.once = true + close(r.entered) + <-r.resume + } + return r.Reader.Read(p) +} + +func TestFirecrackerRestoreReservesQuotaAndName(t *testing.T) { + d, _, _ := backupDriver(t) + d.cfg.VolumeBudgetMiB = 128 + ctx := context.Background() + source, err := d.CreateVolume(ctx, "source") + if err != nil { + t.Fatal(err) + } + backup, err := d.OpenVolumeBackup(ctx, source.ID) + if err != nil { + t.Fatal(err) + } + manifest := backup.Manifest + _ = backup.Data.Close() + sourceFile, err := os.Open(d.volumeDisk(source.ID)) + if err != nil { + t.Fatal(err) + } + defer sourceFile.Close() + reader := &pausedVolumeReader{Reader: sourceFile, entered: make(chan struct{}), resume: make(chan struct{})} + result := make(chan error, 1) + go func() { _, err := d.RestoreVolume(ctx, "incoming", manifest, reader); result <- err }() + <-reader.entered + // The full reservation counts even while the transfer is paused outside volumeMu. + _, sameName := d.CreateVolume(ctx, "incoming") + _, overBudget := d.CreateVolume(ctx, "other") + _, secondImport := d.RestoreVolume(ctx, "second", manifest, strings.NewReader("")) + // A different existing volume can still be read/listed/removed without waiting. + volumes, listErr := d.Volumes(ctx) + deleteErr := d.DeleteVolume(ctx, source.ID) + close(reader.resume) + restoreErr := <-result + if !errors.Is(sameName, ErrVolumeExists) || !errors.Is(overBudget, ErrVolumeBudget) || !errors.Is(secondImport, ErrVolumeBudget) { + t.Fatalf("reservations: name=%v create=%v restore=%v", sameName, overBudget, secondImport) + } + if listErr != nil || len(volumes) != 1 || deleteErr != nil || restoreErr != nil { + t.Fatalf("independent progress: list=%v delete=%v restore=%v", listErr, deleteErr, restoreErr) + } +} + +func TestVolumeBackupEnvelopeBounds(t *testing.T) { + digest := sha256.Sum256([]byte("manifest only")) + m := VolumeManifest{SourceVolumeID: "vol-" + strings.Repeat("a", 32), SizeBytes: 64 << 20, SHA256: hex.EncodeToString(digest[:]), ApplicationConsistency: "not-asserted"} + var valid bytes.Buffer + if err := WriteVolumeManifest(&valid, m); err != nil { + t.Fatal(err) + } + raw := valid.Bytes() + for _, kind := range []string{"version", "oversize", "truncated", "metadata", "size"} { + t.Run(kind, func(t *testing.T) { + data := append([]byte(nil), raw...) + switch kind { + case "version": + data[7] = '2' + case "oversize": + data[8] = 1 + case "truncated": + data = data[:len(data)-1] + case "metadata": + data = bytes.Replace(data, []byte("not-asserted"), []byte("app-verified"), 1) + case "size": + data = bytes.Replace(data, []byte("67108864"), []byte("67108865"), 1) + } + if _, err := ReadVolumeManifest(bytes.NewReader(data)); !errors.Is(err, ErrVolumeBackupInvalid) { + t.Fatalf("%s accepted: %v", kind, err) + } + }) + } +} diff --git a/internal/vm/firecracker_instance.go b/internal/vm/firecracker_instance.go index 687e747..f97f76d 100644 --- a/internal/vm/firecracker_instance.go +++ b/internal/vm/firecracker_instance.go @@ -154,6 +154,12 @@ func (d *FirecrackerDriver) Claim() error { _ = file.Close() return fmt.Errorf("write Firecracker claim %s: %w", path, err) } + if d.cfg.VolumeBudgetMiB > 0 { + if err := d.recoverVolumes(); err != nil { + _ = file.Close() + return fmt.Errorf("recover claimed volume storage: %w", err) + } + } d.claim = file return nil } diff --git a/internal/vm/firecracker_test.go b/internal/vm/firecracker_test.go index 8d0275c..b4a23e5 100644 --- a/internal/vm/firecracker_test.go +++ b/internal/vm/firecracker_test.go @@ -1569,10 +1569,15 @@ func TestFirecrackerVolumeCrashLeftovers(t *testing.T) { t.Fatal(err) } } + cfg.ClaimDir = filepath.Join(t.TempDir(), "claims") d, err = newFirecrackerDriver(cfg, newFakeFCHost(t)) if err != nil { t.Fatal(err) } + if err := d.Claim(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.releaseClaim() }) for _, path := range leftovers { if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { t.Errorf("%s remains: %v", path, err) @@ -1611,9 +1616,15 @@ func TestFirecrackerVolumeStartupRepairs(t *testing.T) { t.Fatal(err) } host := newFakeFCHost(t) - if _, err := newFirecrackerDriver(cfg, host); err != nil { + cfg.ClaimDir = filepath.Join(t.TempDir(), "claims") + restarted, err := newFirecrackerDriver(cfg, host) + if err != nil { + t.Fatal(err) + } + if err := restarted.Claim(); err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = restarted.releaseClaim() }) if uid, ok := host.owner(disk); !ok || uid != 0 { t.Errorf("detached disk owner = %d, %t; want root", uid, ok) } @@ -1866,3 +1877,40 @@ func TestFirecrackerBalloon(t *testing.T) { } } } + +// A second daemon must fail its claim before any destructive recovery; a +// temporary import disk is live work until the owning daemon relinquishes it. +func TestFirecrackerVolumeRecoveryRequiresClaim(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + cfg.ClaimDir = filepath.Join(t.TempDir(), "claims") + d.cfg.ClaimDir = cfg.ClaimDir + if err := d.Claim(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.releaseClaim() }) + staging := filepath.Join(cfg.VolumeDir, "vol-"+strings.Repeat("c", 32)+".ext4.tmp") + payload := []byte("live incomplete restore") + if err := os.WriteFile(staging, payload, 0o600); err != nil { + t.Fatal(err) + } + restarted, err := newFirecrackerDriver(cfg, host) + if err != nil { + t.Fatal(err) + } + if err := restarted.Claim(); err == nil { + t.Fatal("second daemon stole active storage ownership") + } + if got, err := os.ReadFile(staging); err != nil || !bytes.Equal(got, payload) { + t.Fatalf("unclaimed constructor erased active restore: %q %v", got, err) + } + if err := d.releaseClaim(); err != nil { + t.Fatal(err) + } + if err := restarted.Claim(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = restarted.releaseClaim() }) + if _, err := os.Stat(staging); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("claimed restart did not recover incomplete restore: %v", err) + } +} diff --git a/internal/vm/firecracker_volumes.go b/internal/vm/firecracker_volumes.go index 6af7170..1b42c6c 100644 --- a/internal/vm/firecracker_volumes.go +++ b/internal/vm/firecracker_volumes.go @@ -50,8 +50,8 @@ func (d *FirecrackerDriver) volumeDisk(id string) string { return filepath.Join(d.cfg.VolumeDir, id+".ext4") } -// initVolumes validates the volume configuration, creates VolumeDir and -// removes what a crash during CreateVolume left behind. +// initVolumes validates configuration and creates VolumeDir. Destructive +// recovery waits until Claim excludes another daemon with an active import. func (d *FirecrackerDriver) initVolumes() error { cfg := d.cfg if !cleanAbs(cfg.VolumeDir) { @@ -103,6 +103,13 @@ func (d *FirecrackerDriver) initVolumes() error { if dir.Sys().(*syscall.Stat_t).Dev != root.Sys().(*syscall.Stat_t).Dev { return fmt.Errorf("Firecracker volume directory %s must be on the filesystem of %s", cfg.VolumeDir, cfg.Root) } + return nil +} + +// recoverVolumes runs only after exclusive instance ownership is established. +// A competing constructor must not erase a live daemon's transfer staging. +func (d *FirecrackerDriver) recoverVolumes() error { + cfg := d.cfg names, err := readDirNames(cfg.VolumeDir) if err != nil { return err @@ -231,6 +238,9 @@ func volumeHolders(vms []fcVM) map[string]string { // volumeInUse reports whether a VM record or a jail's hard link holds the // volume. Callers hold volumeMu. func (d *FirecrackerDriver) volumeInUse(id string) error { + if _, busy := d.volumeTransfers[id]; busy { + return ErrVolumeTransferBusy + } vms, err := d.inventory() if err != nil { return err @@ -262,7 +272,11 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - used := 0 + reserved, err := d.reservedVolumeMiB(name) + if err != nil { + return Volume{}, err + } + used := reserved for _, volume := range volumes { if volume.Name == name { return Volume{}, fmt.Errorf("%w: %q", ErrVolumeExists, name) @@ -277,7 +291,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - if need := uint64(d.cfg.DiskFloorMiB+size) << 20; free < need { + if need := uint64(d.cfg.DiskFloorMiB+size+reserved) << 20; free < need { return Volume{}, fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB volume needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, size) } diff --git a/internal/vm/volume_backup.go b/internal/vm/volume_backup.go new file mode 100644 index 0000000..a018fc4 --- /dev/null +++ b/internal/vm/volume_backup.go @@ -0,0 +1,137 @@ +package vm + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" +) + +const ( + VolumeBackupMediaType = "application/vnd.sandboxd.volume-backup" + VolumeTransferTimeout = 10 * time.Minute + MaxVolumeTransfers = 2 + volumeBackupMagic = "SBXVOL01" + maxVolumeManifest = 4096 + volumeTransferBuffer = 64 << 10 +) + +var ( + ErrVolumeTransferBusy = errors.New("volume storage operation in progress") + ErrVolumeBackupInvalid = errors.New("invalid volume backup") +) + +// VolumeManifest describes raw, credential-bearing block data, including deleted +// blocks. Its hash detects corruption, not authenticity. No application-consistency +// guarantee follows from a detached disk or the guest's best-effort flush. +type VolumeManifest struct { + SourceVolumeID string `json:"sourceVolumeID"` + SizeBytes int64 `json:"sizeBytes"` + SHA256 string `json:"sha256"` + ApplicationConsistency string `json:"applicationConsistency"` +} + +func (m VolumeManifest) Validate() error { + digest, err := hex.DecodeString(m.SHA256) + if !ValidVolumeID(m.SourceVolumeID) || m.SizeBytes < 64<<20 || m.SizeBytes > 1<<40 || m.SizeBytes%(1<<20) != 0 || err != nil || len(digest) != 32 || m.SHA256 != strings.ToLower(m.SHA256) || m.ApplicationConsistency != "not-asserted" { + return ErrVolumeBackupInvalid + } + return nil +} + +// VolumeBackup holds an exclusive reservation until Data.Close, even after EOF. +// Consumers must close it on every success/error path. Data is exactly SizeBytes. +type VolumeBackup struct { + Manifest VolumeManifest + Data io.ReadCloser +} + +// VolumeBackups is the local native block-storage capability. It deliberately +// does not grant a caller arbitrary filesystem paths or an in-place restore. +type VolumeBackups interface { + OpenVolumeBackup(context.Context, string) (VolumeBackup, error) + RestoreVolume(context.Context, string, VolumeManifest, io.Reader) (Volume, error) +} + +func ReadVolumeManifest(r io.Reader) (VolumeManifest, error) { + var m VolumeManifest + var header [12]byte + if _, err := io.ReadFull(r, header[:]); err != nil { + return m, fmt.Errorf("%w: header: %v", ErrVolumeBackupInvalid, err) + } + length := binary.BigEndian.Uint32(header[8:]) + if string(header[:8]) != volumeBackupMagic || length == 0 || length > maxVolumeManifest { + return m, ErrVolumeBackupInvalid + } + data := make([]byte, int(length)) + if _, err := io.ReadFull(r, data); err != nil { + return m, fmt.Errorf("%w: manifest: %v", ErrVolumeBackupInvalid, err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&m); err != nil { + return m, ErrVolumeBackupInvalid + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return m, ErrVolumeBackupInvalid + } + return m, m.Validate() +} + +func WriteVolumeManifest(w io.Writer, m VolumeManifest) error { + if err := m.Validate(); err != nil { + return err + } + data, err := json.Marshal(m) + if err != nil { + return err + } + var header [12]byte + copy(header[:8], volumeBackupMagic) + binary.BigEndian.PutUint32(header[8:], uint32(len(data))) + if err := writeVolumeBytes(w, header[:]); err != nil { + return err + } + return writeVolumeBytes(w, data) +} + +func writeVolumeBytes(w io.Writer, p []byte) error { + for len(p) > 0 { + n, err := w.Write(p) + if err != nil { + return err + } + if n <= 0 { + return io.ErrShortWrite + } + p = p[n:] + } + return nil +} + +// copyVolumeBytes never allocates in proportion to the disk, bypasses no context +// checks through WriterTo/ReaderFrom, and distinguishes truncation from success. +func copyVolumeBytes(ctx context.Context, dst io.Writer, src io.Reader, size int64) error { + buffer := make([]byte, volumeTransferBuffer) + for size > 0 { + if err := ctx.Err(); err != nil { + return err + } + n, err := io.ReadFull(src, buffer[:min(int64(len(buffer)), size)]) + if err != nil { + return err + } + if err := writeVolumeBytes(dst, buffer[:n]); err != nil { + return err + } + size -= int64(n) + } + return ctx.Err() +}