From c36d1541ec7ef1b0f3d37b0265dbc595e40b446b Mon Sep 17 00:00:00 2001 From: jerryfane Date: Wed, 4 Nov 2026 07:55:52 +0100 Subject: [PATCH 1/4] Grow detached Firecracker volumes: POST /volumes/{id}/resize --- docs/compatibility.md | 1 + docs/firecracker.md | 32 +++ internal/control/volumes.go | 50 +++- internal/control/volumes_test.go | 71 ++++++ internal/vm/firecracker.go | 3 + internal/vm/firecracker_host.go | 15 ++ internal/vm/firecracker_test.go | 22 ++ internal/vm/firecracker_volume_resize.go | 188 +++++++++++++++ internal/vm/firecracker_volume_resize_test.go | 219 ++++++++++++++++++ internal/vm/firecracker_volumes.go | 2 +- internal/vm/volume.go | 12 + 11 files changed, 613 insertions(+), 2 deletions(-) create mode 100644 internal/vm/firecracker_volume_resize.go create mode 100644 internal/vm/firecracker_volume_resize_test.go diff --git a/docs/compatibility.md b/docs/compatibility.md index 80624a6..d1d8d38 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,6 +31,7 @@ Volumes are off by default. They stay off on the Apple (Mac) driver and on a gat - `POST /volumes` `{"name"}` creates one: `201` `{"volumeID","name","token"}`. A name is 1–63 letters, digits, `_` or `-`, starting with a letter or digit (else `400`); a taken name is `409`; a volume that would take the sum of all volume sizes over `-fc-volume-budget-mib` is `400` (`volume budget exhausted`). Unknown body fields are `400`. - `GET /volumes` lists `[{"volumeID","name"}]` by name; `GET /volumes/{id}` (the SDKs' `Volume.connect`, `get_info`) answers `{"volumeID","name","token"}` or `404`. The `token` is an HMAC of the volume ID under the token secret, and grants nothing: the volume content API it would authorize is refused. +- `POST /volumes/{id}/resize` `{"sizeMiB": N}` is a sandboxd operator route outside the E2B API (control API key): on the Firecracker driver it grows a detached volume to N MiB and answers `200 {"volumeID","name","sizeMiB"}`; it never shrinks (`400`), and an attached or busy volume is `409` (see [firecracker.md](firecracker.md#growing-a-detached-volume)). - `DELETE /volumes/{id}` is `204` and deletes the data; an unknown ID is `404`, and a volume attached to a sandbox is `409` (kill the sandbox first). - `/volumecontent/…` (the SDKs' volume file methods: `list`, `read_file`, `write_file`, …) answers `501` `the volume content API is not supported: mount the volume in a sandbox (volumeMounts) and use its files there`. - `POST /v2/sandboxes` with `"volumeMounts":[{"name","path"}]` mounts the named volume at `path` (absolute, clean, not `/`; else `400`), owned by `user`. At most one mount per sandbox (`400` otherwise); absent, `null` or `[]` mounts nothing. An unknown name is `404`; a volume attached to another sandbox is `409` `volume "" is attached to sandbox `, refused before any capacity is reserved (two creates racing for one volume: the driver admits one, the other is `409` too). A volume is attached to at most one sandbox at a time, until that sandbox ends. diff --git a/docs/firecracker.md b/docs/firecracker.md index 349713e..21c86d3 100644 --- a/docs/firecracker.md +++ b/docs/firecracker.md @@ -232,6 +232,38 @@ lost there; the volume's journal keeps it consistent. A volume survives its VM's end, a daemon restart or `kill -9` and that destroy; only `DELETE /volumes/{id}` removes it. +### Growing a detached volume + +`POST /volumes/{id}/resize` with `{"sizeMiB": N}` (control API key, like the +backup route; sandbox envd tokens cannot reach it) grows a **detached** volume +in place and answers `200 {"volumeID", "name", "sizeMiB"}`. It never shrinks: +a smaller size is `400`, the same size is a no-op `200`. An attached, +backing-up, restoring or already-growing volume is `409`; an unknown ID `404`; +a grow past `-fc-volume-budget-mib` or below the disk floor (the growth, not +the whole size, counts) is `400`. The new size must be between 64 MiB and +1 TiB. Drivers without the capability answer `501`. Use it to raise an +existing volume after `-fc-volume-mib` was raised, which only affects new +volumes. + +Steps, with the volume held (no attach, delete or backup) throughout: check +budget and floor, extend the disk file, `e2fsck -f -p`, `resize2fs`, then +replace the record with one atomic rename. The host uses its own trusted +`e2fsck` and `resize2fs` (root-owned regular files under `/usr/sbin` or +`/sbin`, found when a grow runs; `e2fsprogs` ships them with `mke2fs`). Nothing is mounted. If `e2fsck` cannot +repair the filesystem safely (exit 2 or higher, e.g. `4`) the file goes back to its +old size and the volume is unchanged. If `resize2fs` fails, or the daemon +stops mid-grow, the disk file is larger than its record until the same +request is repeated: the steps are repeatable, and a disk file larger than the +requested size is refused rather than cut. Until then the volume can still +be attached (the guest sees a larger device holding the old filesystem) but a +backup of it is refused. + +Trust: `e2fsck` and `resize2fs` run as root on a filesystem the guest wrote, +which is the one place the host parses guest-controlled metadata (`mke2fs` +only writes a fresh one). They take an image file, not a mount, and run with +the daemon's minimal environment. An operator who does not accept that parsing +should not call the route. + ### Controller-only cold backup and new-volume restore These operations use the control `X-API-Key`, not a guest token, and are available diff --git a/internal/control/volumes.go b/internal/control/volumes.go index ec33d35..8a5efa3 100644 --- a/internal/control/volumes.go +++ b/internal/control/volumes.go @@ -33,6 +33,17 @@ type newVolumeRequest struct { Name string `json:"name"` } +// resizeVolumeRequest asks to grow a detached volume to SizeMiB. +type resizeVolumeRequest struct { + SizeMiB int `json:"sizeMiB"` +} + +// resizedVolume is the grow response: the volume with its new size. +type resizedVolume struct { + volumeEntry + SizeMiB int `json:"sizeMiB"` +} + // volumeToken is the token the SDKs require with a volume. It grants // nothing: the volume content API it would authorize is refused, and a // volume's data is reached only from a sandbox it is mounted in. @@ -47,7 +58,7 @@ func (s *Service) withToken(volume vm.Volume) volumeAndToken { // serveVolumes answers E2B's volume API when volumes are configured, and // reports whether path was a volume route. func (s *Service) serveVolumes(w http.ResponseWriter, r *http.Request) bool { - if s.serveVolumeTransfers(w, r) { + if s.serveVolumeTransfers(w, r) || s.serveVolumeResize(w, r) { return true } path := r.URL.Path @@ -96,6 +107,8 @@ func (s *Service) volumeFailure(w http.ResponseWriter, action string, err error) s.fail(w, ProfileE2B, http.StatusConflict, "volume is attached to a sandbox: kill the sandbox first") case errors.Is(err, vm.ErrVolumeTransferBusy): s.fail(w, ProfileE2B, http.StatusConflict, "volume storage operation in progress") + case errors.Is(err, vm.ErrVolumeSize): + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume size: it can only grow, between 64 MiB and 1 TiB") case errors.Is(err, vm.ErrVolumeBackupInvalid): s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume backup") case errors.Is(err, vm.ErrVolumeBudget): @@ -216,3 +229,38 @@ func (s *Service) volumeMount(ctx context.Context, raw json.RawMessage, hardened } return nil, http.StatusNotFound, fmt.Sprintf("volume %q not found", requested.Name) } + +// serveVolumeResize answers POST /volumes/{id}/resize, an operator route +// behind the control API key like backup: it grows a detached volume and never +// shrinks one. Customers, who hold only a sandbox's envd token, cannot reach it. +func (s *Service) serveVolumeResize(w http.ResponseWriter, r *http.Request) bool { + id, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/volumes/"), "/resize") + if !ok || !strings.HasPrefix(r.URL.Path, "/volumes/") { + return false + } + if r.Method != http.MethodPost { + s.fail(w, ProfileE2B, http.StatusMethodNotAllowed, "method not allowed") + return true + } + resizer, ok := s.volumes.(vm.VolumeResizer) + if !ok { + s.fail(w, ProfileE2B, http.StatusNotImplemented, "volume grow unavailable") + return true + } + if !vm.ValidVolumeID(id) { + s.fail(w, ProfileE2B, http.StatusNotFound, "volume not found") + return true + } + var request resizeVolumeRequest + if err := decode(w, r, &request); !errors.Is(err, io.EOF) { + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume resize body") + return true + } + volume, err := resizer.ResizeVolume(r.Context(), id, request.SizeMiB) + if err != nil { + s.volumeFailure(w, "resize", err) + return true + } + jsonResponse(w, http.StatusOK, resizedVolume{volumeEntry{volume.ID, volume.Name}, volume.SizeMiB}) + return true +} diff --git a/internal/control/volumes_test.go b/internal/control/volumes_test.go index f79489e..4ab6a49 100644 --- a/internal/control/volumes_test.go +++ b/internal/control/volumes_test.go @@ -75,6 +75,32 @@ func (f *fakeVolumes) DeleteVolume(_ context.Context, id string) error { return fmt.Errorf("volume %s: %w", id, vm.ErrVolumeNotFound) } +// ResizeVolume grows like the driver: only larger, only detached. +func (f *fakeVolumes) ResizeVolume(_ context.Context, id string, sizeMiB int) (vm.Volume, error) { + f.mu.Lock() + defer f.mu.Unlock() + for i := range f.volumes { + volume := &f.volumes[i] + if volume.ID != id { + continue + } + switch { + case volume.AttachedTo != "": + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeInUse) + case sizeMiB < volume.SizeMiB: + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeSize) + } + volume.SizeMiB = sizeMiB + return *volume, nil + } + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeNotFound) +} + +// plainVolumes is a store that cannot grow volumes. +type plainVolumes struct{ *fakeVolumes } + +func (plainVolumes) ResizeVolume() {} + func (f *fakeVolumes) attach(name, sandbox string) { f.mu.Lock() defer f.mu.Unlock() @@ -396,3 +422,48 @@ func TestVolumesNeedLocalDriver(t *testing.T) { t.Fatal("volumes accepted without a local driver") } } + +func TestVolumeGrowAPI(t *testing.T) { + s, _, volumes := openVolumes(t, 1) + data := createVolume(t, s, "data") + path := "/volumes/" + data.ID + "/resize" + + got := request(t, s, http.MethodPost, path, map[string]any{"sizeMiB": 1024}) + if want := fmt.Sprintf(`{"volumeID":%q,"name":"data","sizeMiB":1024}`+"\n", data.ID); got.Code != http.StatusOK || got.Body.String() != want { + t.Fatalf("grow: %d %s, want %s", got.Code, got.Body.String(), want) + } + if list, _ := volumes.Volumes(context.Background()); list[0].SizeMiB != 1024 { + t.Fatalf("driver size = %d", list[0].SizeMiB) + } + volumes.attach("data", "sbx-1") + for _, c := range []struct { + label, method, path string + body any + status int + message string + }{ + {"attached", http.MethodPost, path, map[string]any{"sizeMiB": 2048}, http.StatusConflict, "attached to a sandbox"}, + {"unknown volume", http.MethodPost, "/volumes/vol-ffffffffffffffffffffffffffffffff/resize", map[string]any{"sizeMiB": 2048}, http.StatusNotFound, "volume not found"}, + {"malformed id", http.MethodPost, "/volumes/data/resize", map[string]any{"sizeMiB": 2048}, http.StatusNotFound, "volume not found"}, + {"wrong method", http.MethodGet, path, nil, http.StatusMethodNotAllowed, "method not allowed"}, + {"unknown field", http.MethodPost, path, map[string]any{"sizeMiB": 2048, "force": true}, http.StatusBadRequest, "invalid volume resize body"}, + } { + wantFailure(t, c.label, request(t, s, c.method, c.path, c.body), c.status, c.message) + } + volumes.mu.Lock() + volumes.volumes[0].AttachedTo = "" + volumes.mu.Unlock() + wantFailure(t, "shrink", request(t, s, http.MethodPost, path, map[string]any{"sizeMiB": 512}), http.StatusBadRequest, "can only grow") + // A missing size is size 0, which the driver refuses; it is never a no-op. + wantFailure(t, "no body", request(t, s, http.MethodPost, path, nil), http.StatusBadRequest, "can only grow") +} + +func TestVolumeGrowNeedsDriverSupport(t *testing.T) { + driver := &specDriver{fakeDriver: &fakeDriver{instances: map[string]vm.Instance{}}} + cfg := mixedConfig(e2bBase, 1) + cfg.Volumes = plainVolumes{&fakeVolumes{}} + s := openAt(t, filepath.Join(t.TempDir(), "ledger.sqlite"), driver, cfg) + created := createVolume(t, s, "data") + wantFailure(t, "unsupported", request(t, s, http.MethodPost, "/volumes/"+created.ID+"/resize", map[string]any{"sizeMiB": 1024}), + http.StatusNotImplemented, "grow unavailable") +} diff --git a/internal/vm/firecracker.go b/internal/vm/firecracker.go index a2c14b4..29abc8b 100644 --- a/internal/vm/firecracker.go +++ b/internal/vm/firecracker.go @@ -176,6 +176,9 @@ type fcHost interface { // FormatVolume runs the trusted mkfs binary on the volume image file // path; nothing is mounted on the host. FormatVolume(ctx context.Context, mkfs, path string) error + // ResizeVolume runs the trusted e2fsck (-f -p) and resize2fs on a + // volume image file already extended to its new size. Nothing is mounted. + ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string) error ApplyFirewall(ctx context.Context, ruleset string) error FirewallState(ctx context.Context) (string, error) diff --git a/internal/vm/firecracker_host.go b/internal/vm/firecracker_host.go index d724220..6a047ca 100644 --- a/internal/vm/firecracker_host.go +++ b/internal/vm/firecracker_host.go @@ -136,6 +136,21 @@ func (h *linuxFCHost) FormatVolume(ctx context.Context, mkfs, path string) error return err } +// ResizeVolume checks the volume's filesystem, then grows it to fill the +// image file. e2fsck exit 1 means it repaired errors, which -p only does when +// that is safe; anything else stops the grow before resize2fs, with +// errVolumeFsck so the caller knows the filesystem was not resized. +func (h *linuxFCHost) ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string) error { + if _, err := h.run(ctx, nil, e2fsck, "-f", "-p", path); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 { + return fmt.Errorf("%w: %w", errVolumeFsck, err) + } + } + _, err := h.run(ctx, nil, resize2fs, path) + return err +} + // MonitorRuleset runs `nft monitor` in the daemon's (host) namespace. Every // line it prints is one committed ruleset change; the content does not // matter, since the caller re-reads the table itself. The subprocess dies diff --git a/internal/vm/firecracker_test.go b/internal/vm/firecracker_test.go index b4a23e5..6326145 100644 --- a/internal/vm/firecracker_test.go +++ b/internal/vm/firecracker_test.go @@ -92,6 +92,11 @@ type fakeFCHost struct { // FormatVolume target. owners map[string]int formatted []string + // resized records each ResizeVolume call; resizeGate blocks it until + // closed and resizeErr is what it returns. + resized []string + resizeGate chan struct{} + resizeErr error // tables is each namespace's current table: the one it was created with, // or the last it was replaced with; // failReplace fails the next replacements, and readback, when set, @@ -154,6 +159,23 @@ func (h *fakeFCHost) FormatVolume(_ context.Context, mkfs, path string) error { h.formatted = append(h.formatted, mkfs+" "+path) return nil } + +// ResizeVolume stands in for e2fsck and resize2fs: it records the disk size it +// was handed, optionally blocks on resizeGate, and fails with resizeErr. +func (h *fakeFCHost) ResizeVolume(_ context.Context, e2fsck, resize2fs, path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + h.mu.Lock() + h.resized = append(h.resized, fmt.Sprintf("%s %s %s %d", e2fsck, resize2fs, path, info.Size())) + gate, fail := h.resizeGate, h.resizeErr + h.mu.Unlock() + if gate != nil { + <-gate + } + return fail +} func (h *fakeFCHost) ApplyFirewall(_ context.Context, ruleset string) error { h.mu.Lock() defer h.mu.Unlock() diff --git a/internal/vm/firecracker_volume_resize.go b/internal/vm/firecracker_volume_resize.go new file mode 100644 index 0000000..4eb795d --- /dev/null +++ b/internal/vm/firecracker_volume_resize.go @@ -0,0 +1,188 @@ +//go:build linux + +package vm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "syscall" +) + +// fcE2fsck and fcResize2fs are the host e2fsprogs candidates for a grow. They +// are resolved when a grow runs, not at startup, so a worker that never grows +// a volume needs nothing beyond mke2fs. +var ( + fcE2fsck = []string{"/usr/sbin/e2fsck", "/sbin/e2fsck"} + fcResize2fs = []string{"/usr/sbin/resize2fs", "/sbin/resize2fs"} +) + +// errVolumeFsck marks a host ResizeVolume that stopped at e2fsck: the +// filesystem was not resized, so the disk file can go back to its old size. +var errVolumeFsck = errors.New("volume filesystem check failed") + +func (d *FirecrackerDriver) trustedTool(candidates []string) (string, error) { + var untrusted error + for _, tool := range candidates { + err := d.host.TrustedFile(tool, true) + if err == nil { + return tool, nil + } + untrusted = errors.Join(untrusted, err) + } + return "", fmt.Errorf("Firecracker volume grow needs a trusted host tool: %w", untrusted) +} + +// ResizeVolume grows a detached volume to sizeMiB. The volume stays reserved +// (no attach, delete or backup) from the budget check until the new size is +// recorded. Order: extend the disk file, e2fsck -f -p, resize2fs, then replace +// the record. A failed e2fsck puts the file back. A crash, or a failed +// resize2fs, leaves a disk larger than its record; the same grow can be +// retried, because each step is idempotent and a disk larger than the request +// is refused. The host tools read a +// filesystem the guest wrote; nothing is mounted. +func (d *FirecrackerDriver) ResizeVolume(ctx context.Context, id string, sizeMiB int) (Volume, error) { + if err := d.volumesOn(); err != nil { + return Volume{}, err + } + if sizeMiB < 64 || sizeMiB > 1<<20 { + return Volume{}, fmt.Errorf("%w: %d MiB is outside 64 MiB to 1 TiB", ErrVolumeSize, sizeMiB) + } + ctx, cancel := context.WithTimeout(ctx, VolumeTransferTimeout) + defer cancel() + + d.volumeMu.Lock() + volume, err := d.readVolume(id) + if err == nil { + err = d.volumeInUse(id) + } + switch { + case err != nil: + case sizeMiB == volume.SizeMiB: + d.volumeMu.Unlock() + return Volume{ID: volume.ID, Name: volume.Name, SizeMiB: volume.SizeMiB}, nil + case sizeMiB < volume.SizeMiB: + err = fmt.Errorf("%w: a volume cannot shrink from %d to %d MiB", ErrVolumeSize, volume.SizeMiB, sizeMiB) + } + var e2fsck, resize2fs string + if err == nil { + if e2fsck, err = d.trustedTool(fcE2fsck); err == nil { + resize2fs, err = d.trustedTool(fcResize2fs) + } + } + growth := sizeMiB - volume.SizeMiB + if err == nil { + err = d.reserveGrowth(volume, growth) + } + d.volumeMu.Unlock() + if err != nil { + return Volume{}, err + } + defer d.releaseVolumeTransfer(id) + + if err := d.growDisk(ctx, id, volume.SizeMiB, sizeMiB, e2fsck, resize2fs); err != nil { + return Volume{}, err + } + volume.SizeMiB = sizeMiB + d.volumeMu.Lock() + defer d.volumeMu.Unlock() + if err := d.replaceVolumeRecord(volume); err != nil { + return Volume{}, err + } + return Volume{ID: volume.ID, Name: volume.Name, SizeMiB: volume.SizeMiB}, nil +} + +// reserveGrowth checks the volume budget and the host disk floor for growth +// MiB more, then reserves it like an in-flight restore, so a concurrent create +// counts it. Callers hold volumeMu. +func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { + volumes, err := d.readVolumes() + if err != nil { + return err + } + reserved, err := d.reservedVolumeMiB(volume.Name) + if err != nil { + return err + } + used := reserved + for _, existing := range volumes { + used += existing.SizeMiB + } + if used+growth > d.cfg.VolumeBudgetMiB { + return fmt.Errorf("%w: %d of %d MiB used, growing %s takes %d MiB more", ErrVolumeBudget, used, d.cfg.VolumeBudgetMiB, volume.ID, growth) + } + free, err := d.host.FreeBytes(d.cfg.VolumeDir) + if err != nil { + return err + } + if need := uint64(d.cfg.DiskFloorMiB+growth+reserved) << 20; free < need { + return fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB growth needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, growth) + } + return d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: volume.Name, SizeMiB: growth}) +} + +// growDisk extends the detached disk file and its filesystem. A disk larger +// than the target is not a state this grow made: it is refused, not cut. +func (d *FirecrackerDriver) growDisk(ctx context.Context, id string, oldMiB, newMiB int, e2fsck, resize2fs string) error { + fd, err := syscall.Open(d.volumeDisk(id), syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return err + } + file := os.NewFile(uintptr(fd), "volume-grow") + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || stat.Nlink != 1 || stat.Uid != uint32(os.Geteuid()) { + return errors.New("volume disk is not a detached native disk") + } + size, target := int64(oldMiB)<<20, int64(newMiB)<<20 + if info.Size() < size || info.Size() > target { + return fmt.Errorf("%w: disk is %d bytes, expected %d to %d", ErrVolumeSize, info.Size(), size, target) + } + extended := info.Size() < target + if extended { + if err := file.Truncate(target); err != nil { + return err + } + } + if err := file.Sync(); err != nil { + return err + } + err = d.host.ResizeVolume(ctx, e2fsck, resize2fs, d.volumeDisk(id)) + if errors.Is(err, errVolumeFsck) && extended { + // Nothing was resized: put the file back, so the volume still matches + // its record and can be backed up or attached. + err = errors.Join(err, file.Truncate(info.Size()), file.Sync()) + } + return err +} + +// replaceVolumeRecord swaps the record in with a rename, so a reader or a +// crash sees the old size or the new one, never a partial record. Callers +// hold volumeMu. +func (d *FirecrackerDriver) replaceVolumeRecord(volume fcVolume) error { + data, err := json.Marshal(volume) + if err != nil { + return err + } + record := d.volumeRecord(volume.ID) + tmp := record + ".tmp" + file, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return err + } + _, err = file.Write(data) + err = errors.Join(err, file.Sync(), file.Close()) + if err == nil { + err = os.Rename(tmp, record) + } + if err != nil { + return errors.Join(err, fcRemoveMissing(tmp)) + } + return fcSyncPath(d.cfg.VolumeDir) +} diff --git a/internal/vm/firecracker_volume_resize_test.go b/internal/vm/firecracker_volume_resize_test.go new file mode 100644 index 0000000..b9f0ac9 --- /dev/null +++ b/internal/vm/firecracker_volume_resize_test.go @@ -0,0 +1,219 @@ +//go:build linux + +package vm + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func volumeDiskSize(t *testing.T, cfg FirecrackerConfig, id string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(cfg.VolumeDir, id+".ext4")) + if err != nil { + t.Fatal(err) + } + return info.Size() +} + +func TestFirecrackerVolumeGrow(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + disk := filepath.Join(cfg.VolumeDir, volume.ID+".ext4") + + grown, err := d.ResizeVolume(ctx, volume.ID, 192) + if err != nil || grown.SizeMiB != 192 || grown.ID != volume.ID || grown.Name != "data" { + t.Fatalf("grow = %+v %v", grown, err) + } + // The host tools saw the already-extended file, and the record agrees. + if want := fcE2fsck[0] + " " + fcResize2fs[0] + " " + disk + " " + "201326592"; len(host.resized) != 1 || host.resized[0] != want { + t.Fatalf("resized = %v, want %q", host.resized, want) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size = %d", got) + } + volumes, err := d.Volumes(ctx) + if err != nil || len(volumes) != 1 || volumes[0].SizeMiB != 192 { + t.Fatalf("volumes = %+v %v", volumes, err) + } + // The grown size counts against the budget (256 MiB): 64 MiB are left. + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create after grow: %v", err) + } + // The same size again does nothing. + if again, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || again.SizeMiB != 192 || len(host.resized) != 1 { + t.Fatalf("repeat = %+v %v calls=%v", again, err, host.resized) + } + // No stray record temp file is left behind. + if _, err := os.Stat(filepath.Join(cfg.VolumeDir, volume.ID+".json.tmp")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("record temp file remains: %v", err) + } +} + +func TestFirecrackerVolumeGrowRefusals(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + for name, test := range map[string]struct { + id string + size int + want error + }{ + "shrink": {volume.ID, 64, ErrVolumeSize}, + "too small": {volume.ID, 63, ErrVolumeSize}, + "too large": {volume.ID, 1<<20 + 1, ErrVolumeSize}, + "over budget": {volume.ID, 300, ErrVolumeBudget}, + "unknown": {"vol-" + strings.Repeat("e", 32), 192, ErrVolumeNotFound}, + "malformed id": {"../data", 192, ErrVolumeNotFound}, + } { + if _, err := d.ResizeVolume(ctx, test.id, test.size); !errors.Is(err, test.want) { + t.Fatalf("%s: %v, want %v", name, err, test.want) + } + } + // The host disk floor counts too. + host.free = uint64(cfg.DiskFloorMiB+63) << 20 + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("below floor: %v", err) + } + host.free = 100 << 30 + // An untrusted e2fsck or resize2fs stops the grow before the file moves. + for _, tool := range []string{fcE2fsck[0], fcResize2fs[0]} { + host.untrusted = map[string]bool{tool: true, fcE2fsck[1]: true, fcResize2fs[1]: true} + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil || !strings.Contains(err.Error(), "trusted host tool") { + t.Fatalf("untrusted %s: %v", tool, err) + } + } + host.untrusted = nil + if len(host.resized) != 0 || volumeDiskSize(t, cfg, volume.ID) != 128<<20 { + t.Fatalf("a refused grow changed the volume: calls=%v size=%d", host.resized, volumeDiskSize(t, cfg, volume.ID)) + } + // An attached volume cannot grow. + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, ErrVolumeInUse) { + t.Fatalf("attached: %v", err) + } + if len(host.resized) != 0 || volumeDiskSize(t, cfg, volume.ID) != 128<<20 { + t.Fatal("an attached volume was changed") + } +} + +// A filesystem that fails its check is not resized, and its file goes back to +// the recorded size, so the volume is exactly as before. +func TestFirecrackerVolumeGrowFsckFailureRestoresDisk(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + host.resizeErr = fmt.Errorf("%w: exit status 4", errVolumeFsck) + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, errVolumeFsck) { + t.Fatalf("grow of a damaged filesystem: %v", err) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 128<<20 { + t.Fatalf("disk size after a failed check = %d", got) + } + host.resizeErr = nil + if grown, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || grown.SizeMiB != 192 { + t.Fatalf("grow after the failed check = %+v %v", grown, err) + } +} + +// A failed filesystem step leaves the disk file larger than its record. The +// same grow then completes, since each step can be repeated. +func TestFirecrackerVolumeGrowRetryAfterFailure(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + host.resizeErr = errors.New("e2fsck: errors left uncorrected") + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil { + t.Fatal("failed resize reported success") + } + if volumes, _ := d.Volumes(ctx); len(volumes) != 1 || volumes[0].SizeMiB != 128 { + t.Fatalf("record changed by a failed grow: %+v", volumes) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size after failure = %d", got) + } + // The failed grow released its reservation: the volume can be deleted or retried. + host.resizeErr = nil + grown, err := d.ResizeVolume(ctx, volume.ID, 192) + if err != nil || grown.SizeMiB != 192 { + t.Fatalf("retry = %+v %v", grown, err) + } + // A disk already larger than the request is not cut down. + if err := os.Truncate(filepath.Join(cfg.VolumeDir, volume.ID+".ext4"), 250<<20); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeSize) { + t.Fatalf("oversized disk: %v", err) + } +} + +// While e2fsck and resize2fs run, the volume is held: it cannot be attached, +// deleted or backed up, and the growth counts against the budget. +func TestFirecrackerVolumeGrowHoldsVolume(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + if err := d.DeleteVolume(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("delete during grow: %v", err) + } + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("attach during grow: %v", err) + } + if _, err := d.OpenVolumeBackup(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("backup during grow: %v", err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("second grow: %v", err) + } + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create during grow: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } + if err := d.DeleteVolume(ctx, volume.ID); err != nil { + t.Fatalf("delete after grow: %v", err) + } +} diff --git a/internal/vm/firecracker_volumes.go b/internal/vm/firecracker_volumes.go index 1b42c6c..2a94777 100644 --- a/internal/vm/firecracker_volumes.go +++ b/internal/vm/firecracker_volumes.go @@ -116,7 +116,7 @@ func (d *FirecrackerDriver) recoverVolumes() error { } for _, name := range names { path := filepath.Join(cfg.VolumeDir, name) - if strings.HasSuffix(name, ".ext4.tmp") { + if strings.HasSuffix(name, ".ext4.tmp") || strings.HasSuffix(name, ".json.tmp") { if err := os.Remove(path); err != nil { return err } diff --git a/internal/vm/volume.go b/internal/vm/volume.go index 5473c28..439566f 100644 --- a/internal/vm/volume.go +++ b/internal/vm/volume.go @@ -17,6 +17,9 @@ var ( // ErrVolumeInUse: the volume is attached to a VM (any trace of one: a // running, stopped or half-destroyed VM still holds it). ErrVolumeInUse = errors.New("volume is attached to a sandbox") + // ErrVolumeSize: the requested volume size is invalid, or smaller than + // the current one. + ErrVolumeSize = errors.New("invalid volume size") // ErrVolumeBudget: creating the volume would exceed the total volume // budget or the host disk floor. ErrVolumeBudget = errors.New("volume budget exhausted") @@ -53,6 +56,15 @@ type Volumes interface { DeleteVolume(ctx context.Context, id string) error } +// VolumeResizer is implemented by drivers that can grow a detached volume in +// place (the Linux Firecracker driver). It never shrinks: a smaller size +// fails with ErrVolumeSize, the same size is a no-op, and an attached, +// transferring or missing volume fails with ErrVolumeInUse, +// ErrVolumeTransferBusy or ErrVolumeNotFound. +type VolumeResizer interface { + ResizeVolume(ctx context.Context, id string, sizeMiB int) (Volume, error) +} + var ( volumeName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}$`) volumeID = regexp.MustCompile(`^vol-[0-9a-f]{32}$`) From b96aebf840ce53f64d336327c64136ca6a9d553e Mon Sep 17 00:00:00 2001 From: jerryfane Date: Wed, 4 Nov 2026 08:33:39 +0100 Subject: [PATCH 2/4] Volume grow: fence the disk with an inherited flock; count unrecorded growth against the budget --- docs/firecracker.md | 11 +- internal/vm/firecracker.go | 3 +- internal/vm/firecracker_backup.go | 2 +- internal/vm/firecracker_host.go | 21 ++- internal/vm/firecracker_test.go | 2 +- internal/vm/firecracker_volume_resize.go | 44 +++++- internal/vm/firecracker_volume_resize_test.go | 143 ++++++++++++++++++ internal/vm/firecracker_volumes.go | 4 +- 8 files changed, 216 insertions(+), 14 deletions(-) diff --git a/docs/firecracker.md b/docs/firecracker.md index 21c86d3..c742fd6 100644 --- a/docs/firecracker.md +++ b/docs/firecracker.md @@ -246,7 +246,7 @@ existing volume after `-fc-volume-mib` was raised, which only affects new volumes. Steps, with the volume held (no attach, delete or backup) throughout: check -budget and floor, extend the disk file, `e2fsck -f -p`, `resize2fs`, then +budget and floor, take an exclusive `flock` on the disk file, extend it, `e2fsck -f -p`, `resize2fs`, then replace the record with one atomic rename. The host uses its own trusted `e2fsck` and `resize2fs` (root-owned regular files under `/usr/sbin` or `/sbin`, found when a grow runs; `e2fsprogs` ships them with `mke2fs`). Nothing is mounted. If `e2fsck` cannot @@ -256,7 +256,14 @@ stops mid-grow, the disk file is larger than its record until the same request is repeated: the steps are repeatable, and a disk file larger than the requested size is refused rather than cut. Until then the volume can still be attached (the guest sees a larger device holding the old filesystem) but a -backup of it is refused. +backup of it is refused. Budget accounting counts a volume at the larger of +its record and its disk file, so an unfinished grow is never free space. + +`e2fsck` and `resize2fs` do not die with the daemon. They inherit the locked +disk, so the `flock` outlives a killed daemon until the last tool has exited; +every disk user (attach, delete, backup, grow) checks it and answers busy +(`409`) meanwhile. A restarted daemon therefore cannot attach a disk a tool +is still writing. Trust: `e2fsck` and `resize2fs` run as root on a filesystem the guest wrote, which is the one place the host parses guest-controlled metadata (`mke2fs` diff --git a/internal/vm/firecracker.go b/internal/vm/firecracker.go index 29abc8b..30a2c18 100644 --- a/internal/vm/firecracker.go +++ b/internal/vm/firecracker.go @@ -178,7 +178,8 @@ type fcHost interface { FormatVolume(ctx context.Context, mkfs, path string) error // ResizeVolume runs the trusted e2fsck (-f -p) and resize2fs on a // volume image file already extended to its new size. Nothing is mounted. - ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string) error + // lock is the flocked disk file; the tools inherit it (see volumeLocked). + ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string, lock *os.File) error ApplyFirewall(ctx context.Context, ruleset string) error FirewallState(ctx context.Context) (string, error) diff --git a/internal/vm/firecracker_backup.go b/internal/vm/firecracker_backup.go index b3159b3..0042d95 100644 --- a/internal/vm/firecracker_backup.go +++ b/internal/vm/firecracker_backup.go @@ -287,7 +287,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if volume.Name == name { return fcVolume{}, ErrVolumeExists } - used += volume.SizeMiB + used += d.volumeFootprintMiB(volume) } if used+size > d.cfg.VolumeBudgetMiB { return fcVolume{}, ErrVolumeBudget diff --git a/internal/vm/firecracker_host.go b/internal/vm/firecracker_host.go index 6a047ca..988cd21 100644 --- a/internal/vm/firecracker_host.go +++ b/internal/vm/firecracker_host.go @@ -62,11 +62,21 @@ func newLinuxFCHost(names fcNames) *linuxFCHost { } func (h *linuxFCHost) run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) { + return h.runInheriting(ctx, nil, stdin, name, args...) +} + +// runInheriting runs a tool that also inherits keep as its fd 3. A flock on +// keep is held by the open file description, which the child shares: it stays +// locked until the tool exits, even if the daemon dies first. +func (h *linuxFCHost) runInheriting(ctx context.Context, keep *os.File, stdin []byte, name string, args ...string) ([]byte, error) { if h.toolsErr != nil { return nil, h.toolsErr } c := exec.CommandContext(ctx, name, args...) c.Env = []string{fcHostPath} + if keep != nil { + c.ExtraFiles = []*os.File{keep} + } if stdin != nil { c.Stdin = bytes.NewReader(stdin) } @@ -139,15 +149,18 @@ func (h *linuxFCHost) FormatVolume(ctx context.Context, mkfs, path string) error // ResizeVolume checks the volume's filesystem, then grows it to fill the // image file. e2fsck exit 1 means it repaired errors, which -p only does when // that is safe; anything else stops the grow before resize2fs, with -// errVolumeFsck so the caller knows the filesystem was not resized. -func (h *linuxFCHost) ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string) error { - if _, err := h.run(ctx, nil, e2fsck, "-f", "-p", path); err != nil { +// errVolumeFsck so the caller knows the filesystem was not resized. lock is +// the caller's open, flocked file for path; each tool inherits it, so the +// disk stays fenced while a tool runs, even if the daemon dies (the tools do +// not die with it). +func (h *linuxFCHost) ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string, lock *os.File) error { + if _, err := h.runInheriting(ctx, lock, nil, e2fsck, "-f", "-p", path); err != nil { var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 1 { return fmt.Errorf("%w: %w", errVolumeFsck, err) } } - _, err := h.run(ctx, nil, resize2fs, path) + _, err := h.runInheriting(ctx, lock, nil, resize2fs, path) return err } diff --git a/internal/vm/firecracker_test.go b/internal/vm/firecracker_test.go index 6326145..c0631c4 100644 --- a/internal/vm/firecracker_test.go +++ b/internal/vm/firecracker_test.go @@ -162,7 +162,7 @@ func (h *fakeFCHost) FormatVolume(_ context.Context, mkfs, path string) error { // ResizeVolume stands in for e2fsck and resize2fs: it records the disk size it // was handed, optionally blocks on resizeGate, and fails with resizeErr. -func (h *fakeFCHost) ResizeVolume(_ context.Context, e2fsck, resize2fs, path string) error { +func (h *fakeFCHost) ResizeVolume(_ context.Context, e2fsck, resize2fs, path string, _ *os.File) error { info, err := os.Stat(path) if err != nil { return err diff --git a/internal/vm/firecracker_volume_resize.go b/internal/vm/firecracker_volume_resize.go index 4eb795d..be99d06 100644 --- a/internal/vm/firecracker_volume_resize.go +++ b/internal/vm/firecracker_volume_resize.go @@ -72,7 +72,7 @@ func (d *FirecrackerDriver) ResizeVolume(ctx context.Context, id string, sizeMiB resize2fs, err = d.trustedTool(fcResize2fs) } } - growth := sizeMiB - volume.SizeMiB + growth := max(sizeMiB-d.volumeFootprintMiB(volume), 0) if err == nil { err = d.reserveGrowth(volume, growth) } @@ -88,6 +88,8 @@ func (d *FirecrackerDriver) ResizeVolume(ctx context.Context, id string, sizeMiB volume.SizeMiB = sizeMiB d.volumeMu.Lock() defer d.volumeMu.Unlock() + // If this fails, the enlarged disk keeps counting against the budget: + // accounting reads the disk file's size, not only the record. if err := d.replaceVolumeRecord(volume); err != nil { return Volume{}, err } @@ -108,7 +110,7 @@ func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { } used := reserved for _, existing := range volumes { - used += existing.SizeMiB + used += d.volumeFootprintMiB(existing) } if used+growth > d.cfg.VolumeBudgetMiB { return fmt.Errorf("%w: %d of %d MiB used, growing %s takes %d MiB more", ErrVolumeBudget, used, d.cfg.VolumeBudgetMiB, volume.ID, growth) @@ -140,6 +142,15 @@ func (d *FirecrackerDriver) growDisk(ctx context.Context, id string, oldMiB, new if !ok || !info.Mode().IsRegular() || stat.Nlink != 1 || stat.Uid != uint32(os.Geteuid()) { return errors.New("volume disk is not a detached native disk") } + // Fence the disk until the last tool has exited. The tools inherit this + // open file, so the lock outlives the daemon if it is killed mid-grow; a + // restarted daemon sees the disk as busy (volumeInUse) until they finish. + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return ErrVolumeTransferBusy + } + return err + } size, target := int64(oldMiB)<<20, int64(newMiB)<<20 if info.Size() < size || info.Size() > target { return fmt.Errorf("%w: disk is %d bytes, expected %d to %d", ErrVolumeSize, info.Size(), size, target) @@ -153,7 +164,7 @@ func (d *FirecrackerDriver) growDisk(ctx context.Context, id string, oldMiB, new if err := file.Sync(); err != nil { return err } - err = d.host.ResizeVolume(ctx, e2fsck, resize2fs, d.volumeDisk(id)) + err = d.host.ResizeVolume(ctx, e2fsck, resize2fs, d.volumeDisk(id), file) if errors.Is(err, errVolumeFsck) && extended { // Nothing was resized: put the file back, so the volume still matches // its record and can be backed up or attached. @@ -186,3 +197,30 @@ func (d *FirecrackerDriver) replaceVolumeRecord(volume fcVolume) error { } return fcSyncPath(d.cfg.VolumeDir) } + +// volumeLocked reports ErrVolumeTransferBusy while any process, this daemon or +// a e2fsck/resize2fs that outlived a previous one, holds the disk's flock. +func (d *FirecrackerDriver) volumeLocked(id string) error { + fd, err := syscall.Open(d.volumeDisk(id), syscall.O_RDONLY|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return err + } + defer syscall.Close(fd) + if err := syscall.Flock(fd, syscall.LOCK_SH|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return fmt.Errorf("%w: a filesystem tool still holds %s", ErrVolumeTransferBusy, id) + } + return err + } + return syscall.Flock(fd, syscall.LOCK_UN) +} + +// volumeFootprintMiB is what a volume costs the budget: its recorded size, or +// its disk file when that is larger (a grow whose record was not published). +func (d *FirecrackerDriver) volumeFootprintMiB(volume fcVolume) int { + info, err := os.Lstat(d.volumeDisk(volume.ID)) + if err != nil { + return volume.SizeMiB + } + return max(volume.SizeMiB, int((info.Size()+1<<20-1)>>20)) +} diff --git a/internal/vm/firecracker_volume_resize_test.go b/internal/vm/firecracker_volume_resize_test.go index b9f0ac9..c077ce3 100644 --- a/internal/vm/firecracker_volume_resize_test.go +++ b/internal/vm/firecracker_volume_resize_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" "time" ) @@ -217,3 +218,145 @@ func TestFirecrackerVolumeGrowHoldsVolume(t *testing.T) { t.Fatalf("delete after grow: %v", err) } } + +// A grow whose record could not be published leaves an enlarged disk. It keeps +// counting against the budget (also after a restart, which re-reads the disk), +// so a create cannot admit more disk than the budget allows. +func TestFirecrackerVolumeGrowFailedRecordKeepsBudget(t *testing.T) { + d, _, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + // A directory where the record's temp file goes makes publication fail. + block := filepath.Join(cfg.VolumeDir, volume.ID+".json.tmp") + if err := os.Mkdir(block, 0o700); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil { + t.Fatal("grow reported success although its record was not written") + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size = %d", got) + } + // Budget 256 MiB: the 192 MiB disk leaves room for no second 128 MiB volume. + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create next to an unrecorded 192 MiB disk: %v", err) + } + if err := os.Remove(block); err != nil { + t.Fatal(err) + } + // The same request finishes the grow, without counting the extension twice. + if grown, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || grown.SizeMiB != 192 { + t.Fatalf("retry = %+v %v", grown, err) + } +} + +func lockedByAnother(t *testing.T, path string) bool { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + err = syscall.Flock(int(file.Fd()), syscall.LOCK_SH|syscall.LOCK_NB) + if err == nil { + return false + } + if !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatal(err) + } + return true +} + +// The fence is the disk's flock, not process memory: with the in-memory +// reservation gone (a restarted daemon has none), a disk a tool still holds +// cannot be attached, deleted, backed up or grown. +func TestFirecrackerVolumeGrowFencedByDiskLock(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + disk := filepath.Join(cfg.VolumeDir, volume.ID+".ext4") + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for !lockedByAnother(t, disk) { + if time.Now().After(deadline) { + t.Fatal("the disk was not locked while the tools ran") + } + time.Sleep(time.Millisecond) + } + d.volumeMu.Lock() + clear(d.volumeTransfers) + d.volumeMu.Unlock() + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("attach with the tools running: %v", err) + } + if err := d.DeleteVolume(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("delete with the tools running: %v", err) + } + if _, err := d.OpenVolumeBackup(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("backup with the tools running: %v", err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("second grow with the tools running: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } + if lockedByAnother(t, disk) { + t.Fatal("the disk is still locked after the grow") + } +} + +// The real host passes the locked file to e2fsck and resize2fs. The tool keeps +// working after the daemon's own descriptor is closed, which is what a daemon +// killed mid-grow leaves behind: here resize2fs leaves a child that still runs +// after the call returned and the daemon's descriptor was closed. +func TestLinuxHostResizeToolsKeepDiskLocked(t *testing.T) { + dir := t.TempDir() + disk := filepath.Join(dir, "v.ext4") + if err := os.WriteFile(disk, make([]byte, 4096), 0o600); err != nil { + t.Fatal(err) + } + script := func(name, body string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o700); err != nil { + t.Fatal(err) + } + return path + } + fsck := script("e2fsck", "exit 0\n") + resize := script("resize2fs", "sleep 1 /dev/null 2>&1 &\nexit 0\n") + + lock, err := os.OpenFile(disk, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatal(err) + } + if err := (&linuxFCHost{}).ResizeVolume(context.Background(), fsck, resize, disk, lock); err != nil { + t.Fatal(err) + } + if err := lock.Close(); err != nil { + t.Fatal(err) + } + if !lockedByAnother(t, disk) { + t.Fatal("the disk was not locked by the tool after the daemon's descriptor closed") + } + deadline := time.Now().Add(10 * time.Second) + for lockedByAnother(t, disk) { + if time.Now().After(deadline) { + t.Fatal("the disk stayed locked after the tool exited") + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/internal/vm/firecracker_volumes.go b/internal/vm/firecracker_volumes.go index 2a94777..7d745a9 100644 --- a/internal/vm/firecracker_volumes.go +++ b/internal/vm/firecracker_volumes.go @@ -255,7 +255,7 @@ func (d *FirecrackerDriver) volumeInUse(id string) error { if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Nlink != 1 { return fmt.Errorf("%w: a jail still links %s", ErrVolumeInUse, id) } - return nil + return d.volumeLocked(id) } // CreateVolume makes a VolumeMiB ext4 disk named name. @@ -281,7 +281,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if volume.Name == name { return Volume{}, fmt.Errorf("%w: %q", ErrVolumeExists, name) } - used += volume.SizeMiB + used += d.volumeFootprintMiB(volume) } size := d.cfg.VolumeMiB if used+size > d.cfg.VolumeBudgetMiB { From 3683edc5550813febe041aab4a7880a8dff6122d Mon Sep 17 00:00:00 2001 From: jerryfane Date: Wed, 4 Nov 2026 08:59:38 +0100 Subject: [PATCH 3/4] Volume grow: charge in-flight growth once (reserve only the unextended remainder) --- internal/vm/firecracker_backup.go | 16 +++++-- internal/vm/firecracker_volume_resize.go | 19 ++++++--- internal/vm/firecracker_volume_resize_test.go | 42 +++++++++++++++++++ 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/internal/vm/firecracker_backup.go b/internal/vm/firecracker_backup.go index 0042d95..c8974b7 100644 --- a/internal/vm/firecracker_backup.go +++ b/internal/vm/firecracker_backup.go @@ -17,12 +17,14 @@ import ( ) // volumeTransfers is protected by volumeMu. A zero SizeMiB is a backup of an -// existing volume; a positive size reserves a new name and restore capacity. +// existing volume; a positive size reserves a new name and restore capacity; +// GrowTo > 0 is a grow of an existing volume from GrowFrom MiB to GrowTo MiB. // Reservations are process-local: the daemon's instance/ledger claim excludes // other writers, and restart removes incomplete .ext4.tmp imports. type fcVolumeTransfer struct { - Name string - SizeMiB int + Name string + SizeMiB int + GrowFrom, GrowTo int } func (d *FirecrackerDriver) reserveVolumeTransfer(id string, transfer fcVolumeTransfer) error { @@ -50,7 +52,13 @@ func (d *FirecrackerDriver) releaseVolumeTransfer(id string) { // partially allocated its staging file; a slow import must never overcommit it. func (d *FirecrackerDriver) reservedVolumeMiB(name string) (int, error) { reserved := 0 - for _, transfer := range d.volumeTransfers { + for id, transfer := range d.volumeTransfers { + if transfer.GrowTo > 0 { + // The volume's own footprint already counts what the disk file + // has been extended by; reserve only the rest of the growth. + reserved += max(transfer.GrowTo-max(transfer.GrowFrom, d.diskMiB(id)), 0) + continue + } if transfer.SizeMiB == 0 { continue } diff --git a/internal/vm/firecracker_volume_resize.go b/internal/vm/firecracker_volume_resize.go index be99d06..6c06fb2 100644 --- a/internal/vm/firecracker_volume_resize.go +++ b/internal/vm/firecracker_volume_resize.go @@ -97,9 +97,11 @@ func (d *FirecrackerDriver) ResizeVolume(ctx context.Context, id string, sizeMiB } // reserveGrowth checks the volume budget and the host disk floor for growth -// MiB more, then reserves it like an in-flight restore, so a concurrent create -// counts it. Callers hold volumeMu. +// MiB more than the volume's footprint, then reserves the grow, so a +// concurrent create counts it: the not yet extended remainder through the +// reservation, the extended part through the disk file. Callers hold volumeMu. func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { + target := d.volumeFootprintMiB(volume) + growth volumes, err := d.readVolumes() if err != nil { return err @@ -122,7 +124,7 @@ func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { if need := uint64(d.cfg.DiskFloorMiB+growth+reserved) << 20; free < need { return fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB growth needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, growth) } - return d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: volume.Name, SizeMiB: growth}) + return d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: volume.Name, GrowFrom: volume.SizeMiB, GrowTo: target}) } // growDisk extends the detached disk file and its filesystem. A disk larger @@ -218,9 +220,14 @@ func (d *FirecrackerDriver) volumeLocked(id string) error { // volumeFootprintMiB is what a volume costs the budget: its recorded size, or // its disk file when that is larger (a grow whose record was not published). func (d *FirecrackerDriver) volumeFootprintMiB(volume fcVolume) int { - info, err := os.Lstat(d.volumeDisk(volume.ID)) + return max(volume.SizeMiB, d.diskMiB(volume.ID)) +} + +// diskMiB is a volume's disk file size rounded up to MiB, 0 if unreadable. +func (d *FirecrackerDriver) diskMiB(id string) int { + info, err := os.Lstat(d.volumeDisk(id)) if err != nil { - return volume.SizeMiB + return 0 } - return max(volume.SizeMiB, int((info.Size()+1<<20-1)>>20)) + return int((info.Size() + 1<<20 - 1) >> 20) } diff --git a/internal/vm/firecracker_volume_resize_test.go b/internal/vm/firecracker_volume_resize_test.go index c077ce3..18a5560 100644 --- a/internal/vm/firecracker_volume_resize_test.go +++ b/internal/vm/firecracker_volume_resize_test.go @@ -360,3 +360,45 @@ func TestLinuxHostResizeToolsKeepDiskLocked(t *testing.T) { time.Sleep(20 * time.Millisecond) } } + +// While a grow runs, the budget charges the volume's growth once. The extended +// disk file and the reserved growth must not both count: a create that fits +// next to the grown volume is admitted, one that does not is refused. +func TestFirecrackerVolumeGrowInFlightChargedOnce(t *testing.T) { + d, host, _ := newTestFirecracker(t, func(c *FirecrackerConfig) { + c.VolumeBudgetMiB, c.VolumeMiB, c.VolumeDir = 320, 128, filepath.Join(c.Root, "volumes") + }) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + // 192 MiB growing + 128 MiB new = the 320 MiB budget exactly. + if _, err := d.CreateVolume(ctx, "next"); err != nil { + t.Fatalf("a volume that fits next to the growing one was refused: %v", err) + } + if _, err := d.CreateVolume(ctx, "over"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create past the budget during a grow: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } +} From 35efbc38c120b12ef1195281b2a0969be89216bb Mon Sep 17 00:00:00 2001 From: jerryfane Date: Wed, 4 Nov 2026 09:37:43 +0100 Subject: [PATCH 4/4] Volume grow: keep the disk floor's headroom for the whole growth until it ends --- internal/vm/firecracker_backup.go | 26 ++++++---- internal/vm/firecracker_volume_resize.go | 4 +- internal/vm/firecracker_volume_resize_test.go | 49 +++++++++++++++++++ internal/vm/firecracker_volumes.go | 4 +- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/internal/vm/firecracker_backup.go b/internal/vm/firecracker_backup.go index c8974b7..66987ea 100644 --- a/internal/vm/firecracker_backup.go +++ b/internal/vm/firecracker_backup.go @@ -50,24 +50,30 @@ func (d *FirecrackerDriver) releaseVolumeTransfer(id string) { // reservedVolumeMiB also rejects names reserved by an in-flight restore. Count // full restore sizes against disk headroom conservatively, even if a stream has // partially allocated its staging file; a slow import must never overcommit it. -func (d *FirecrackerDriver) reservedVolumeMiB(name string) (int, error) { - reserved := 0 +// +// It returns two figures. budget is what the volume budget must add to the +// volumes' footprints: restore sizes, and the part of a grow the disk file has +// not been extended by yet (the extended part is in the footprint). headroom is +// what the host disk floor must hold back: restore sizes and each grow's whole +// growth until it finishes, because extending the file is sparse and the +// allocation only happens while e2fsck and resize2fs run. +func (d *FirecrackerDriver) reservedVolumeMiB(name string) (budget, headroom int, err error) { for id, transfer := range d.volumeTransfers { if transfer.GrowTo > 0 { - // The volume's own footprint already counts what the disk file - // has been extended by; reserve only the rest of the growth. - reserved += max(transfer.GrowTo-max(transfer.GrowFrom, d.diskMiB(id)), 0) + budget += max(transfer.GrowTo-max(transfer.GrowFrom, d.diskMiB(id)), 0) + headroom += transfer.GrowTo - transfer.GrowFrom continue } if transfer.SizeMiB == 0 { continue } if transfer.Name == name { - return 0, ErrVolumeExists + return 0, 0, ErrVolumeExists } - reserved += transfer.SizeMiB + budget += transfer.SizeMiB + headroom += transfer.SizeMiB } - return reserved, nil + return budget, headroom, nil } func (d *FirecrackerDriver) OpenVolumeBackup(ctx context.Context, id string) (VolumeBackup, error) { @@ -282,7 +288,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if len(d.volumeTransfers) >= MaxVolumeTransfers { return fcVolume{}, ErrVolumeTransferBusy } - reserved, err := d.reservedVolumeMiB(name) + reserved, headroom, err := d.reservedVolumeMiB(name) if err != nil { return fcVolume{}, err } @@ -304,7 +310,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if err != nil { return fcVolume{}, err } - if free < uint64(d.cfg.DiskFloorMiB+size+reserved)<<20 { + if free < uint64(d.cfg.DiskFloorMiB+size+headroom)<<20 { return fcVolume{}, ErrVolumeBudget } var random [16]byte diff --git a/internal/vm/firecracker_volume_resize.go b/internal/vm/firecracker_volume_resize.go index 6c06fb2..c1c4084 100644 --- a/internal/vm/firecracker_volume_resize.go +++ b/internal/vm/firecracker_volume_resize.go @@ -106,7 +106,7 @@ func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { if err != nil { return err } - reserved, err := d.reservedVolumeMiB(volume.Name) + reserved, headroom, err := d.reservedVolumeMiB(volume.Name) if err != nil { return err } @@ -121,7 +121,7 @@ func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { if err != nil { return err } - if need := uint64(d.cfg.DiskFloorMiB+growth+reserved) << 20; free < need { + if need := uint64(d.cfg.DiskFloorMiB+growth+headroom) << 20; free < need { return fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB growth needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, growth) } return d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: volume.Name, GrowFrom: volume.SizeMiB, GrowTo: target}) diff --git a/internal/vm/firecracker_volume_resize_test.go b/internal/vm/firecracker_volume_resize_test.go index 18a5560..210bf98 100644 --- a/internal/vm/firecracker_volume_resize_test.go +++ b/internal/vm/firecracker_volume_resize_test.go @@ -402,3 +402,52 @@ func TestFirecrackerVolumeGrowInFlightChargedOnce(t *testing.T) { t.Fatal(err) } } + +// Extending the disk file is sparse: the space is only used while the tools +// run. The host disk floor therefore keeps holding the grow's whole growth +// back until the grow ends, also after the file reached its new size. +func TestFirecrackerVolumeGrowKeepsDiskHeadroom(t *testing.T) { + d, host, cfg := newTestFirecracker(t, func(c *FirecrackerConfig) { + c.VolumeBudgetMiB, c.VolumeMiB, c.VolumeDir = 1024, 128, filepath.Join(c.Root, "volumes") + }) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + // The file is already 192 MiB, but 64 MiB of it is still unallocated: free + // space for the floor plus a new 128 MiB volume is not enough with it. + host.mu.Lock() + host.free = uint64(cfg.DiskFloorMiB+128+63) << 20 + host.mu.Unlock() + if _, err := d.CreateVolume(ctx, "next"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("a create that would eat the grow's headroom: %v", err) + } + host.mu.Lock() + host.free = uint64(cfg.DiskFloorMiB+128+64) << 20 + host.mu.Unlock() + if _, err := d.CreateVolume(ctx, "next"); err != nil { + t.Fatalf("a create that leaves the grow's headroom: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/vm/firecracker_volumes.go b/internal/vm/firecracker_volumes.go index 7d745a9..fc191b4 100644 --- a/internal/vm/firecracker_volumes.go +++ b/internal/vm/firecracker_volumes.go @@ -272,7 +272,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - reserved, err := d.reservedVolumeMiB(name) + reserved, headroom, err := d.reservedVolumeMiB(name) if err != nil { return Volume{}, err } @@ -291,7 +291,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - if need := uint64(d.cfg.DiskFloorMiB+size+reserved) << 20; free < need { + if need := uint64(d.cfg.DiskFloorMiB+size+headroom) << 20; free < need { return Volume{}, fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB volume needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, size) }