diff --git a/docs/compatibility.md b/docs/compatibility.md index 80624a6..d1d8d38 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,6 +31,7 @@ Volumes are off by default. They stay off on the Apple (Mac) driver and on a gat - `POST /volumes` `{"name"}` creates one: `201` `{"volumeID","name","token"}`. A name is 1–63 letters, digits, `_` or `-`, starting with a letter or digit (else `400`); a taken name is `409`; a volume that would take the sum of all volume sizes over `-fc-volume-budget-mib` is `400` (`volume budget exhausted`). Unknown body fields are `400`. - `GET /volumes` lists `[{"volumeID","name"}]` by name; `GET /volumes/{id}` (the SDKs' `Volume.connect`, `get_info`) answers `{"volumeID","name","token"}` or `404`. The `token` is an HMAC of the volume ID under the token secret, and grants nothing: the volume content API it would authorize is refused. +- `POST /volumes/{id}/resize` `{"sizeMiB": N}` is a sandboxd operator route outside the E2B API (control API key): on the Firecracker driver it grows a detached volume to N MiB and answers `200 {"volumeID","name","sizeMiB"}`; it never shrinks (`400`), and an attached or busy volume is `409` (see [firecracker.md](firecracker.md#growing-a-detached-volume)). - `DELETE /volumes/{id}` is `204` and deletes the data; an unknown ID is `404`, and a volume attached to a sandbox is `409` (kill the sandbox first). - `/volumecontent/…` (the SDKs' volume file methods: `list`, `read_file`, `write_file`, …) answers `501` `the volume content API is not supported: mount the volume in a sandbox (volumeMounts) and use its files there`. - `POST /v2/sandboxes` with `"volumeMounts":[{"name","path"}]` mounts the named volume at `path` (absolute, clean, not `/`; else `400`), owned by `user`. At most one mount per sandbox (`400` otherwise); absent, `null` or `[]` mounts nothing. An unknown name is `404`; a volume attached to another sandbox is `409` `volume "" is attached to sandbox `, refused before any capacity is reserved (two creates racing for one volume: the driver admits one, the other is `409` too). A volume is attached to at most one sandbox at a time, until that sandbox ends. diff --git a/docs/firecracker.md b/docs/firecracker.md index 349713e..c742fd6 100644 --- a/docs/firecracker.md +++ b/docs/firecracker.md @@ -232,6 +232,45 @@ lost there; the volume's journal keeps it consistent. A volume survives its VM's end, a daemon restart or `kill -9` and that destroy; only `DELETE /volumes/{id}` removes it. +### Growing a detached volume + +`POST /volumes/{id}/resize` with `{"sizeMiB": N}` (control API key, like the +backup route; sandbox envd tokens cannot reach it) grows a **detached** volume +in place and answers `200 {"volumeID", "name", "sizeMiB"}`. It never shrinks: +a smaller size is `400`, the same size is a no-op `200`. An attached, +backing-up, restoring or already-growing volume is `409`; an unknown ID `404`; +a grow past `-fc-volume-budget-mib` or below the disk floor (the growth, not +the whole size, counts) is `400`. The new size must be between 64 MiB and +1 TiB. Drivers without the capability answer `501`. Use it to raise an +existing volume after `-fc-volume-mib` was raised, which only affects new +volumes. + +Steps, with the volume held (no attach, delete or backup) throughout: check +budget and floor, take an exclusive `flock` on the disk file, extend it, `e2fsck -f -p`, `resize2fs`, then +replace the record with one atomic rename. The host uses its own trusted +`e2fsck` and `resize2fs` (root-owned regular files under `/usr/sbin` or +`/sbin`, found when a grow runs; `e2fsprogs` ships them with `mke2fs`). Nothing is mounted. If `e2fsck` cannot +repair the filesystem safely (exit 2 or higher, e.g. `4`) the file goes back to its +old size and the volume is unchanged. If `resize2fs` fails, or the daemon +stops mid-grow, the disk file is larger than its record until the same +request is repeated: the steps are repeatable, and a disk file larger than the +requested size is refused rather than cut. Until then the volume can still +be attached (the guest sees a larger device holding the old filesystem) but a +backup of it is refused. Budget accounting counts a volume at the larger of +its record and its disk file, so an unfinished grow is never free space. + +`e2fsck` and `resize2fs` do not die with the daemon. They inherit the locked +disk, so the `flock` outlives a killed daemon until the last tool has exited; +every disk user (attach, delete, backup, grow) checks it and answers busy +(`409`) meanwhile. A restarted daemon therefore cannot attach a disk a tool +is still writing. + +Trust: `e2fsck` and `resize2fs` run as root on a filesystem the guest wrote, +which is the one place the host parses guest-controlled metadata (`mke2fs` +only writes a fresh one). They take an image file, not a mount, and run with +the daemon's minimal environment. An operator who does not accept that parsing +should not call the route. + ### Controller-only cold backup and new-volume restore These operations use the control `X-API-Key`, not a guest token, and are available diff --git a/internal/control/volumes.go b/internal/control/volumes.go index ec33d35..8a5efa3 100644 --- a/internal/control/volumes.go +++ b/internal/control/volumes.go @@ -33,6 +33,17 @@ type newVolumeRequest struct { Name string `json:"name"` } +// resizeVolumeRequest asks to grow a detached volume to SizeMiB. +type resizeVolumeRequest struct { + SizeMiB int `json:"sizeMiB"` +} + +// resizedVolume is the grow response: the volume with its new size. +type resizedVolume struct { + volumeEntry + SizeMiB int `json:"sizeMiB"` +} + // volumeToken is the token the SDKs require with a volume. It grants // nothing: the volume content API it would authorize is refused, and a // volume's data is reached only from a sandbox it is mounted in. @@ -47,7 +58,7 @@ func (s *Service) withToken(volume vm.Volume) volumeAndToken { // serveVolumes answers E2B's volume API when volumes are configured, and // reports whether path was a volume route. func (s *Service) serveVolumes(w http.ResponseWriter, r *http.Request) bool { - if s.serveVolumeTransfers(w, r) { + if s.serveVolumeTransfers(w, r) || s.serveVolumeResize(w, r) { return true } path := r.URL.Path @@ -96,6 +107,8 @@ func (s *Service) volumeFailure(w http.ResponseWriter, action string, err error) s.fail(w, ProfileE2B, http.StatusConflict, "volume is attached to a sandbox: kill the sandbox first") case errors.Is(err, vm.ErrVolumeTransferBusy): s.fail(w, ProfileE2B, http.StatusConflict, "volume storage operation in progress") + case errors.Is(err, vm.ErrVolumeSize): + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume size: it can only grow, between 64 MiB and 1 TiB") case errors.Is(err, vm.ErrVolumeBackupInvalid): s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume backup") case errors.Is(err, vm.ErrVolumeBudget): @@ -216,3 +229,38 @@ func (s *Service) volumeMount(ctx context.Context, raw json.RawMessage, hardened } return nil, http.StatusNotFound, fmt.Sprintf("volume %q not found", requested.Name) } + +// serveVolumeResize answers POST /volumes/{id}/resize, an operator route +// behind the control API key like backup: it grows a detached volume and never +// shrinks one. Customers, who hold only a sandbox's envd token, cannot reach it. +func (s *Service) serveVolumeResize(w http.ResponseWriter, r *http.Request) bool { + id, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/volumes/"), "/resize") + if !ok || !strings.HasPrefix(r.URL.Path, "/volumes/") { + return false + } + if r.Method != http.MethodPost { + s.fail(w, ProfileE2B, http.StatusMethodNotAllowed, "method not allowed") + return true + } + resizer, ok := s.volumes.(vm.VolumeResizer) + if !ok { + s.fail(w, ProfileE2B, http.StatusNotImplemented, "volume grow unavailable") + return true + } + if !vm.ValidVolumeID(id) { + s.fail(w, ProfileE2B, http.StatusNotFound, "volume not found") + return true + } + var request resizeVolumeRequest + if err := decode(w, r, &request); !errors.Is(err, io.EOF) { + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid volume resize body") + return true + } + volume, err := resizer.ResizeVolume(r.Context(), id, request.SizeMiB) + if err != nil { + s.volumeFailure(w, "resize", err) + return true + } + jsonResponse(w, http.StatusOK, resizedVolume{volumeEntry{volume.ID, volume.Name}, volume.SizeMiB}) + return true +} diff --git a/internal/control/volumes_test.go b/internal/control/volumes_test.go index f79489e..4ab6a49 100644 --- a/internal/control/volumes_test.go +++ b/internal/control/volumes_test.go @@ -75,6 +75,32 @@ func (f *fakeVolumes) DeleteVolume(_ context.Context, id string) error { return fmt.Errorf("volume %s: %w", id, vm.ErrVolumeNotFound) } +// ResizeVolume grows like the driver: only larger, only detached. +func (f *fakeVolumes) ResizeVolume(_ context.Context, id string, sizeMiB int) (vm.Volume, error) { + f.mu.Lock() + defer f.mu.Unlock() + for i := range f.volumes { + volume := &f.volumes[i] + if volume.ID != id { + continue + } + switch { + case volume.AttachedTo != "": + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeInUse) + case sizeMiB < volume.SizeMiB: + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeSize) + } + volume.SizeMiB = sizeMiB + return *volume, nil + } + return vm.Volume{}, fmt.Errorf("volume %s: %w", id, vm.ErrVolumeNotFound) +} + +// plainVolumes is a store that cannot grow volumes. +type plainVolumes struct{ *fakeVolumes } + +func (plainVolumes) ResizeVolume() {} + func (f *fakeVolumes) attach(name, sandbox string) { f.mu.Lock() defer f.mu.Unlock() @@ -396,3 +422,48 @@ func TestVolumesNeedLocalDriver(t *testing.T) { t.Fatal("volumes accepted without a local driver") } } + +func TestVolumeGrowAPI(t *testing.T) { + s, _, volumes := openVolumes(t, 1) + data := createVolume(t, s, "data") + path := "/volumes/" + data.ID + "/resize" + + got := request(t, s, http.MethodPost, path, map[string]any{"sizeMiB": 1024}) + if want := fmt.Sprintf(`{"volumeID":%q,"name":"data","sizeMiB":1024}`+"\n", data.ID); got.Code != http.StatusOK || got.Body.String() != want { + t.Fatalf("grow: %d %s, want %s", got.Code, got.Body.String(), want) + } + if list, _ := volumes.Volumes(context.Background()); list[0].SizeMiB != 1024 { + t.Fatalf("driver size = %d", list[0].SizeMiB) + } + volumes.attach("data", "sbx-1") + for _, c := range []struct { + label, method, path string + body any + status int + message string + }{ + {"attached", http.MethodPost, path, map[string]any{"sizeMiB": 2048}, http.StatusConflict, "attached to a sandbox"}, + {"unknown volume", http.MethodPost, "/volumes/vol-ffffffffffffffffffffffffffffffff/resize", map[string]any{"sizeMiB": 2048}, http.StatusNotFound, "volume not found"}, + {"malformed id", http.MethodPost, "/volumes/data/resize", map[string]any{"sizeMiB": 2048}, http.StatusNotFound, "volume not found"}, + {"wrong method", http.MethodGet, path, nil, http.StatusMethodNotAllowed, "method not allowed"}, + {"unknown field", http.MethodPost, path, map[string]any{"sizeMiB": 2048, "force": true}, http.StatusBadRequest, "invalid volume resize body"}, + } { + wantFailure(t, c.label, request(t, s, c.method, c.path, c.body), c.status, c.message) + } + volumes.mu.Lock() + volumes.volumes[0].AttachedTo = "" + volumes.mu.Unlock() + wantFailure(t, "shrink", request(t, s, http.MethodPost, path, map[string]any{"sizeMiB": 512}), http.StatusBadRequest, "can only grow") + // A missing size is size 0, which the driver refuses; it is never a no-op. + wantFailure(t, "no body", request(t, s, http.MethodPost, path, nil), http.StatusBadRequest, "can only grow") +} + +func TestVolumeGrowNeedsDriverSupport(t *testing.T) { + driver := &specDriver{fakeDriver: &fakeDriver{instances: map[string]vm.Instance{}}} + cfg := mixedConfig(e2bBase, 1) + cfg.Volumes = plainVolumes{&fakeVolumes{}} + s := openAt(t, filepath.Join(t.TempDir(), "ledger.sqlite"), driver, cfg) + created := createVolume(t, s, "data") + wantFailure(t, "unsupported", request(t, s, http.MethodPost, "/volumes/"+created.ID+"/resize", map[string]any{"sizeMiB": 1024}), + http.StatusNotImplemented, "grow unavailable") +} diff --git a/internal/vm/firecracker.go b/internal/vm/firecracker.go index a2c14b4..30a2c18 100644 --- a/internal/vm/firecracker.go +++ b/internal/vm/firecracker.go @@ -176,6 +176,10 @@ type fcHost interface { // FormatVolume runs the trusted mkfs binary on the volume image file // path; nothing is mounted on the host. FormatVolume(ctx context.Context, mkfs, path string) error + // ResizeVolume runs the trusted e2fsck (-f -p) and resize2fs on a + // volume image file already extended to its new size. Nothing is mounted. + // lock is the flocked disk file; the tools inherit it (see volumeLocked). + ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string, lock *os.File) error ApplyFirewall(ctx context.Context, ruleset string) error FirewallState(ctx context.Context) (string, error) diff --git a/internal/vm/firecracker_backup.go b/internal/vm/firecracker_backup.go index b3159b3..66987ea 100644 --- a/internal/vm/firecracker_backup.go +++ b/internal/vm/firecracker_backup.go @@ -17,12 +17,14 @@ import ( ) // volumeTransfers is protected by volumeMu. A zero SizeMiB is a backup of an -// existing volume; a positive size reserves a new name and restore capacity. +// existing volume; a positive size reserves a new name and restore capacity; +// GrowTo > 0 is a grow of an existing volume from GrowFrom MiB to GrowTo MiB. // Reservations are process-local: the daemon's instance/ledger claim excludes // other writers, and restart removes incomplete .ext4.tmp imports. type fcVolumeTransfer struct { - Name string - SizeMiB int + Name string + SizeMiB int + GrowFrom, GrowTo int } func (d *FirecrackerDriver) reserveVolumeTransfer(id string, transfer fcVolumeTransfer) error { @@ -48,18 +50,30 @@ func (d *FirecrackerDriver) releaseVolumeTransfer(id string) { // reservedVolumeMiB also rejects names reserved by an in-flight restore. Count // full restore sizes against disk headroom conservatively, even if a stream has // partially allocated its staging file; a slow import must never overcommit it. -func (d *FirecrackerDriver) reservedVolumeMiB(name string) (int, error) { - reserved := 0 - for _, transfer := range d.volumeTransfers { +// +// It returns two figures. budget is what the volume budget must add to the +// volumes' footprints: restore sizes, and the part of a grow the disk file has +// not been extended by yet (the extended part is in the footprint). headroom is +// what the host disk floor must hold back: restore sizes and each grow's whole +// growth until it finishes, because extending the file is sparse and the +// allocation only happens while e2fsck and resize2fs run. +func (d *FirecrackerDriver) reservedVolumeMiB(name string) (budget, headroom int, err error) { + for id, transfer := range d.volumeTransfers { + if transfer.GrowTo > 0 { + budget += max(transfer.GrowTo-max(transfer.GrowFrom, d.diskMiB(id)), 0) + headroom += transfer.GrowTo - transfer.GrowFrom + continue + } if transfer.SizeMiB == 0 { continue } if transfer.Name == name { - return 0, ErrVolumeExists + return 0, 0, ErrVolumeExists } - reserved += transfer.SizeMiB + budget += transfer.SizeMiB + headroom += transfer.SizeMiB } - return reserved, nil + return budget, headroom, nil } func (d *FirecrackerDriver) OpenVolumeBackup(ctx context.Context, id string) (VolumeBackup, error) { @@ -274,7 +288,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if len(d.volumeTransfers) >= MaxVolumeTransfers { return fcVolume{}, ErrVolumeTransferBusy } - reserved, err := d.reservedVolumeMiB(name) + reserved, headroom, err := d.reservedVolumeMiB(name) if err != nil { return fcVolume{}, err } @@ -287,7 +301,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if volume.Name == name { return fcVolume{}, ErrVolumeExists } - used += volume.SizeMiB + used += d.volumeFootprintMiB(volume) } if used+size > d.cfg.VolumeBudgetMiB { return fcVolume{}, ErrVolumeBudget @@ -296,7 +310,7 @@ func (d *FirecrackerDriver) reserveRestore(ctx context.Context, name string, siz if err != nil { return fcVolume{}, err } - if free < uint64(d.cfg.DiskFloorMiB+size+reserved)<<20 { + if free < uint64(d.cfg.DiskFloorMiB+size+headroom)<<20 { return fcVolume{}, ErrVolumeBudget } var random [16]byte diff --git a/internal/vm/firecracker_host.go b/internal/vm/firecracker_host.go index d724220..988cd21 100644 --- a/internal/vm/firecracker_host.go +++ b/internal/vm/firecracker_host.go @@ -62,11 +62,21 @@ func newLinuxFCHost(names fcNames) *linuxFCHost { } func (h *linuxFCHost) run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) { + return h.runInheriting(ctx, nil, stdin, name, args...) +} + +// runInheriting runs a tool that also inherits keep as its fd 3. A flock on +// keep is held by the open file description, which the child shares: it stays +// locked until the tool exits, even if the daemon dies first. +func (h *linuxFCHost) runInheriting(ctx context.Context, keep *os.File, stdin []byte, name string, args ...string) ([]byte, error) { if h.toolsErr != nil { return nil, h.toolsErr } c := exec.CommandContext(ctx, name, args...) c.Env = []string{fcHostPath} + if keep != nil { + c.ExtraFiles = []*os.File{keep} + } if stdin != nil { c.Stdin = bytes.NewReader(stdin) } @@ -136,6 +146,24 @@ func (h *linuxFCHost) FormatVolume(ctx context.Context, mkfs, path string) error return err } +// ResizeVolume checks the volume's filesystem, then grows it to fill the +// image file. e2fsck exit 1 means it repaired errors, which -p only does when +// that is safe; anything else stops the grow before resize2fs, with +// errVolumeFsck so the caller knows the filesystem was not resized. lock is +// the caller's open, flocked file for path; each tool inherits it, so the +// disk stays fenced while a tool runs, even if the daemon dies (the tools do +// not die with it). +func (h *linuxFCHost) ResizeVolume(ctx context.Context, e2fsck, resize2fs, path string, lock *os.File) error { + if _, err := h.runInheriting(ctx, lock, nil, e2fsck, "-f", "-p", path); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 { + return fmt.Errorf("%w: %w", errVolumeFsck, err) + } + } + _, err := h.runInheriting(ctx, lock, nil, resize2fs, path) + return err +} + // MonitorRuleset runs `nft monitor` in the daemon's (host) namespace. Every // line it prints is one committed ruleset change; the content does not // matter, since the caller re-reads the table itself. The subprocess dies diff --git a/internal/vm/firecracker_test.go b/internal/vm/firecracker_test.go index b4a23e5..c0631c4 100644 --- a/internal/vm/firecracker_test.go +++ b/internal/vm/firecracker_test.go @@ -92,6 +92,11 @@ type fakeFCHost struct { // FormatVolume target. owners map[string]int formatted []string + // resized records each ResizeVolume call; resizeGate blocks it until + // closed and resizeErr is what it returns. + resized []string + resizeGate chan struct{} + resizeErr error // tables is each namespace's current table: the one it was created with, // or the last it was replaced with; // failReplace fails the next replacements, and readback, when set, @@ -154,6 +159,23 @@ func (h *fakeFCHost) FormatVolume(_ context.Context, mkfs, path string) error { h.formatted = append(h.formatted, mkfs+" "+path) return nil } + +// ResizeVolume stands in for e2fsck and resize2fs: it records the disk size it +// was handed, optionally blocks on resizeGate, and fails with resizeErr. +func (h *fakeFCHost) ResizeVolume(_ context.Context, e2fsck, resize2fs, path string, _ *os.File) error { + info, err := os.Stat(path) + if err != nil { + return err + } + h.mu.Lock() + h.resized = append(h.resized, fmt.Sprintf("%s %s %s %d", e2fsck, resize2fs, path, info.Size())) + gate, fail := h.resizeGate, h.resizeErr + h.mu.Unlock() + if gate != nil { + <-gate + } + return fail +} func (h *fakeFCHost) ApplyFirewall(_ context.Context, ruleset string) error { h.mu.Lock() defer h.mu.Unlock() diff --git a/internal/vm/firecracker_volume_resize.go b/internal/vm/firecracker_volume_resize.go new file mode 100644 index 0000000..c1c4084 --- /dev/null +++ b/internal/vm/firecracker_volume_resize.go @@ -0,0 +1,233 @@ +//go:build linux + +package vm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "syscall" +) + +// fcE2fsck and fcResize2fs are the host e2fsprogs candidates for a grow. They +// are resolved when a grow runs, not at startup, so a worker that never grows +// a volume needs nothing beyond mke2fs. +var ( + fcE2fsck = []string{"/usr/sbin/e2fsck", "/sbin/e2fsck"} + fcResize2fs = []string{"/usr/sbin/resize2fs", "/sbin/resize2fs"} +) + +// errVolumeFsck marks a host ResizeVolume that stopped at e2fsck: the +// filesystem was not resized, so the disk file can go back to its old size. +var errVolumeFsck = errors.New("volume filesystem check failed") + +func (d *FirecrackerDriver) trustedTool(candidates []string) (string, error) { + var untrusted error + for _, tool := range candidates { + err := d.host.TrustedFile(tool, true) + if err == nil { + return tool, nil + } + untrusted = errors.Join(untrusted, err) + } + return "", fmt.Errorf("Firecracker volume grow needs a trusted host tool: %w", untrusted) +} + +// ResizeVolume grows a detached volume to sizeMiB. The volume stays reserved +// (no attach, delete or backup) from the budget check until the new size is +// recorded. Order: extend the disk file, e2fsck -f -p, resize2fs, then replace +// the record. A failed e2fsck puts the file back. A crash, or a failed +// resize2fs, leaves a disk larger than its record; the same grow can be +// retried, because each step is idempotent and a disk larger than the request +// is refused. The host tools read a +// filesystem the guest wrote; nothing is mounted. +func (d *FirecrackerDriver) ResizeVolume(ctx context.Context, id string, sizeMiB int) (Volume, error) { + if err := d.volumesOn(); err != nil { + return Volume{}, err + } + if sizeMiB < 64 || sizeMiB > 1<<20 { + return Volume{}, fmt.Errorf("%w: %d MiB is outside 64 MiB to 1 TiB", ErrVolumeSize, sizeMiB) + } + ctx, cancel := context.WithTimeout(ctx, VolumeTransferTimeout) + defer cancel() + + d.volumeMu.Lock() + volume, err := d.readVolume(id) + if err == nil { + err = d.volumeInUse(id) + } + switch { + case err != nil: + case sizeMiB == volume.SizeMiB: + d.volumeMu.Unlock() + return Volume{ID: volume.ID, Name: volume.Name, SizeMiB: volume.SizeMiB}, nil + case sizeMiB < volume.SizeMiB: + err = fmt.Errorf("%w: a volume cannot shrink from %d to %d MiB", ErrVolumeSize, volume.SizeMiB, sizeMiB) + } + var e2fsck, resize2fs string + if err == nil { + if e2fsck, err = d.trustedTool(fcE2fsck); err == nil { + resize2fs, err = d.trustedTool(fcResize2fs) + } + } + growth := max(sizeMiB-d.volumeFootprintMiB(volume), 0) + if err == nil { + err = d.reserveGrowth(volume, growth) + } + d.volumeMu.Unlock() + if err != nil { + return Volume{}, err + } + defer d.releaseVolumeTransfer(id) + + if err := d.growDisk(ctx, id, volume.SizeMiB, sizeMiB, e2fsck, resize2fs); err != nil { + return Volume{}, err + } + volume.SizeMiB = sizeMiB + d.volumeMu.Lock() + defer d.volumeMu.Unlock() + // If this fails, the enlarged disk keeps counting against the budget: + // accounting reads the disk file's size, not only the record. + if err := d.replaceVolumeRecord(volume); err != nil { + return Volume{}, err + } + return Volume{ID: volume.ID, Name: volume.Name, SizeMiB: volume.SizeMiB}, nil +} + +// reserveGrowth checks the volume budget and the host disk floor for growth +// MiB more than the volume's footprint, then reserves the grow, so a +// concurrent create counts it: the not yet extended remainder through the +// reservation, the extended part through the disk file. Callers hold volumeMu. +func (d *FirecrackerDriver) reserveGrowth(volume fcVolume, growth int) error { + target := d.volumeFootprintMiB(volume) + growth + volumes, err := d.readVolumes() + if err != nil { + return err + } + reserved, headroom, err := d.reservedVolumeMiB(volume.Name) + if err != nil { + return err + } + used := reserved + for _, existing := range volumes { + used += d.volumeFootprintMiB(existing) + } + if used+growth > d.cfg.VolumeBudgetMiB { + return fmt.Errorf("%w: %d of %d MiB used, growing %s takes %d MiB more", ErrVolumeBudget, used, d.cfg.VolumeBudgetMiB, volume.ID, growth) + } + free, err := d.host.FreeBytes(d.cfg.VolumeDir) + if err != nil { + return err + } + if need := uint64(d.cfg.DiskFloorMiB+growth+headroom) << 20; free < need { + return fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB growth needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, growth) + } + return d.reserveVolumeTransfer(volume.ID, fcVolumeTransfer{Name: volume.Name, GrowFrom: volume.SizeMiB, GrowTo: target}) +} + +// growDisk extends the detached disk file and its filesystem. A disk larger +// than the target is not a state this grow made: it is refused, not cut. +func (d *FirecrackerDriver) growDisk(ctx context.Context, id string, oldMiB, newMiB int, e2fsck, resize2fs string) error { + fd, err := syscall.Open(d.volumeDisk(id), syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return err + } + file := os.NewFile(uintptr(fd), "volume-grow") + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || stat.Nlink != 1 || stat.Uid != uint32(os.Geteuid()) { + return errors.New("volume disk is not a detached native disk") + } + // Fence the disk until the last tool has exited. The tools inherit this + // open file, so the lock outlives the daemon if it is killed mid-grow; a + // restarted daemon sees the disk as busy (volumeInUse) until they finish. + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return ErrVolumeTransferBusy + } + return err + } + size, target := int64(oldMiB)<<20, int64(newMiB)<<20 + if info.Size() < size || info.Size() > target { + return fmt.Errorf("%w: disk is %d bytes, expected %d to %d", ErrVolumeSize, info.Size(), size, target) + } + extended := info.Size() < target + if extended { + if err := file.Truncate(target); err != nil { + return err + } + } + if err := file.Sync(); err != nil { + return err + } + err = d.host.ResizeVolume(ctx, e2fsck, resize2fs, d.volumeDisk(id), file) + if errors.Is(err, errVolumeFsck) && extended { + // Nothing was resized: put the file back, so the volume still matches + // its record and can be backed up or attached. + err = errors.Join(err, file.Truncate(info.Size()), file.Sync()) + } + return err +} + +// replaceVolumeRecord swaps the record in with a rename, so a reader or a +// crash sees the old size or the new one, never a partial record. Callers +// hold volumeMu. +func (d *FirecrackerDriver) replaceVolumeRecord(volume fcVolume) error { + data, err := json.Marshal(volume) + if err != nil { + return err + } + record := d.volumeRecord(volume.ID) + tmp := record + ".tmp" + file, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return err + } + _, err = file.Write(data) + err = errors.Join(err, file.Sync(), file.Close()) + if err == nil { + err = os.Rename(tmp, record) + } + if err != nil { + return errors.Join(err, fcRemoveMissing(tmp)) + } + return fcSyncPath(d.cfg.VolumeDir) +} + +// volumeLocked reports ErrVolumeTransferBusy while any process, this daemon or +// a e2fsck/resize2fs that outlived a previous one, holds the disk's flock. +func (d *FirecrackerDriver) volumeLocked(id string) error { + fd, err := syscall.Open(d.volumeDisk(id), syscall.O_RDONLY|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return err + } + defer syscall.Close(fd) + if err := syscall.Flock(fd, syscall.LOCK_SH|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return fmt.Errorf("%w: a filesystem tool still holds %s", ErrVolumeTransferBusy, id) + } + return err + } + return syscall.Flock(fd, syscall.LOCK_UN) +} + +// volumeFootprintMiB is what a volume costs the budget: its recorded size, or +// its disk file when that is larger (a grow whose record was not published). +func (d *FirecrackerDriver) volumeFootprintMiB(volume fcVolume) int { + return max(volume.SizeMiB, d.diskMiB(volume.ID)) +} + +// diskMiB is a volume's disk file size rounded up to MiB, 0 if unreadable. +func (d *FirecrackerDriver) diskMiB(id string) int { + info, err := os.Lstat(d.volumeDisk(id)) + if err != nil { + return 0 + } + return int((info.Size() + 1<<20 - 1) >> 20) +} diff --git a/internal/vm/firecracker_volume_resize_test.go b/internal/vm/firecracker_volume_resize_test.go new file mode 100644 index 0000000..210bf98 --- /dev/null +++ b/internal/vm/firecracker_volume_resize_test.go @@ -0,0 +1,453 @@ +//go:build linux + +package vm + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" +) + +func volumeDiskSize(t *testing.T, cfg FirecrackerConfig, id string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(cfg.VolumeDir, id+".ext4")) + if err != nil { + t.Fatal(err) + } + return info.Size() +} + +func TestFirecrackerVolumeGrow(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + disk := filepath.Join(cfg.VolumeDir, volume.ID+".ext4") + + grown, err := d.ResizeVolume(ctx, volume.ID, 192) + if err != nil || grown.SizeMiB != 192 || grown.ID != volume.ID || grown.Name != "data" { + t.Fatalf("grow = %+v %v", grown, err) + } + // The host tools saw the already-extended file, and the record agrees. + if want := fcE2fsck[0] + " " + fcResize2fs[0] + " " + disk + " " + "201326592"; len(host.resized) != 1 || host.resized[0] != want { + t.Fatalf("resized = %v, want %q", host.resized, want) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size = %d", got) + } + volumes, err := d.Volumes(ctx) + if err != nil || len(volumes) != 1 || volumes[0].SizeMiB != 192 { + t.Fatalf("volumes = %+v %v", volumes, err) + } + // The grown size counts against the budget (256 MiB): 64 MiB are left. + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create after grow: %v", err) + } + // The same size again does nothing. + if again, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || again.SizeMiB != 192 || len(host.resized) != 1 { + t.Fatalf("repeat = %+v %v calls=%v", again, err, host.resized) + } + // No stray record temp file is left behind. + if _, err := os.Stat(filepath.Join(cfg.VolumeDir, volume.ID+".json.tmp")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("record temp file remains: %v", err) + } +} + +func TestFirecrackerVolumeGrowRefusals(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + for name, test := range map[string]struct { + id string + size int + want error + }{ + "shrink": {volume.ID, 64, ErrVolumeSize}, + "too small": {volume.ID, 63, ErrVolumeSize}, + "too large": {volume.ID, 1<<20 + 1, ErrVolumeSize}, + "over budget": {volume.ID, 300, ErrVolumeBudget}, + "unknown": {"vol-" + strings.Repeat("e", 32), 192, ErrVolumeNotFound}, + "malformed id": {"../data", 192, ErrVolumeNotFound}, + } { + if _, err := d.ResizeVolume(ctx, test.id, test.size); !errors.Is(err, test.want) { + t.Fatalf("%s: %v, want %v", name, err, test.want) + } + } + // The host disk floor counts too. + host.free = uint64(cfg.DiskFloorMiB+63) << 20 + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("below floor: %v", err) + } + host.free = 100 << 30 + // An untrusted e2fsck or resize2fs stops the grow before the file moves. + for _, tool := range []string{fcE2fsck[0], fcResize2fs[0]} { + host.untrusted = map[string]bool{tool: true, fcE2fsck[1]: true, fcResize2fs[1]: true} + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil || !strings.Contains(err.Error(), "trusted host tool") { + t.Fatalf("untrusted %s: %v", tool, err) + } + } + host.untrusted = nil + if len(host.resized) != 0 || volumeDiskSize(t, cfg, volume.ID) != 128<<20 { + t.Fatalf("a refused grow changed the volume: calls=%v size=%d", host.resized, volumeDiskSize(t, cfg, volume.ID)) + } + // An attached volume cannot grow. + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, ErrVolumeInUse) { + t.Fatalf("attached: %v", err) + } + if len(host.resized) != 0 || volumeDiskSize(t, cfg, volume.ID) != 128<<20 { + t.Fatal("an attached volume was changed") + } +} + +// A filesystem that fails its check is not resized, and its file goes back to +// the recorded size, so the volume is exactly as before. +func TestFirecrackerVolumeGrowFsckFailureRestoresDisk(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + host.resizeErr = fmt.Errorf("%w: exit status 4", errVolumeFsck) + if _, err := d.ResizeVolume(ctx, volume.ID, 192); !errors.Is(err, errVolumeFsck) { + t.Fatalf("grow of a damaged filesystem: %v", err) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 128<<20 { + t.Fatalf("disk size after a failed check = %d", got) + } + host.resizeErr = nil + if grown, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || grown.SizeMiB != 192 { + t.Fatalf("grow after the failed check = %+v %v", grown, err) + } +} + +// A failed filesystem step leaves the disk file larger than its record. The +// same grow then completes, since each step can be repeated. +func TestFirecrackerVolumeGrowRetryAfterFailure(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + host.resizeErr = errors.New("e2fsck: errors left uncorrected") + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil { + t.Fatal("failed resize reported success") + } + if volumes, _ := d.Volumes(ctx); len(volumes) != 1 || volumes[0].SizeMiB != 128 { + t.Fatalf("record changed by a failed grow: %+v", volumes) + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size after failure = %d", got) + } + // The failed grow released its reservation: the volume can be deleted or retried. + host.resizeErr = nil + grown, err := d.ResizeVolume(ctx, volume.ID, 192) + if err != nil || grown.SizeMiB != 192 { + t.Fatalf("retry = %+v %v", grown, err) + } + // A disk already larger than the request is not cut down. + if err := os.Truncate(filepath.Join(cfg.VolumeDir, volume.ID+".ext4"), 250<<20); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeSize) { + t.Fatalf("oversized disk: %v", err) + } +} + +// While e2fsck and resize2fs run, the volume is held: it cannot be attached, +// deleted or backed up, and the growth counts against the budget. +func TestFirecrackerVolumeGrowHoldsVolume(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + if err := d.DeleteVolume(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("delete during grow: %v", err) + } + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("attach during grow: %v", err) + } + if _, err := d.OpenVolumeBackup(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("backup during grow: %v", err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("second grow: %v", err) + } + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create during grow: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } + if err := d.DeleteVolume(ctx, volume.ID); err != nil { + t.Fatalf("delete after grow: %v", err) + } +} + +// A grow whose record could not be published leaves an enlarged disk. It keeps +// counting against the budget (also after a restart, which re-reads the disk), +// so a create cannot admit more disk than the budget allows. +func TestFirecrackerVolumeGrowFailedRecordKeepsBudget(t *testing.T) { + d, _, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + // A directory where the record's temp file goes makes publication fail. + block := filepath.Join(cfg.VolumeDir, volume.ID+".json.tmp") + if err := os.Mkdir(block, 0o700); err != nil { + t.Fatal(err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 192); err == nil { + t.Fatal("grow reported success although its record was not written") + } + if got := volumeDiskSize(t, cfg, volume.ID); got != 192<<20 { + t.Fatalf("disk size = %d", got) + } + // Budget 256 MiB: the 192 MiB disk leaves room for no second 128 MiB volume. + if _, err := d.CreateVolume(ctx, "other"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create next to an unrecorded 192 MiB disk: %v", err) + } + if err := os.Remove(block); err != nil { + t.Fatal(err) + } + // The same request finishes the grow, without counting the extension twice. + if grown, err := d.ResizeVolume(ctx, volume.ID, 192); err != nil || grown.SizeMiB != 192 { + t.Fatalf("retry = %+v %v", grown, err) + } +} + +func lockedByAnother(t *testing.T, path string) bool { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + err = syscall.Flock(int(file.Fd()), syscall.LOCK_SH|syscall.LOCK_NB) + if err == nil { + return false + } + if !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatal(err) + } + return true +} + +// The fence is the disk's flock, not process memory: with the in-memory +// reservation gone (a restarted daemon has none), a disk a tool still holds +// cannot be attached, deleted, backed up or grown. +func TestFirecrackerVolumeGrowFencedByDiskLock(t *testing.T) { + d, host, cfg := newVolumeFirecracker(t) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + disk := filepath.Join(cfg.VolumeDir, volume.ID+".ext4") + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for !lockedByAnother(t, disk) { + if time.Now().After(deadline) { + t.Fatal("the disk was not locked while the tools ran") + } + time.Sleep(time.Millisecond) + } + d.volumeMu.Lock() + clear(d.volumeTransfers) + d.volumeMu.Unlock() + if _, err := d.Create(ctx, volumeSpec(cfg, fcTestID, "sbx1", volume.ID)); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("attach with the tools running: %v", err) + } + if err := d.DeleteVolume(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("delete with the tools running: %v", err) + } + if _, err := d.OpenVolumeBackup(ctx, volume.ID); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("backup with the tools running: %v", err) + } + if _, err := d.ResizeVolume(ctx, volume.ID, 224); !errors.Is(err, ErrVolumeTransferBusy) { + t.Fatalf("second grow with the tools running: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } + if lockedByAnother(t, disk) { + t.Fatal("the disk is still locked after the grow") + } +} + +// The real host passes the locked file to e2fsck and resize2fs. The tool keeps +// working after the daemon's own descriptor is closed, which is what a daemon +// killed mid-grow leaves behind: here resize2fs leaves a child that still runs +// after the call returned and the daemon's descriptor was closed. +func TestLinuxHostResizeToolsKeepDiskLocked(t *testing.T) { + dir := t.TempDir() + disk := filepath.Join(dir, "v.ext4") + if err := os.WriteFile(disk, make([]byte, 4096), 0o600); err != nil { + t.Fatal(err) + } + script := func(name, body string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o700); err != nil { + t.Fatal(err) + } + return path + } + fsck := script("e2fsck", "exit 0\n") + resize := script("resize2fs", "sleep 1 /dev/null 2>&1 &\nexit 0\n") + + lock, err := os.OpenFile(disk, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatal(err) + } + if err := (&linuxFCHost{}).ResizeVolume(context.Background(), fsck, resize, disk, lock); err != nil { + t.Fatal(err) + } + if err := lock.Close(); err != nil { + t.Fatal(err) + } + if !lockedByAnother(t, disk) { + t.Fatal("the disk was not locked by the tool after the daemon's descriptor closed") + } + deadline := time.Now().Add(10 * time.Second) + for lockedByAnother(t, disk) { + if time.Now().After(deadline) { + t.Fatal("the disk stayed locked after the tool exited") + } + time.Sleep(20 * time.Millisecond) + } +} + +// While a grow runs, the budget charges the volume's growth once. The extended +// disk file and the reserved growth must not both count: a create that fits +// next to the grown volume is admitted, one that does not is refused. +func TestFirecrackerVolumeGrowInFlightChargedOnce(t *testing.T) { + d, host, _ := newTestFirecracker(t, func(c *FirecrackerConfig) { + c.VolumeBudgetMiB, c.VolumeMiB, c.VolumeDir = 320, 128, filepath.Join(c.Root, "volumes") + }) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + // 192 MiB growing + 128 MiB new = the 320 MiB budget exactly. + if _, err := d.CreateVolume(ctx, "next"); err != nil { + t.Fatalf("a volume that fits next to the growing one was refused: %v", err) + } + if _, err := d.CreateVolume(ctx, "over"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("create past the budget during a grow: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +// Extending the disk file is sparse: the space is only used while the tools +// run. The host disk floor therefore keeps holding the grow's whole growth +// back until the grow ends, also after the file reached its new size. +func TestFirecrackerVolumeGrowKeepsDiskHeadroom(t *testing.T) { + d, host, cfg := newTestFirecracker(t, func(c *FirecrackerConfig) { + c.VolumeBudgetMiB, c.VolumeMiB, c.VolumeDir = 1024, 128, filepath.Join(c.Root, "volumes") + }) + ctx := context.Background() + volume, err := d.CreateVolume(ctx, "data") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + host.resizeGate = gate + done := make(chan error, 1) + go func() { _, err := d.ResizeVolume(ctx, volume.ID, 192); done <- err }() + deadline := time.Now().Add(5 * time.Second) + for { + host.mu.Lock() + started := len(host.resized) == 1 + host.mu.Unlock() + if started { + break + } + if time.Now().After(deadline) { + t.Fatal("resize did not reach the host") + } + time.Sleep(time.Millisecond) + } + // The file is already 192 MiB, but 64 MiB of it is still unallocated: free + // space for the floor plus a new 128 MiB volume is not enough with it. + host.mu.Lock() + host.free = uint64(cfg.DiskFloorMiB+128+63) << 20 + host.mu.Unlock() + if _, err := d.CreateVolume(ctx, "next"); !errors.Is(err, ErrVolumeBudget) { + t.Fatalf("a create that would eat the grow's headroom: %v", err) + } + host.mu.Lock() + host.free = uint64(cfg.DiskFloorMiB+128+64) << 20 + host.mu.Unlock() + if _, err := d.CreateVolume(ctx, "next"); err != nil { + t.Fatalf("a create that leaves the grow's headroom: %v", err) + } + close(gate) + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/vm/firecracker_volumes.go b/internal/vm/firecracker_volumes.go index 1b42c6c..fc191b4 100644 --- a/internal/vm/firecracker_volumes.go +++ b/internal/vm/firecracker_volumes.go @@ -116,7 +116,7 @@ func (d *FirecrackerDriver) recoverVolumes() error { } for _, name := range names { path := filepath.Join(cfg.VolumeDir, name) - if strings.HasSuffix(name, ".ext4.tmp") { + if strings.HasSuffix(name, ".ext4.tmp") || strings.HasSuffix(name, ".json.tmp") { if err := os.Remove(path); err != nil { return err } @@ -255,7 +255,7 @@ func (d *FirecrackerDriver) volumeInUse(id string) error { if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Nlink != 1 { return fmt.Errorf("%w: a jail still links %s", ErrVolumeInUse, id) } - return nil + return d.volumeLocked(id) } // CreateVolume makes a VolumeMiB ext4 disk named name. @@ -272,7 +272,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - reserved, err := d.reservedVolumeMiB(name) + reserved, headroom, err := d.reservedVolumeMiB(name) if err != nil { return Volume{}, err } @@ -281,7 +281,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if volume.Name == name { return Volume{}, fmt.Errorf("%w: %q", ErrVolumeExists, name) } - used += volume.SizeMiB + used += d.volumeFootprintMiB(volume) } size := d.cfg.VolumeMiB if used+size > d.cfg.VolumeBudgetMiB { @@ -291,7 +291,7 @@ func (d *FirecrackerDriver) CreateVolume(ctx context.Context, name string) (Volu if err != nil { return Volume{}, err } - if need := uint64(d.cfg.DiskFloorMiB+size+reserved) << 20; free < need { + if need := uint64(d.cfg.DiskFloorMiB+size+headroom) << 20; free < need { return Volume{}, fmt.Errorf("%w: %d MiB free, %d MiB floor plus %d MiB volume needed", ErrVolumeBudget, free>>20, d.cfg.DiskFloorMiB, size) } diff --git a/internal/vm/volume.go b/internal/vm/volume.go index 5473c28..439566f 100644 --- a/internal/vm/volume.go +++ b/internal/vm/volume.go @@ -17,6 +17,9 @@ var ( // ErrVolumeInUse: the volume is attached to a VM (any trace of one: a // running, stopped or half-destroyed VM still holds it). ErrVolumeInUse = errors.New("volume is attached to a sandbox") + // ErrVolumeSize: the requested volume size is invalid, or smaller than + // the current one. + ErrVolumeSize = errors.New("invalid volume size") // ErrVolumeBudget: creating the volume would exceed the total volume // budget or the host disk floor. ErrVolumeBudget = errors.New("volume budget exhausted") @@ -53,6 +56,15 @@ type Volumes interface { DeleteVolume(ctx context.Context, id string) error } +// VolumeResizer is implemented by drivers that can grow a detached volume in +// place (the Linux Firecracker driver). It never shrinks: a smaller size +// fails with ErrVolumeSize, the same size is a no-op, and an attached, +// transferring or missing volume fails with ErrVolumeInUse, +// ErrVolumeTransferBusy or ErrVolumeNotFound. +type VolumeResizer interface { + ResizeVolume(ctx context.Context, id string, sizeMiB int) (Volume, error) +} + var ( volumeName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}$`) volumeID = regexp.MustCompile(`^vol-[0-9a-f]{32}$`)