From 446ac8326b18a521a1c7cd9795168313c0150350 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sat, 31 Oct 2026 10:38:45 +0100 Subject: [PATCH 1/2] Add scoped renewable guest TCP tunnels with directional EOF --- docs/compatibility.md | 52 +++++ internal/control/control.go | 14 +- internal/control/tunnels.go | 333 ++++++++++++++++++++++++++++ internal/control/tunnels_test.go | 285 ++++++++++++++++++++++++ internal/control/workers.go | 1 + internal/envd/proxy.go | 4 + internal/envd/tunnels.go | 81 +++++++ internal/guestagent/client.go | 12 +- internal/guestagent/protocol.go | 3 + internal/guestagent/server_linux.go | 23 ++ internal/guestagent/tunnel_test.go | 134 +++++++++++ internal/tunnel/stream.go | 180 +++++++++++++++ internal/vm/devvm/envd.go | 11 +- internal/vm/firecracker.go | 21 +- internal/vm/vm.go | 6 + internal/worker/client.go | 19 +- internal/worker/server.go | 30 ++- internal/worker/tunnel_test.go | 89 ++++++++ internal/worker/worker.go | 8 + 19 files changed, 1288 insertions(+), 18 deletions(-) create mode 100644 internal/control/tunnels.go create mode 100644 internal/control/tunnels_test.go create mode 100644 internal/envd/tunnels.go create mode 100644 internal/guestagent/tunnel_test.go create mode 100644 internal/tunnel/stream.go create mode 100644 internal/worker/tunnel_test.go diff --git a/docs/compatibility.md b/docs/compatibility.md index 89edf6d..fd950c0 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -198,6 +198,58 @@ Guest ports other than envd's are **allowlist-only per template** and **never pu - `get_host(port)` returns `-.`; the SDKs never apply `E2B_SANDBOX_URL` to it. That form needs wildcard DNS and TLS for `*.` in front of sandboxd. Owner decision D2 defers wildcard DNS, so host routing of guest ports is **off by default** in `sandboxd` and `sandboxd-dev` and `get_host` URLs do not resolve in a default deployment; the code-interpreter SDKs use the routing headers instead. `-port-hosts` turns host routing on once an operator has both. The token rules are the same; envd's own `49983-.` host stays routed as before. For a client on the gateway's own machine, a domain under `localhost` (for example `sbx.localhost:`) resolves without DNS setup on resolvers that implement RFC 6761, but still needs a TLS front. - Signed file URLs (`download_url`/`upload_url`) are built from `E2B_SANDBOX_URL` and carry no routing headers. sandboxd routes a `/files` request with a `signature` on the gateway host to the one running e2b sandbox whose envd token made that signature (the signature binds path, operation, user, expiry and token), so no wildcard DNS is needed. Such a request proves nothing until a signature matches, so a signature of the wrong shape or with an unparsable expiration is refused before any lookup, as is an expired one (envd's own `401 {"code":401,"message":"signature is already expired"}`), and the match reads only an in-memory index of running e2b sandboxes (kept on create, delete, reconcile and restart), never the ledger or the control lock. - Template start and ready commands: `start-cmd=` runs once per sandbox as root through envd right after `/init`, in the background; `ready-cmd=` then runs as root until it exits 0 before create returns 201. Both together are bounded by `-template-ready-timeout` (default 3m); a sandbox not ready by then is destroyed (503). Give the client's create request a longer timeout than a cold start (the stock SDKs default to 60 s). E2B runs the start command at template build time and snapshots the result; sandboxd has no snapshots, so this is a per-create cold start. The `code-interpreter-v1` image and its registration line are in [firecracker.md](firecracker.md#code-interpreter-image). +### Raw guest TCP tunnels + +HTTP/1.1 CONNECT exposes a template-allowlisted guest-loopback TCP port to a +trusted adapter (for example, an SSH host/port adapter). It is not an HTTP proxy +to arbitrary hosts, an exec relay, or a public unauthenticated port. Keep the +control endpoint on a trusted network or behind authenticated TLS. Customer SSH +authentication and host-key checking remain end-to-end; the adapter holds the +tunnel credential, never the guest or customer. + +The controller uses its usual `X-API-Key` for these routes: + +| Method and path | Request | Result | +|---|---|---| +| `POST /sandboxes/{id}/tunnel-leases` | `{"port":22,"ttl":300}` | 201: `leaseID`, `sandboxID`, `port`, `token`, `expiresAt` | +| `POST /sandboxes/{id}/tunnel-leases/{leaseID}/renew` | `{"ttl":300}` | 200: same identity/token, extended expiry | +| `DELETE /sandboxes/{id}/tunnel-leases/{leaseID}` | none | 204, idempotent | + +TTL is positive integer seconds, capped by the daemon's maximum TTL and the +sandbox's current expiry. Renew the sandbox first, then its tunnel leases. +Renewal extends only a still-live lease; expired/revoked leases return 403 and +cannot revive. There are at most 256 live leases per sandbox. Leases are +memory-only: daemon restart invalidates all of them. Provision a new lease +after restart or revocation; renewal preserves established connections. + +The adapter opens a dedicated HTTP/1.1 connection and sends: + +```http +CONNECT sandboxd-0123456789abcdef0123456789abcdef:22 HTTP/1.1 +Host: sandboxd-0123456789abcdef0123456789abcdef:22 +E2b-Tunnel-Access-Token: + +``` + +After `200 Connection Established`, send raw bytes on that socket. No request +body, TLS inside the adapter protocol, or WebSocket framing is required by +CONNECT itself. The gateway resolves only the sandbox ID and its guest-loopback +port; it never resolves the authority as a host address. The lease must match +both ID and port. Control keys, traffic tokens, and envd tokens are not fallback +CONNECT credentials, and no HTTP headers reach the guest. + +Directional EOF preserves reverse traffic until the reverse side ends. Errors, +lease revoke/expiry, sandbox teardown, and daemon shutdown close both directions. +Disconnect alone releases the stream without deleting the sandbox or lease. +CONNECT shares the per-sandbox `-port-max-streams` cap with HTTP/WebSocket guest +ports (429 when full). Invalid/expired/revoked leases return 403; an unavailable +guest port or unsupported guest tunnel transport returns 502. + +Firecracker images must contain the matching guest agent with the framed +`tunnel` operation. Older agents fail explicitly, without degraded half-close. +Remote workers must also support `sandboxd-tunnel-v1`; mixed versions refuse +the upgrade. The Apple driver does not implement this tunnel operation. + ### CI conformance gate (dev driver) diff --git a/internal/control/control.go b/internal/control/control.go index 47a53bc..e5aa44f 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -105,7 +105,8 @@ type Service struct { // volumes is Config.Volumes: nil when volumes are off. volumes vm.Volumes // egress is Config.Egress: nil when egress policies are off. - egress vm.EgressPolicies + egress vm.EgressPolicies + tunnels tunnelLeases } // Open creates the durable ledger before accepting any VM allocations. The @@ -236,6 +237,7 @@ func bootsHardened(driver vm.Driver) bool { func (s *Service) setState(ctx context.Context, id, state string) error { err := s.ledger.SetState(ctx, id, state) if state != "running" { + s.revokeSandboxTunnels(id) s.signersMu.Lock() delete(s.signers, id) s.signersMu.Unlock() @@ -269,6 +271,7 @@ func (s *Service) clearBusy(id string) { } func (s *Service) Close() error { + s.closeTunnels() close(s.stop) <-s.done s.mu.Lock() @@ -395,6 +398,9 @@ func (s *Service) serve(w http.ResponseWriter, r *http.Request) { return } path := r.URL.Path + if s.templates.e2b && s.serveTunnelLeases(w, r) { + return + } if path == "/sandboxes" && r.Method == http.MethodPost { s.create(w, r) return @@ -1028,7 +1034,11 @@ func (s *Service) extend(ctx context.Context, id string, m *member, ends time.Ti if onlyLater { return s.ledger.ExtendAtLeast(ctx, id, ends) } - return s.ledger.Extend(ctx, id, ends) + if err := s.ledger.Extend(ctx, id, ends); err != nil { + return err + } + s.capTunnelLeases(id, ends) + return nil } // delete confirms teardown through the sandbox's own worker. An offline diff --git a/internal/control/tunnels.go b/internal/control/tunnels.go new file mode 100644 index 0000000..d446063 --- /dev/null +++ b/internal/control/tunnels.go @@ -0,0 +1,333 @@ +package control + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "net" + "net/http" + "slices" + "strings" + "sync" + "time" + + "github.com/gitmoot/sandboxd/internal/envd" + "github.com/gitmoot/sandboxd/internal/vm" +) + +// Leases are process-local capabilities, separate from envd/traffic credentials. +// The lock order is Service.mu then tunnels.mu, never the reverse. Revocation +// removes the capability before closing streams, including pending dials. +const maxTunnelLeases = 256 + +type tunnelLeases struct { + mu sync.Mutex + closed bool + byID map[string]*tunnelLease + byToken map[[32]byte]*tunnelLease +} + +type tunnelLease struct { + ID string `json:"leaseID"` + SandboxID string `json:"sandboxID"` + Port int `json:"port"` + ExpiresAt time.Time `json:"expiresAt"` + hash [32]byte + timer *time.Timer + sessions map[*tunnelSession]struct{} +} + +type tunnelSession struct { + ctx context.Context + cancel context.CancelFunc + conn net.Conn +} + +type tunnelLeaseResponse struct { + LeaseID string `json:"leaseID"` + SandboxID string `json:"sandboxID"` + Port int `json:"port"` + ExpiresAt time.Time `json:"expiresAt"` + Token string `json:"token"` +} + +func (s *Service) tunnelResponse(l *tunnelLease) tunnelLeaseResponse { + return tunnelLeaseResponse{l.ID, l.SandboxID, l.Port, l.ExpiresAt, s.derive("sandboxd tunnel lease v1", l.ID)} +} + +// serveTunnelLeases is reached only after the normal control-key check. +func (s *Service) serveTunnelLeases(w http.ResponseWriter, r *http.Request) bool { + rest, ok := strings.CutPrefix(r.URL.Path, "/sandboxes/") + if !ok { + return false + } + parts := strings.Split(rest, "/") + if len(parts) < 2 || parts[1] != "tunnel-leases" { + return false + } + if !validID(parts[0]) { + s.notFound(w, r) + return true + } + id := parts[0] + switch { + case len(parts) == 2 && r.Method == http.MethodPost: + s.mintTunnelLease(w, r, id, "") + case len(parts) == 4 && parts[3] == "renew" && r.Method == http.MethodPost: + s.mintTunnelLease(w, r, id, parts[2]) + case len(parts) == 3 && r.Method == http.MethodDelete: + s.tunnels.mu.Lock() + if l := s.tunnels.byID[parts[2]]; l != nil && l.SandboxID == id { + s.revokeTunnelLocked(l) + } + s.tunnels.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + s.notFound(w, r) + } + return true +} + +func (s *Service) mintTunnelLease(w http.ResponseWriter, r *http.Request, id, renew string) { + var request struct { + Port int `json:"port"` + TTL int64 `json:"ttl"` + } + if !readJSON(w, r, &request) || request.TTL <= 0 || (renew == "" && !vm.ValidPort(request.Port)) || (renew != "" && request.Port != 0) { + s.fail(w, ProfileE2B, http.StatusBadRequest, "invalid tunnel port or ttl") + return + } + if _, _, err := s.live(r.Context(), id); err != nil { + s.fail(w, ProfileE2B, statusFor(err), "sandbox unavailable") + return + } + s.mu.Lock() + defer s.mu.Unlock() + row, m, err := s.owned(r.Context(), id) + template := s.templates.byID[row.TemplateID] + if err != nil || row.State != "running" || s.busy[id] != nil || rowProfile(row.Profile) != ProfileE2B { + s.fail(w, ProfileE2B, http.StatusForbidden, "sandbox unavailable") + return + } + if _, ok := m.api.(vm.TunnelDialer); !ok { + s.fail(w, ProfileE2B, http.StatusNotImplemented, "worker has no tunnel transport") + return + } + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + if s.tunnels.closed { + s.fail(w, ProfileE2B, http.StatusServiceUnavailable, "service closing") + return + } + now := time.Now().UTC() + expires := now.Add(time.Duration(min(request.TTL, int64(s.cfg.MaxTTL/time.Second))) * time.Second) + if row.Ends.Before(expires) { + expires = row.Ends + } + if !now.Before(expires) { + s.fail(w, ProfileE2B, http.StatusForbidden, "sandbox expired") + return + } + if renew != "" { + l := s.tunnels.byID[renew] + if l == nil || l.SandboxID != id || !now.Before(l.ExpiresAt) { + if l != nil && l.SandboxID == id { + s.revokeTunnelLocked(l) + } + s.fail(w, ProfileE2B, http.StatusForbidden, "tunnel lease expired or revoked") + return + } + if !slices.Contains(template.Ports, l.Port) { + s.revokeTunnelLocked(l) + s.fail(w, ProfileE2B, http.StatusForbidden, "port not exposed") + return + } + // Renewal extends a live lease, never shortens it. Sandbox shortening is + // enforced separately by capTunnelLeases when the sandbox timeout changes. + if expires.After(l.ExpiresAt) { + l.ExpiresAt = expires + l.timer.Reset(time.Until(expires)) + } + jsonResponse(w, http.StatusOK, s.tunnelResponse(l)) + return + } + if !slices.Contains(template.Ports, request.Port) { + s.fail(w, ProfileE2B, http.StatusForbidden, "port not exposed") + return + } + count := 0 + for _, l := range s.tunnels.byID { + if !now.Before(l.ExpiresAt) { + s.revokeTunnelLocked(l) + continue + } + if l.SandboxID == id { + count++ + } + } + if count >= maxTunnelLeases { + s.fail(w, ProfileE2B, http.StatusTooManyRequests, "too many sandbox tunnel leases") + return + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + s.fail(w, ProfileE2B, http.StatusServiceUnavailable, "lease creation failed") + return + } + l := &tunnelLease{ID: hex.EncodeToString(random[:]), SandboxID: id, Port: request.Port, ExpiresAt: expires, sessions: make(map[*tunnelSession]struct{})} + response := s.tunnelResponse(l) + l.hash = sha256.Sum256([]byte(response.Token)) + if s.tunnels.byID == nil { + s.tunnels.byID = make(map[string]*tunnelLease) + s.tunnels.byToken = make(map[[32]byte]*tunnelLease) + } + s.tunnels.byID[l.ID], s.tunnels.byToken[l.hash] = l, l + l.timer = time.AfterFunc(time.Until(expires), func() { s.expireTunnel(l) }) + jsonResponse(w, http.StatusCreated, response) +} + +func (s *Service) expireTunnel(l *tunnelLease) { + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + if s.tunnels.byID[l.ID] != l { + return + } + if delay := time.Until(l.ExpiresAt); delay > 0 { + l.timer.Reset(delay) + return + } + s.revokeTunnelLocked(l) +} + +func (s *Service) revokeTunnelLocked(l *tunnelLease) { + delete(s.tunnels.byID, l.ID) + delete(s.tunnels.byToken, l.hash) + l.timer.Stop() + for session := range l.sessions { + session.cancel() + if session.conn != nil { + _ = session.conn.Close() + } + } +} + +func (s *Service) revokeSandboxTunnels(id string) { + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + for _, l := range s.tunnels.byID { + if l.SandboxID == id { + s.revokeTunnelLocked(l) + } + } +} + +func (s *Service) capTunnelLeases(id string, ends time.Time) { + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + for _, l := range s.tunnels.byID { + if l.SandboxID == id && ends.Before(l.ExpiresAt) { + l.ExpiresAt = ends + l.timer.Reset(max(0, time.Until(ends))) + } + } +} + +func (s *Service) closeTunnels() { + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + s.tunnels.closed = true + for _, l := range s.tunnels.byID { + s.revokeTunnelLocked(l) + } +} + +// OpenTunnel authenticates the lease before consulting the worker. Registration +// precedes dialing, so revoke cannot miss an in-flight connection. +func (s *Service) OpenTunnel(ctx context.Context, id string, port int, token string) (envd.TunnelConn, error) { + hash := sha256.Sum256([]byte(token)) + s.tunnels.mu.Lock() + l := s.tunnels.byToken[hash] + valid := l != nil && l.SandboxID == id && l.Port == port && time.Now().Before(l.ExpiresAt) + s.tunnels.mu.Unlock() + if !valid { + return nil, envd.ErrTunnelUnauthorized + } + if _, _, err := s.live(ctx, id); err != nil { + return nil, envd.ErrTunnelUnauthorized + } + s.mu.Lock() + row, m, err := s.owned(ctx, id) + if err != nil || row.State != "running" || s.busy[id] != nil || !time.Now().Before(row.Ends) || !slices.Contains(s.templates.byID[row.TemplateID].Ports, port) { + s.mu.Unlock() + return nil, envd.ErrTunnelUnauthorized + } + dialer, ok := m.api.(vm.TunnelDialer) + if !ok { + s.mu.Unlock() + return nil, errors.New("worker has no tunnel transport") + } + s.tunnels.mu.Lock() + if s.tunnels.byToken[hash] != l || !time.Now().Before(l.ExpiresAt) || s.tunnels.closed { + s.tunnels.mu.Unlock() + s.mu.Unlock() + return nil, envd.ErrTunnelUnauthorized + } + streamCtx, cancel := context.WithCancel(context.Background()) + session := &tunnelSession{ctx: streamCtx, cancel: cancel} + l.sessions[session] = struct{}{} + s.tunnels.mu.Unlock() + s.mu.Unlock() + // Request cancellation bounds the dial. Afterward, lease cancellation owns + // the stream and the caller closes it when the CONNECT handler returns. + stop := context.AfterFunc(ctx, cancel) + dialCtx, timeout := context.WithTimeout(streamCtx, workerTimeout) + conn, err := dialer.DialTunnel(dialCtx, id, port) + timeout() + stopped := stop() + s.tunnels.mu.Lock() + defer s.tunnels.mu.Unlock() + unauthorized := !stopped || streamCtx.Err() != nil || !time.Now().Before(l.ExpiresAt) + if err != nil || unauthorized { + cancel() + delete(l.sessions, session) + if conn != nil { + _ = conn.Close() + } + if unauthorized { + return nil, envd.ErrTunnelUnauthorized + } + return nil, err + } + if _, ok := conn.(interface{ CloseWrite() error }); !ok { + cancel() + delete(l.sessions, session) + _ = conn.Close() + return nil, errors.New("worker tunnel lacks directional EOF") + } + session.conn = conn + return &leasedTunnel{Conn: conn, service: s, lease: l, session: session}, nil +} + +type leasedTunnel struct { + net.Conn + service *Service + lease *tunnelLease + session *tunnelSession + once sync.Once +} + +func (c *leasedTunnel) TunnelContext() context.Context { return c.session.ctx } +func (c *leasedTunnel) CloseWrite() error { + return c.Conn.(interface{ CloseWrite() error }).CloseWrite() +} +func (c *leasedTunnel) Close() error { + c.once.Do(func() { + c.session.cancel() + c.service.tunnels.mu.Lock() + delete(c.lease.sessions, c.session) + c.service.tunnels.mu.Unlock() + }) + return c.Conn.Close() +} diff --git a/internal/control/tunnels_test.go b/internal/control/tunnels_test.go new file mode 100644 index 0000000..2d2097a --- /dev/null +++ b/internal/control/tunnels_test.go @@ -0,0 +1,285 @@ +package control + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gitmoot/sandboxd/internal/envd" + "github.com/gitmoot/sandboxd/internal/vm" +) + +type tunnelTestDriver struct { + *fakeDriver + target string +} + +func (d *tunnelTestDriver) DialTunnel(ctx context.Context, _ string, _ int) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", d.target) +} + +type testLease struct { + ID string `json:"leaseID"` + Token string `json:"token"` + Expires time.Time `json:"expiresAt"` +} + +type tunnelFixture struct { + service *Service + server *httptest.Server + listener net.Listener + id string + close func() +} + +func newTunnelFixture(t *testing.T, serve func(net.Conn)) tunnelFixture { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { defer conn.Close(); _ = conn.SetDeadline(time.Now().Add(10 * time.Second)); serve(conn) }() + } + }() + driver := &tunnelTestDriver{fakeDriver: &fakeDriver{instances: make(map[string]vm.Instance)}, target: listener.Addr().String()} + cfg := mixedConfig(map[string]Template{"base": {Image: "guest", Profile: ProfileE2B, Ports: []int{22}, EnvdVersion: "0.9.0"}}, 2) + cfg.MaxTTL = time.Hour + service, err := Open(context.Background(), filepath.Join(t.TempDir(), "ledger.sqlite"), driver, cfg) + if err != nil { + t.Fatal(err) + } + var closeOnce sync.Once + closeService := func() { closeOnce.Do(func() { _ = service.Close() }) } + t.Cleanup(closeService) + created := request(t, service, http.MethodPost, "/v2/sandboxes", map[string]any{"templateID": "base", "timeout": 600}) + if created.Code != http.StatusCreated { + t.Fatalf("create: %d %s", created.Code, created.Body.String()) + } + var body struct { + ID string `json:"sandboxID"` + } + if err := json.Unmarshal(created.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + proxy := envd.NewProxy(service, "sandbox.test", "127.0.0.1") + proxy.MaxPortStreams = 1 + handler := &envd.Handler{Driver: service.Guests(), Authorizer: service} + server := httptest.NewServer(envd.Routes(handler, proxy, service.Handler())) + t.Cleanup(server.Close) + return tunnelFixture{service, server, listener, body.ID, closeService} +} + +func (f tunnelFixture) mint(t *testing.T, ttl int) testLease { + t.Helper() + response := request(t, f.service, "POST", "/sandboxes/"+f.id+"/tunnel-leases", map[string]int{"port": 22, "ttl": ttl}) + if response.Code != http.StatusCreated { + t.Fatalf("mint: %d %s", response.Code, response.Body.String()) + } + var lease testLease + if err := json.Unmarshal(response.Body.Bytes(), &lease); err != nil { + t.Fatal(err) + } + return lease +} + +func (f tunnelFixture) connect(t *testing.T, target, token, extra, initial string) (*net.TCPConn, *bufio.Reader, int) { + t.Helper() + raw, err := net.DialTimeout("tcp", strings.TrimPrefix(f.server.URL, "http://"), 2*time.Second) + if err != nil { + t.Fatal(err) + } + conn := raw.(*net.TCPConn) + t.Cleanup(func() { _ = conn.Close() }) + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\nE2b-Tunnel-Access-Token: %s\r\n%s\r\n%s", target, target, token, extra, initial) + reader := bufio.NewReader(conn) + response, err := http.ReadResponse(reader, &http.Request{Method: http.MethodConnect}) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + _ = response.Body.Close() + _ = conn.Close() + } + return conn, reader, response.StatusCode +} + +func TestTunnelCONNECTHalfCloseAndIsolation(t *testing.T) { + f := newTunnelFixture(t, func(conn net.Conn) { + data, err := io.ReadAll(conn) + if err == nil { + _, _ = conn.Write(append([]byte("after EOF:"), data...)) + } + }) + lease := f.mint(t, 300) + for _, test := range []struct { + name, target, token string + status int + }{ + {"wrong token", f.id + ":22", "wrong", 403}, + {"wrong port", f.id + ":23", lease.Token, 403}, + {"arbitrary host", "127.0.0.1:22", lease.Token, 400}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, status := f.connect(t, test.target, test.token, "", "") + if status != test.status { + t.Fatalf("status=%d want %d", status, test.status) + } + }) + } + second := request(t, f.service, "POST", "/v2/sandboxes", map[string]any{"templateID": "base", "timeout": 600}) + if second.Code != 201 { + t.Fatalf("second sandbox: %s", second.Body.String()) + } + var other struct { + ID string `json:"sandboxID"` + } + _ = json.Unmarshal(second.Body.Bytes(), &other) + if _, _, status := f.connect(t, other.ID+":22", lease.Token, "", ""); status != 403 { + t.Fatalf("cross-sandbox token status=%d", status) + } + // Pipelined bytes following CONNECT must survive the HTTP parser buffer. + conn, reader, status := f.connect(t, f.id+":22", lease.Token, "X-API-Key: must-not-reach-guest\r\n", "first\x00") + if status != 200 { + t.Fatalf("CONNECT=%d", status) + } + if _, _, status := f.connect(t, f.id+":22", lease.Token, "", ""); status != 429 { + t.Fatalf("stream cap=%d", status) + } + payload := bytes.Repeat([]byte("\xffssh\x00"), 40000) + if _, err := conn.Write(payload); err != nil { + t.Fatal(err) + } + if err := conn.CloseWrite(); err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + want := append([]byte("after EOF:first\x00"), payload...) + if !bytes.Equal(got, want) { + t.Fatalf("response after client EOF differs: got %d bytes want %d", len(got), len(want)) + } + if response := request(t, f.service, "GET", "/sandboxes/"+f.id, nil); response.Code != 200 { + t.Fatalf("disconnect destroyed VM: %d", response.Code) + } +} + +func TestTunnelRenewRevokeAndExpiredReconnect(t *testing.T) { + received := make(chan error, 1) + f := newTunnelFixture(t, func(conn net.Conn) { + _, _ = conn.Write([]byte("guest EOF")) + _ = conn.(*net.TCPConn).CloseWrite() + data := make([]byte, len("still receiving")) + _, err := io.ReadFull(conn, data) + if err == nil && string(data) != "still receiving" { + err = fmt.Errorf("wrong reverse bytes: %q", data) + } + received <- err + _, _ = io.Copy(io.Discard, conn) + }) + lease := f.mint(t, 1) + conn, reader, status := f.connect(t, f.id+":22", lease.Token, "", "") + if status != 200 { + t.Fatalf("CONNECT=%d", status) + } + data, err := io.ReadAll(reader) + if err != nil || string(data) != "guest EOF" { + t.Fatalf("guest EOF: %q %v", data, err) + } + renew := request(t, f.service, "POST", "/sandboxes/"+f.id+"/tunnel-leases/"+lease.ID+"/renew", map[string]int{"ttl": 300}) + if renew.Code != 200 { + t.Fatalf("renew: %d %s", renew.Code, renew.Body.String()) + } + var renewed testLease + _ = json.Unmarshal(renew.Body.Bytes(), &renewed) + if renewed.ID != lease.ID || renewed.Token != lease.Token || !renewed.Expires.After(lease.Expires) { + t.Fatalf("renew changed identity or failed to extend: %+v", renewed) + } + // Wait past the ORIGINAL expiry, not an arbitrary sleep: the active stream + // must still accept reverse traffic after the guest has half-closed. + time.Sleep(max(0, time.Until(lease.Expires.Add(30*time.Millisecond)))) + if _, err := conn.Write([]byte("still receiving")); err != nil { + t.Fatalf("renewed stream closed: %v", err) + } + select { + case err := <-received: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("renewed stream did not deliver reverse bytes") + } + revoke := request(t, f.service, "DELETE", "/sandboxes/"+f.id+"/tunnel-leases/"+lease.ID, nil) + if revoke.Code != 204 { + t.Fatalf("revoke=%d", revoke.Code) + } + if _, _, status := f.connect(t, f.id+":22", lease.Token, "", ""); status != 403 { + t.Fatalf("revoked reconnect=%d", status) + } + if response := request(t, f.service, "POST", "/sandboxes/"+f.id+"/tunnel-leases/"+lease.ID+"/renew", map[string]int{"ttl": 300}); response.Code != 403 { + t.Fatalf("revived revoked lease=%d", response.Code) + } + if response := request(t, f.service, "GET", "/sandboxes/"+f.id, nil); response.Code != 200 { + t.Fatalf("revocation destroyed VM=%d", response.Code) + } + expired := f.mint(t, 1) + time.Sleep(max(0, time.Until(expired.Expires.Add(30*time.Millisecond)))) + if response := request(t, f.service, "POST", "/sandboxes/"+f.id+"/tunnel-leases/"+expired.ID+"/renew", map[string]int{"ttl": 300}); response.Code != 403 { + t.Fatalf("revived expired lease=%d", response.Code) + } + if _, _, status := f.connect(t, f.id+":22", expired.Token, "", ""); status != 403 { + t.Fatalf("expired reconnect=%d", status) + } +} + +func TestTunnelRevocationAndDestroyCloseActiveStream(t *testing.T) { + for _, action := range []string{"revoke", "destroy", "shutdown"} { + t.Run(action, func(t *testing.T) { + f := newTunnelFixture(t, func(conn net.Conn) { _, _ = io.Copy(io.Discard, conn) }) + lease := f.mint(t, 300) + _, reader, status := f.connect(t, f.id+":22", lease.Token, "", "") + if status != 200 { + t.Fatalf("CONNECT=%d", status) + } + switch action { + case "revoke": + if response := request(t, f.service, "DELETE", "/sandboxes/"+f.id+"/tunnel-leases/"+lease.ID, nil); response.Code != 204 { + t.Fatal(response.Code) + } + case "destroy": + if response := request(t, f.service, "DELETE", "/sandboxes/"+f.id, nil); response.Code != 204 { + t.Fatal(response.Code) + } + case "shutdown": + f.close() + } + _, err := reader.ReadByte() + if err == nil { + t.Fatal("stream stayed readable after teardown") + } + if timeout, ok := err.(net.Error); ok && timeout.Timeout() { + t.Fatalf("teardown did not close stream: %v", err) + } + }) + } +} diff --git a/internal/control/workers.go b/internal/control/workers.go index f6a5f16..39b6ef0 100644 --- a/internal/control/workers.go +++ b/internal/control/workers.go @@ -689,6 +689,7 @@ func (s *Service) ForgetWorker(ctx context.Context, id string) ([]string, error) } s.signersMu.Unlock() for _, sandbox := range ids { + s.revokeSandboxTunnels(sandbox) log.Printf("forget-worker %s: sandbox %s released as destroyed-unverified; its VM was NOT proven gone", id, sandbox) } if err != nil { diff --git a/internal/envd/proxy.go b/internal/envd/proxy.go index 5f2a9b8..f6c45f4 100644 --- a/internal/envd/proxy.go +++ b/internal/envd/proxy.go @@ -193,6 +193,10 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request) bool { if p == nil { return false } + if r.Method == http.MethodConnect { + p.serveTunnel(w, r) + return true + } id, port, ok := p.target(r) switch { case ok && port != EnvdPort: diff --git a/internal/envd/tunnels.go b/internal/envd/tunnels.go new file mode 100644 index 0000000..16f89ba --- /dev/null +++ b/internal/envd/tunnels.go @@ -0,0 +1,81 @@ +package envd + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strconv" + "time" + + "github.com/gitmoot/sandboxd/internal/tunnel" +) + +// ErrTunnelUnauthorized covers invalid, expired, revoked or mismatched leases. +var ErrTunnelUnauthorized = errors.New("tunnel lease unauthorized") + +type TunnelSandboxes interface { + OpenTunnel(context.Context, string, int, string) (TunnelConn, error) +} + +// TunnelConn exposes cancellation separately from EOF: revocation must close the +// client even if the guest has already finished sending but is still receiving. +type TunnelConn interface { + net.Conn + CloseWrite() error + TunnelContext() context.Context +} + +func (p *Proxy) serveTunnel(w http.ResponseWriter, r *http.Request) { + id, text, err := net.SplitHostPort(r.RequestURI) + port, parseErr := strconv.Atoi(text) + if err != nil || parseErr != nil || !validSandboxID(id) || port < 1 || port > 65535 || strconv.Itoa(port) != text || r.Host != r.RequestURI || r.ContentLength > 0 || len(r.TransferEncoding) != 0 { + http.Error(w, "CONNECT requires sandboxID:port without a request body", http.StatusBadRequest) + return + } + if r.ProtoMajor != 1 { + http.Error(w, "CONNECT requires HTTP/1.1", http.StatusHTTPVersionNotSupported) + return + } + tokens := r.Header.Values("E2b-Tunnel-Access-Token") + if len(tokens) != 1 || tokens[0] == "" { + http.Error(w, "tunnel lease unauthorized", http.StatusForbidden) + return + } + service, ok := p.Sandboxes.(TunnelSandboxes) + if !ok { + http.Error(w, "tunnels unavailable", http.StatusNotImplemented) + return + } + if !p.acquireStream(id) { + http.Error(w, "too many sandbox streams", http.StatusTooManyRequests) + return + } + defer p.releaseStream(id) + guest, err := service.OpenTunnel(r.Context(), id, port, tokens[0]) + if err != nil { + status := http.StatusBadGateway + if errors.Is(err, ErrTunnelUnauthorized) { + status = http.StatusForbidden + } + http.Error(w, http.StatusText(status), status) + return + } + defer guest.Close() + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + http.Error(w, "CONNECT requires HTTP/1.1", http.StatusHTTPVersionNotSupported) + return + } + defer conn.Close() + stop := context.AfterFunc(guest.TunnelContext(), func() { _ = conn.Close() }) + defer stop() + if err := conn.SetDeadline(time.Time{}); err != nil { + return + } + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + tunnel.Bridge(conn, buffered.Reader, guest) +} diff --git a/internal/guestagent/client.go b/internal/guestagent/client.go index e290885..4ae3a42 100644 --- a/internal/guestagent/client.go +++ b/internal/guestagent/client.go @@ -79,8 +79,18 @@ func Ping(ctx context.Context, conn io.ReadWriteCloser) ([]string, error) { // success conn carries raw bytes of that port from then on; on failure it is // closed. func OpenPort(ctx context.Context, conn io.ReadWriteCloser, port int) error { + return openPort(ctx, conn, port, OpDial) +} + +// OpenTunnel negotiates a framed, half-close-capable guest port stream. +// The caller wraps conn with tunnel.New only after this succeeds. +func OpenTunnel(ctx context.Context, conn io.ReadWriteCloser, port int) error { + return openPort(ctx, conn, port, OpTunnel) +} + +func openPort(ctx context.Context, conn io.ReadWriteCloser, port int, op string) error { stop := closeOnCancel(ctx, conn) - err := WriteRequest(conn, Request{Op: OpDial, Port: port}) + err := WriteRequest(conn, Request{Op: op, Port: port}) var result Result if err == nil { result, err = readResult(conn) diff --git a/internal/guestagent/protocol.go b/internal/guestagent/protocol.go index 2c388e0..a733f50 100644 --- a/internal/guestagent/protocol.go +++ b/internal/guestagent/protocol.go @@ -39,6 +39,9 @@ const ( // After a successful Result frame the connection carries raw bytes in // both directions until either side closes; no further frames follow. OpDial = "dial" + // OpTunnel uses internal/tunnel data and directional EOF frames after + // the Result acknowledgement. Unsupported agents reject the operation. + OpTunnel = "tunnel" // OpDisk reports the guest's writable filesystem: DiskTotal and DiskUsed. OpDisk = "disk" // OpSync flushes the guest's filesystems before the host kills the VM, diff --git a/internal/guestagent/server_linux.go b/internal/guestagent/server_linux.go index 9513cbc..c5cc13f 100644 --- a/internal/guestagent/server_linux.go +++ b/internal/guestagent/server_linux.go @@ -18,6 +18,8 @@ import ( "sync/atomic" "syscall" "time" + + "github.com/gitmoot/sandboxd/internal/tunnel" ) var envName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*$`) @@ -88,6 +90,10 @@ func (s *Server) ServeConn(conn io.ReadWriteCloser) { if result = s.dial(conn, reader, request.Port); result.Error == "" { return } + case OpTunnel: + if result = s.dialTunnel(conn, reader, request.Port); result.Error == "" { + return + } case OpDisk: result = s.disk() case OpSync: @@ -133,6 +139,23 @@ func (s *Server) dial(conn io.ReadWriteCloser, reader *bufio.Reader, port int) R return Result{} } +func (s *Server) dialTunnel(conn io.ReadWriteCloser, reader *bufio.Reader, port int) Result { + if s.DialHost == "" || port < 1 || port > 65535 { + return Result{Error: "guest tunnel port unavailable"} + } + upstream, err := net.DialTimeout("tcp", net.JoinHostPort(s.DialHost, strconv.Itoa(port)), 5*time.Second) + if err != nil { + return Result{Error: fmt.Sprintf("guest port %d unreachable: %v", port, err)} + } + defer upstream.Close() + if err := WriteJSONFrame(conn, FrameResult, Result{}); err != nil { + return Result{} + } + stream := tunnel.New(conn, reader) + tunnel.Bridge(stream, stream, upstream) + return Result{} +} + func (s *Server) disk() Result { if s.DiskPath == "" { return Result{Error: "guest disk reporting is not configured"} diff --git a/internal/guestagent/tunnel_test.go b/internal/guestagent/tunnel_test.go new file mode 100644 index 0000000..d6f18c7 --- /dev/null +++ b/internal/guestagent/tunnel_test.go @@ -0,0 +1,134 @@ +//go:build linux + +package guestagent + +import ( + "bytes" + "encoding/json" + "io" + "net" + "os" + "syscall" + "testing" + "time" +) + +// Use the actual accepted-socket wrapper, not net.Pipe's different close +// semantics. Speak the wire protocol so this test also compiles on old agents. +func TestTunnelDirectionalEOF(t *testing.T) { + for _, first := range []string{"host", "guest"} { + t.Run(first, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + payload := bytes.Repeat([]byte("\x00SSH\xff"), 10000) + upstreamDone := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + upstreamDone <- err + return + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + if first == "guest" { + _, err = conn.Write(payload) + if err == nil { + err = conn.(*net.TCPConn).CloseWrite() + } + } + got, readErr := io.ReadAll(conn) + if err == nil { + err = readErr + } + if err == nil && !bytes.Equal(got, payload) { + err = io.ErrUnexpectedEOF + } + if first == "host" && err == nil { + _, err = conn.Write(payload) + } + upstreamDone <- err + }() + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + file := os.NewFile(uintptr(fds[0]), "tunnel-host") + host, err := net.FileConn(file) + _ = file.Close() + if err != nil { + t.Fatal(err) + } + defer host.Close() + _ = host.SetDeadline(time.Now().Add(5 * time.Second)) + guest, err := SocketConn(fds[1], "tunnel-guest") + if err != nil { + t.Fatal(err) + } + server := testServer(t) + server.DialHost = "127.0.0.1" + done := make(chan struct{}) + go func() { server.ServeConn(guest); close(done) }() + if err := WriteRequest(host, Request{Op: "tunnel", Port: listener.Addr().(*net.TCPAddr).Port}); err != nil { + t.Fatal(err) + } + kind, data, err := ReadFrame(host) + var result Result + if err != nil || kind != FrameResult { + t.Fatalf("ack: %d %v", kind, err) + } + if err := json.Unmarshal(data, &result); err != nil || result.Error != "" { + t.Fatalf("ack result: %s %v", result.Error, err) + } + send := func() { + for start := 0; start < len(payload); start += 8192 { + if err := WriteFrame(host, 0, payload[start:min(start+8192, len(payload))]); err != nil { + t.Fatal(err) + } + } + if err := WriteFrame(host, 1, nil); err != nil { + t.Fatal(err) + } + } + receive := func() { + var got []byte + for { + kind, data, err := ReadFrame(host) + if err != nil { + t.Fatal(err) + } + if kind == 1 { + if len(data) != 0 { + t.Fatal("EOF payload") + } + break + } + if kind != 0 { + t.Fatalf("frame kind=%d", kind) + } + got = append(got, data...) + } + if !bytes.Equal(got, payload) { + t.Fatalf("reverse payload differs: got %d want %d", len(got), len(payload)) + } + } + if first == "host" { + send() + receive() + } else { + receive() + send() + } + if err := <-upstreamDone; err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("guest bridge retained after both EOFs") + } + }) + } +} diff --git a/internal/tunnel/stream.go b/internal/tunnel/stream.go new file mode 100644 index 0000000..90dd5ca --- /dev/null +++ b/internal/tunnel/stream.go @@ -0,0 +1,180 @@ +// Package tunnel carries directional EOF across transports that cannot half-close. +package tunnel + +import ( + "encoding/binary" + "errors" + "io" + "net" + "sync" + "time" +) + +const maxData = 32 << 10 + +// Conn frames data and write-side EOF. Closing the underlying transport without +// an EOF frame is an error, not a directional EOF. Buffers are bounded. +type Conn struct { + io.ReadWriteCloser + reader io.Reader + readMu, writeMu sync.Mutex + left uint32 + readEOF, writeEOF bool +} + +func New(conn io.ReadWriteCloser, reader io.Reader) *Conn { + if reader == nil { + reader = conn + } + return &Conn{ReadWriteCloser: conn, reader: reader} +} + +func (c *Conn) Read(p []byte) (int, error) { + c.readMu.Lock() + defer c.readMu.Unlock() + if len(p) == 0 { + return 0, nil + } + if c.readEOF { + return 0, io.EOF + } + if c.left == 0 { + var h [5]byte + if _, err := io.ReadFull(c.reader, h[:]); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, err + } + size := binary.BigEndian.Uint32(h[1:]) + if h[0] == 1 && size == 0 { + c.readEOF = true + return 0, io.EOF + } + if h[0] != 0 || size == 0 || size > maxData { + return 0, errors.New("invalid tunnel frame") + } + c.left = size + } + n, err := c.reader.Read(p[:min(len(p), int(c.left))]) + c.left -= uint32(n) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return n, err +} + +func (c *Conn) Write(p []byte) (int, error) { + c.writeMu.Lock() + defer c.writeMu.Unlock() + if c.writeEOF { + return 0, net.ErrClosed + } + written := 0 + for len(p) > 0 { + n := min(len(p), maxData) + var h [5]byte + binary.BigEndian.PutUint32(h[1:], uint32(n)) + if err := writeAll(c.ReadWriteCloser, h[:]); err != nil { + return written, err + } + if err := writeAll(c.ReadWriteCloser, p[:n]); err != nil { + return written, err + } + written += n + p = p[n:] + } + return written, nil +} + +func writeAll(w io.Writer, p []byte) error { + for len(p) > 0 { + n, err := w.Write(p) + if err != nil { + return err + } + if n == 0 { + return io.ErrShortWrite + } + p = p[n:] + } + return nil +} + +func (c *Conn) CloseWrite() error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + if c.writeEOF { + return nil + } + c.writeEOF = true + return writeAll(c.ReadWriteCloser, []byte{1, 0, 0, 0, 0}) +} + +func (c *Conn) LocalAddr() net.Addr { + if n, ok := c.ReadWriteCloser.(net.Conn); ok { + return n.LocalAddr() + } + return addr{} +} +func (c *Conn) RemoteAddr() net.Addr { + if n, ok := c.ReadWriteCloser.(net.Conn); ok { + return n.RemoteAddr() + } + return addr{} +} +func (c *Conn) SetDeadline(t time.Time) error { + if n, ok := c.ReadWriteCloser.(interface{ SetDeadline(time.Time) error }); ok { + return n.SetDeadline(t) + } + return errors.New("tunnel has no deadlines") +} +func (c *Conn) SetReadDeadline(t time.Time) error { + if n, ok := c.ReadWriteCloser.(interface{ SetReadDeadline(time.Time) error }); ok { + return n.SetReadDeadline(t) + } + return errors.New("tunnel has no deadlines") +} +func (c *Conn) SetWriteDeadline(t time.Time) error { + if n, ok := c.ReadWriteCloser.(interface{ SetWriteDeadline(time.Time) error }); ok { + return n.SetWriteDeadline(t) + } + return errors.New("tunnel has no deadlines") +} + +type addr struct{} + +func (addr) Network() string { return "sandboxd-tunnel" } +func (addr) String() string { return "sandboxd-tunnel" } + +// Bridge propagates EOF independently in each direction. Errors, cancellation +// (by closing either connection), or both EOFs close both connections. +func Bridge(client net.Conn, in io.Reader, upstream net.Conn) { + defer client.Close() + defer upstream.Close() + a, aok := client.(interface{ CloseWrite() error }) + b, bok := upstream.(interface{ CloseWrite() error }) + if !aok || !bok { + return + } + done := make(chan error, 2) + go func() { + _, err := io.Copy(upstream, in) + if err == nil { + err = b.CloseWrite() + } + done <- err + }() + go func() { + _, err := io.Copy(client, upstream) + if err == nil { + err = a.CloseWrite() + } + done <- err + }() + if err := <-done; err != nil { + _ = client.Close() + _ = upstream.Close() + } + <-done +} diff --git a/internal/vm/devvm/envd.go b/internal/vm/devvm/envd.go index a203e20..4e9ecdd 100644 --- a/internal/vm/devvm/envd.go +++ b/internal/vm/devvm/envd.go @@ -21,6 +21,7 @@ import ( "golang.org/x/sys/unix" + "github.com/gitmoot/sandboxd/internal/tunnel" "github.com/gitmoot/sandboxd/internal/vm" ) @@ -172,6 +173,11 @@ func (d *Driver) DialPort(ctx context.Context, id string, port int) (net.Conn, e return conn, nil } +// The dev bridge uses Unix/TCP sockets, both of which preserve CloseWrite. +func (d *Driver) DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) { + return d.DialPort(ctx, id, port) +} + // destroyEnvd closes the helper's lifeline: it kills every process of the // guest, removes what it created and exits. func (g *guest) destroyEnvd(ctx context.Context) error { @@ -516,10 +522,7 @@ func bridgeEnvd(listener net.Listener) { if _, err := client.Write([]byte{bridgeConnected}); err != nil { return } - done := make(chan struct{}, 2) - go func() { _, _ = io.Copy(upstream, client); done <- struct{}{} }() - go func() { _, _ = io.Copy(client, upstream); done <- struct{}{} }() - <-done + tunnel.Bridge(client, client, upstream) }() } } diff --git a/internal/vm/firecracker.go b/internal/vm/firecracker.go index 2ceb84a..dc3d434 100644 --- a/internal/vm/firecracker.go +++ b/internal/vm/firecracker.go @@ -22,6 +22,7 @@ import ( "github.com/gitmoot/sandboxd/internal/egress" "github.com/gitmoot/sandboxd/internal/guestagent" + "github.com/gitmoot/sandboxd/internal/tunnel" ) const ( @@ -1309,6 +1310,14 @@ func (d *FirecrackerDriver) Usage(ctx context.Context, id string) (Usage, error) // firewall monitor, as Run is; losing the firewall destroys the VM and ends // it. func (d *FirecrackerDriver) DialPort(ctx context.Context, id string, port int) (net.Conn, error) { + return d.dialPort(ctx, id, port, false) +} + +func (d *FirecrackerDriver) DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) { + return d.dialPort(ctx, id, port, true) +} + +func (d *FirecrackerDriver) dialPort(ctx context.Context, id string, port int, framed bool) (net.Conn, error) { if !ValidPort(port) { return nil, fmt.Errorf("invalid guest port %d", port) } @@ -1329,11 +1338,19 @@ func (d *FirecrackerDriver) DialPort(ctx context.Context, id string, port int) ( if err != nil { return nil, err } - if err := guestagent.OpenPort(ctx, conn, port); err != nil { - return nil, fmt.Errorf("port %d of %q: %w", port, id, err) + open := guestagent.OpenPort + if framed { + open = guestagent.OpenTunnel + } + if err := open(ctx, conn, port); err != nil { + _ = conn.Close() + return nil, err } guarded := &fcGuestConn{Conn: conn} guarded.unwatch = d.watchFirewall(id, func(error) { _ = conn.Close() }) + if framed { + return tunnel.New(guarded, nil), nil + } return guarded, nil } diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 004712b..08f1125 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -51,6 +51,12 @@ type PortDialer interface { DialPort(ctx context.Context, id string, port int) (net.Conn, error) } +// TunnelDialer opens a guest-loopback stream with directional EOF support. +// The returned net.Conn must implement CloseWrite; unsupported guests fail. +type TunnelDialer interface { + DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) +} + // ValidPort reports whether port is a TCP port number DialPort accepts. func ValidPort(port int) bool { return port >= 1 && port <= 65535 diff --git a/internal/worker/client.go b/internal/worker/client.go index b14a223..ce63249 100644 --- a/internal/worker/client.go +++ b/internal/worker/client.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/gitmoot/sandboxd/internal/tunnel" "github.com/gitmoot/sandboxd/internal/vm" ) @@ -386,6 +387,20 @@ func (c *Client) Usage(ctx context.Context, id string) (vm.Usage, error) { // HTTP/1.1 upgrade the worker bridges to its driver's guest port channel. ctx // bounds only the dial; the stream lasts until it is closed. func (c *Client) DialPort(ctx context.Context, id string, port int) (net.Conn, error) { + return c.dialPort(ctx, id, port, portUpgrade) +} + +const tunnelUpgrade = "sandboxd-tunnel-v1" + +func (c *Client) DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) { + conn, err := c.dialPort(ctx, id, port, tunnelUpgrade) + if err != nil { + return nil, err + } + return tunnel.New(conn, nil), nil +} + +func (c *Client) dialPort(ctx context.Context, id string, port int, upgrade string) (net.Conn, error) { if err := c.checkVM(id); err != nil { return nil, err } @@ -408,7 +423,7 @@ func (c *Client) DialPort(ctx context.Context, id string, port int) (net.Conn, e req.Header.Set("Authorization", c.auth) req.Header.Set(leaseHeader, strconv.FormatInt(lease, 10)) req.Header.Set("Connection", "Upgrade") - req.Header.Set("Upgrade", portUpgrade) + req.Header.Set("Upgrade", upgrade) resp, err := c.client.Do(req) if !stop() { if err == nil { @@ -427,7 +442,7 @@ func (c *Client) DialPort(ctx context.Context, id string, port int) (net.Conn, e return nil, fmt.Errorf("worker %s: %w", c.id, vm.ErrNoEnvd) } stream, ok := resp.Body.(io.ReadWriteCloser) - if resp.StatusCode != http.StatusSwitchingProtocols || !ok || !strings.EqualFold(resp.Header.Get("Upgrade"), portUpgrade) { + if resp.StatusCode != http.StatusSwitchingProtocols || !ok || !strings.EqualFold(resp.Header.Get("Upgrade"), upgrade) { defer resp.Body.Close() defer cancel() if resp.StatusCode == http.StatusSwitchingProtocols { diff --git a/internal/worker/server.go b/internal/worker/server.go index c2e2603..ef84783 100644 --- a/internal/worker/server.go +++ b/internal/worker/server.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "github.com/gitmoot/sandboxd/internal/tunnel" "github.com/gitmoot/sandboxd/internal/vm" ) @@ -516,18 +517,28 @@ func (s *Server) run(w http.ResponseWriter, r *http.Request, id string, leaseCtx // either side closes it, or when a newer lease supersedes the one it was // opened under. func (s *Server) port(w http.ResponseWriter, r *http.Request, id string, port int, leaseCtx context.Context) { - dialer, ok := s.driver.(vm.PortDialer) - if !ok { - http.Error(w, vm.ErrNoEnvd.Error(), http.StatusNotImplemented) + upgrade := r.Header.Get("Upgrade") + if !headerHasToken(r.Header, "Connection", "upgrade") || + (!strings.EqualFold(upgrade, portUpgrade) && !strings.EqualFold(upgrade, tunnelUpgrade)) { + http.Error(w, "unsupported port stream upgrade", http.StatusBadRequest) return } - if !headerHasToken(r.Header, "Connection", "upgrade") || !strings.EqualFold(r.Header.Get("Upgrade"), portUpgrade) { - http.Error(w, "a port stream must upgrade to "+portUpgrade, http.StatusBadRequest) + framed := strings.EqualFold(upgrade, tunnelUpgrade) + var dial func(context.Context, string, int) (net.Conn, error) + if framed { + if d, ok := s.driver.(vm.TunnelDialer); ok { + dial = d.DialTunnel + } + } else if d, ok := s.driver.(vm.PortDialer); ok { + dial = d.DialPort + } + if dial == nil { + http.Error(w, vm.ErrNoEnvd.Error(), http.StatusNotImplemented) return } ctx, cancel := fenced(r, leaseCtx) defer cancel() - guest, err := dialer.DialPort(ctx, id, port) + guest, err := dial(ctx, id, port) if err != nil { driverFailed(w, "port", err) return @@ -543,11 +554,16 @@ func (s *Server) port(w http.ResponseWriter, r *http.Request, id string, port in if err := conn.SetDeadline(time.Time{}); err != nil { return } - if _, err := io.WriteString(conn, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: "+portUpgrade+"\r\n\r\n"); err != nil { + if _, err := io.WriteString(conn, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: "+upgrade+"\r\n\r\n"); err != nil { return } stop := context.AfterFunc(leaseCtx, func() { _ = conn.Close(); _ = guest.Close() }) defer stop() + if framed { + stream := tunnel.New(conn, buffered.Reader) + tunnel.Bridge(stream, stream, guest) + return + } Bridge(conn, buffered.Reader, guest) } diff --git a/internal/worker/tunnel_test.go b/internal/worker/tunnel_test.go new file mode 100644 index 0000000..32e9c9a --- /dev/null +++ b/internal/worker/tunnel_test.go @@ -0,0 +1,89 @@ +package worker + +import ( + "context" + "io" + "net" + "testing" + "time" +) + +type tunnelDriver struct { + *fakeDriver + target string +} + +func (d *tunnelDriver) DialTunnel(ctx context.Context, _ string, _ int) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", d.target) +} + +func TestTunnelRemoteDirectionalEOF(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan error, 1) + go func() { + c, err := listener.Accept() + if err != nil { + done <- err + return + } + defer c.Close() + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + data, err := io.ReadAll(c) + if err == nil { + _, err = c.Write(append([]byte("response:"), data...)) + } + done <- err + }() + driver := &tunnelDriver{fakeDriver: newFake(), target: listener.Addr().String()} + _, server := serve(t, driver) + client := enrolled(t, server.URL, 1) + // Keep the test old-code compatible so absence of the new transport is a + // behavioral failure, not a compile failure in the regression proof. + dialer, ok := any(client).(interface { + DialTunnel(context.Context, string, int) (net.Conn, error) + }) + if !ok { + t.Fatal("worker client does not support directional tunnel transport") + } + conn, err := dialer.DialTunnel(context.Background(), "sandboxd-00000000000000000000000000000e2b", 22) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := io.WriteString(conn, "SSH\x00binary\xff"); err != nil { + t.Fatal(err) + } + half, ok := conn.(interface{ CloseWrite() error }) + if !ok { + t.Fatal("missing CloseWrite") + } + if err := half.CloseWrite(); err != nil { + t.Fatal(err) + } + result := make(chan struct { + data []byte + err error + }, 1) + go func() { + data, err := io.ReadAll(conn) + result <- struct { + data []byte + err error + }{data, err} + }() + select { + case got := <-result: + if got.err != nil || string(got.data) != "response:SSH\x00binary\xff" { + t.Fatalf("reverse response after EOF: %q %v", got.data, got.err) + } + case <-time.After(5 * time.Second): + t.Fatal("remote bridge lost directional EOF") + } + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 5131478..3c5a332 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -206,6 +206,14 @@ func (l *local) DialPort(ctx context.Context, id string, port int) (net.Conn, er return dialer.DialPort(ctx, id, port) } +func (l *local) DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) { + dialer, ok := l.Driver.(vm.TunnelDialer) + if !ok { + return nil, vm.ErrNoEnvd + } + return dialer.DialTunnel(ctx, id, port) +} + func (l *local) CreateUntil(ctx context.Context, spec vm.Spec, _ time.Time) (vm.Instance, error) { return l.Driver.Create(ctx, spec) } From 1920aa086904bca8cda1eae836bc477b4caee94d Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sat, 31 Oct 2026 17:05:01 +0100 Subject: [PATCH 2/2] Observe tunnel transport cancellation after directional EOF --- docs/compatibility.md | 3 ++ internal/guestagent/tunnel_test.go | 21 +++++++++- internal/tunnel/stream.go | 50 +++++++++++++++-------- internal/tunnel/stream_test.go | 58 ++++++++++++++++++++++++++ internal/worker/tunnel_test.go | 65 ++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 internal/tunnel/stream_test.go diff --git a/docs/compatibility.md b/docs/compatibility.md index fd950c0..80624a6 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -241,6 +241,9 @@ CONNECT credentials, and no HTTP headers reach the guest. Directional EOF preserves reverse traffic until the reverse side ends. Errors, lease revoke/expiry, sandbox teardown, and daemon shutdown close both directions. Disconnect alone releases the stream without deleting the sandbox or lease. +Worker and guest-agent framed hops continue observing transport closure after +directional EOF, so an idle reverse stream cannot retain downstream connections +after cancellation. Bytes arriving after a framed EOF are a protocol error. CONNECT shares the per-sandbox `-port-max-streams` cap with HTTP/WebSocket guest ports (429 when full). Invalid/expired/revoked leases return 403; an unavailable guest port or unsupported guest tunnel transport returns 502. diff --git a/internal/guestagent/tunnel_test.go b/internal/guestagent/tunnel_test.go index d6f18c7..19d58c9 100644 --- a/internal/guestagent/tunnel_test.go +++ b/internal/guestagent/tunnel_test.go @@ -16,7 +16,7 @@ import ( // Use the actual accepted-socket wrapper, not net.Pipe's different close // semantics. Speak the wire protocol so this test also compiles on old agents. func TestTunnelDirectionalEOF(t *testing.T) { - for _, first := range []string{"host", "guest"} { + for _, first := range []string{"host", "guest", "host-disconnect"} { t.Run(first, func(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -24,6 +24,8 @@ func TestTunnelDirectionalEOF(t *testing.T) { } defer listener.Close() payload := bytes.Repeat([]byte("\x00SSH\xff"), 10000) + release := make(chan struct{}) + defer close(release) upstreamDone := make(chan error, 1) go func() { conn, err := listener.Accept() @@ -50,6 +52,10 @@ func TestTunnelDirectionalEOF(t *testing.T) { _, err = conn.Write(payload) } upstreamDone <- err + if first == "host-disconnect" { + // Stay open and idle so the guest cannot release the bridge. + <-release + } }() fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0) if err != nil { @@ -114,6 +120,19 @@ func TestTunnelDirectionalEOF(t *testing.T) { t.Fatalf("reverse payload differs: got %d want %d", len(got), len(payload)) } } + if first == "host-disconnect" { + send() + if err := <-upstreamDone; err != nil { + t.Fatal(err) + } + _ = host.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("guest agent retained bridge after host transport close") + } + return + } if first == "host" { send() receive() diff --git a/internal/tunnel/stream.go b/internal/tunnel/stream.go index 90dd5ca..a2f9cc7 100644 --- a/internal/tunnel/stream.go +++ b/internal/tunnel/stream.go @@ -152,29 +152,45 @@ func (addr) String() string { return "sandboxd-tunnel" } func Bridge(client net.Conn, in io.Reader, upstream net.Conn) { defer client.Close() defer upstream.Close() - a, aok := client.(interface{ CloseWrite() error }) - b, bok := upstream.(interface{ CloseWrite() error }) + _, aok := client.(interface{ CloseWrite() error }) + _, bok := upstream.(interface{ CloseWrite() error }) if !aok || !bok { return } - done := make(chan error, 2) - go func() { - _, err := io.Copy(upstream, in) + // Each copy reports directional EOF, then keeps observing a framed source + // for transport cancellation. No data is legal after its EOF frame. + done := make(chan error, 4) + var copies sync.WaitGroup + copies.Add(2) + copyDirection := func(dst net.Conn, src io.Reader) { + defer copies.Done() + _, err := io.Copy(dst, src) if err == nil { - err = b.CloseWrite() + err = dst.(interface{ CloseWrite() error }).CloseWrite() } done <- err - }() - go func() { - _, err := io.Copy(client, upstream) - if err == nil { - err = a.CloseWrite() + if err != nil { + return + } + if framed, ok := src.(*Conn); ok { + framed.readMu.Lock() + var extra [1]byte + _, err = io.ReadFull(framed.reader, extra[:]) + framed.readMu.Unlock() + if err == nil { + err = errors.New("data after tunnel EOF") + } + done <- err + } + } + go copyDirection(upstream, in) + go copyDirection(client, upstream) + for completed := 0; completed < 2; completed++ { + if err := <-done; err != nil { + break } - done <- err - }() - if err := <-done; err != nil { - _ = client.Close() - _ = upstream.Close() } - <-done + _ = client.Close() + _ = upstream.Close() + copies.Wait() } diff --git a/internal/tunnel/stream_test.go b/internal/tunnel/stream_test.go new file mode 100644 index 0000000..ffd803c --- /dev/null +++ b/internal/tunnel/stream_test.go @@ -0,0 +1,58 @@ +package tunnel + +import ( + "io" + "net" + "testing" + "time" +) + +func TestBridgeTransportCloseAfterDirectionalEOF(t *testing.T) { + for _, direction := range []string{"client", "upstream"} { + t.Run(direction, func(t *testing.T) { + clientWire, clientPeer := net.Pipe() + upstreamWire, upstreamPeer := net.Pipe() + client, upstream := New(clientWire, nil), New(upstreamWire, nil) + first, second := New(clientPeer, nil), New(upstreamPeer, nil) + defer first.Close() + defer second.Close() + defer client.Close() + defer upstream.Close() + _ = first.SetDeadline(time.Now().Add(5 * time.Second)) + _ = second.SetDeadline(time.Now().Add(5 * time.Second)) + if direction == "upstream" { + first, second = second, first + } + done := make(chan struct{}) + go func() { Bridge(client, client, upstream); close(done) }() + sent := make(chan error, 1) + go func() { sent <- first.CloseWrite() }() + var one [1]byte + if n, err := second.Read(one[:]); n != 0 || err != io.EOF { + t.Fatalf("directional EOF: n=%d err=%v", n, err) + } + if err := <-sent; err != nil { + t.Fatal(err) + } + // Directional EOF must leave reverse traffic usable, not cancel it. + go func() { _, err := second.Write([]byte{42}); sent <- err }() + if _, err := io.ReadFull(first, one[:]); err != nil || one[0] != 42 { + t.Fatalf("reverse data after EOF: %v %v", one, err) + } + if err := <-sent; err != nil { + t.Fatal(err) + } + // The opposite peer stays open and idle: only this transport close + // can release the bridge and its downstream resources. + _ = first.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("transport close after directional EOF retained bridge") + } + if _, err := second.Write([]byte{1}); err == nil { + t.Fatal("bridge left the opposite transport open") + } + }) + } +} diff --git a/internal/worker/tunnel_test.go b/internal/worker/tunnel_test.go index 32e9c9a..1103359 100644 --- a/internal/worker/tunnel_test.go +++ b/internal/worker/tunnel_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "net" + "sync" "testing" "time" ) @@ -87,3 +88,67 @@ func TestTunnelRemoteDirectionalEOF(t *testing.T) { t.Fatal(err) } } + +type observedTunnelDriver struct { + *tunnelDriver + closed chan struct{} +} + +type observedTunnelConn struct { + *net.TCPConn + closed chan struct{} + once sync.Once +} + +func (c *observedTunnelConn) Close() error { + err := c.TCPConn.Close() + c.once.Do(func() { close(c.closed) }) + return err +} + +func (d *observedTunnelDriver) DialTunnel(ctx context.Context, id string, port int) (net.Conn, error) { + conn, err := d.tunnelDriver.DialTunnel(ctx, id, port) + if err != nil { + return nil, err + } + return &observedTunnelConn{TCPConn: conn.(*net.TCPConn), closed: d.closed}, nil +} + +func TestTunnelRemoteCloseAfterDirectionalEOF(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + driver := &observedTunnelDriver{ + tunnelDriver: &tunnelDriver{fakeDriver: newFake(), target: listener.Addr().String()}, + closed: make(chan struct{}), + } + _, server := serve(t, driver) + client := enrolled(t, server.URL, 1) + conn, err := client.DialTunnel(context.Background(), "sandboxd-00000000000000000000000000000e2b", 22) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + guest, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + defer guest.Close() + _ = guest.SetDeadline(time.Now().Add(5 * time.Second)) + if err := conn.(interface{ CloseWrite() error }).CloseWrite(); err != nil { + t.Fatal(err) + } + if data, err := io.ReadAll(guest); err != nil || len(data) != 0 { + t.Fatalf("guest directional EOF: %q %v", data, err) + } + // Keep the guest open and idle after its read EOF. Closing the upgraded + // client transport must release the worker's guest connection anyway. + _ = conn.Close() + select { + case <-driver.closed: + case <-time.After(time.Second): + t.Fatal("worker retained guest connection after client transport close") + } +}