feat(vmm): pass pre-opened tap chardevs and drop VM privileges via systemd

docs/experimental-systemd-vm-processes.md

@@ -57,8 +57,20 @@ KillSignal=SIGTERM
 SendSIGKILL=yes
 TimeoutStopSec=<systemd.stop_timeout>
 Restart=no
+User=<cvm.user>                 # when cvm.user is set
+OpenFile=/dev/tapN              # one entry per NIC with networking.open_file
 ```
 
+`User=` replaces the Supervisor `sudo -u` path. systemd opens each `OpenFile=`
+path with the manager's privileges and hands the descriptors to the service in
+declaration order starting at fd 3, which is what QEMU's generated
+`-netdev tap,id=netN,fd=M` arguments expect. That combination needs systemd
+253 or newer.
+
+When both are set, the chardev stays root-owned on the host and the unit still
+runs QEMU unprivileged. For software-TPM VMs the whole launcher unit runs as
+`cvm.user`, so the VMM chowns the swtpm state directory before start.
+
 The existing launcher remains responsible for swtpm readiness and graceful
 child shutdown. systemd owns the final cgroup lifetime. A stop request is
 submitted asynchronously so the VMM can report a VM as stopping while QEMU is
@@ -88,8 +100,10 @@ atomic property handling and event-driven state updates.
 
 ## Limitations
 
-- The host must run systemd with support for `ExitType=cgroup` and
-  `StandardOutput=append:`.
+- The host must run systemd 253+ with support for `OpenFile=`,
+  `ExitType=cgroup`, and `StandardOutput=append:`.
+- `networking.open_file` is manifest-only, requires `mode = "custom"`, and is
+  rejected with Supervisor, one-shot execution, and swtpm-backed VMs.
 - The VMM must be authorized to create and stop system services.
 - Transient services inherit the systemd manager environment rather than the
   VMM environment. Variables in `ProcessConfig.env` are forwarded; unrelated

dstack/supervisor/client/src/main.rs

@@ -74,6 +74,8 @@ async fn main() -> Result<()> {
                 pidfile: String::new(),
                 cid: None,
                 note: String::new(),
+                user: String::new(),
+                open_files: Vec::new(),
             };
             print_json(&client.deploy(&config).await?)?;
         }

dstack/supervisor/src/process.rs

@@ -46,6 +46,25 @@ pub struct ProcessConfig {
     pub cid: Option<u32>,
     #[serde(default)]
     pub note: String,
+    /// User the process manager runs the process as.
+    ///
+    /// Only the VMM's systemd backend implements this, by dropping privileges
+    /// in the transient unit. Supervisor rejects a config that sets it rather
+    /// than running the process with its own privileges. Skipped when empty so
+    /// existing records and requests keep serializing byte-identically.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[builder(default)]
+    pub user: String,
+    /// Files the process manager opens before exec and passes to the process
+    /// as inherited file descriptors, in declaration order starting at fd 3.
+    ///
+    /// Only the VMM's systemd backend implements this. Supervisor rejects a
+    /// config that sets it rather than starting a process without the file
+    /// descriptors it asked for. Skipped when empty so existing records and
+    /// requests keep serializing byte-identically.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[builder(default)]
+    pub open_files: Vec<String>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

dstack/supervisor/src/supervisor.rs

@@ -59,6 +59,18 @@ impl Supervisor {
         if id.is_empty() {
             return Err(anyhow::anyhow!("Process ID is empty"));
         }
+        if !config.user.is_empty() {
+            // Supervisor runs processes with its own privileges. Starting the
+            // process anyway would run a VM as root that asked to be confined
+            // to an unprivileged user.
+            bail!("user is not supported by supervisor");
+        }
+        if !config.open_files.is_empty() {
+            // Supervisor spawns processes without pre-opened file descriptors,
+            // so honoring the rest of the config would start a process that is
+            // missing the files it depends on.
+            bail!("open_files is not supported by supervisor");
+        }
         if self
             .info(&id)
             .is_some_and(|info| info.state.status.is_running())

dstack/vmm/Cargo.toml

@@ -25,7 +25,7 @@ sha2.workspace = true
 hex.workspace = true
 fs-err.workspace = true
 getrandom = { workspace = true, features = ["std"] }
-nix = { workspace = true, features = ["user"] }
+nix = { workspace = true, features = ["fs", "user", "dir"] }
 dirs.workspace = true
 which.workspace = true
 clap = { workspace = true, features = ["derive", "string"] }

dstack/vmm/rpc/proto/vmm_rpc.proto

@@ -126,10 +126,18 @@ message VmConfiguration {
 
 // Per-VM networking configuration.
 message NetworkingConfig {
-  // Networking mode: "bridge", "user"
+  // Networking mode: "bridge", "user", or "custom"
   string mode = 1;
-  // Per-VM bridge interface name. Empty = node default bridge.
+  // Per-VM bridge interface name. Empty = node default bridge. Bridge mode only.
   string bridge_name = 2;
+  // Explicit QEMU netdev string. Custom mode only, mutually exclusive with
+  // open_file.
+  string netdev = 3;
+  // Absolute path to a tap character device an external net daemon already
+  // created, e.g. "/dev/tap7498". The process manager opens it before exec and
+  // QEMU inherits it as a file descriptor. Custom mode only, mutually exclusive
+  // with netdev, and only the systemd process manager can pass the descriptor.
+  string open_file = 4;
 }
 
 // Requested GPU layout for a CVM.
@@ -186,7 +194,11 @@ message UpdateVmRequest {
   optional string image = 17;
   // Disable or re-enable TEE for an existing VM.
   optional bool no_tee = 18;
-  // Optional update networking.
+  // Optional update networking. Rejected while the VM is running: QEMU fixes
+  // its netdev at exec, so stop the VM, update, then start it. This RPC is the
+  // supported way to change networking; editing vm-manifest.json on disk
+  // bypasses the VMM's in-memory state and only takes effect once the whole
+  // VMM service restarts.
   bool update_networking = 19;
   // Networking list. Empty + update_networking=true resets to node default.
   repeated NetworkingConfig networks = 20;

dstack/vmm/src/app.rs

@@ -22,7 +22,7 @@ use dstack_vmm_rpc::{
 use fs_err as fs;
 use guest_api::client::DefaultClient as GuestClient;
 use id_pool::IdPool;
-use nix::unistd::{Uid, User};
+use nix::unistd::Uid;
 use or_panic::ResultOrPanic;
 use ra_rpc::client::RaClient;
 use serde::{Deserialize, Serialize};
@@ -521,11 +521,7 @@ impl App {
         let qemu_uid = if self.config.cvm.user.is_empty() {
             Uid::effective().as_raw()
         } else {
-            User::from_name(&self.config.cvm.user)
-                .context("failed to resolve QEMU user")?
-                .with_context(|| format!("QEMU user {} does not exist", self.config.cvm.user))?
-                .uid
-                .as_raw()
+            qemu::resolve_cvm_user(&self.config.cvm.user)?.uid.as_raw()
         };
         let mut prepared = Vec::new();
         for (nic_index, network) in networks.iter().enumerate() {
@@ -2166,6 +2162,7 @@ mod tests {
             dhcp_start: String::new(),
             restrict: false,
             netdev: String::new(),
+            open_file: String::new(),
         }];
 
         workdir.put_manifest(&manifest)?;
@@ -2428,6 +2425,7 @@ mod tests {
             dhcp_start: String::new(),
             restrict: false,
             netdev: String::new(),
+            open_file: String::new(),
         }];
         let user_manifest = test_manifest(2048);
         let image = test_tdx_image(true);

dstack/vmm/src/app/network.rs

@@ -10,7 +10,9 @@ use anyhow::{bail, Result};
 use sha2::{Digest, Sha256};
 
 use super::Manifest;
-use crate::config::{CvmConfig, Networking, NetworkingMode};
+use crate::config::{
+    validate_open_file, CvmConfig, Networking, NetworkingMode, SD_LISTEN_FDS_START,
+};
 
 pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking {
     let mut resolved = cfg.networking.clone();
@@ -28,9 +30,21 @@ pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Ne
     if !networking.dhcp_start.is_empty() {
         resolved.dhcp_start = networking.dhcp_start.clone();
     }
-    if !networking.netdev.is_empty() {
+    // A pre-opened chardev names one host device and belongs to exactly one
+    // NIC, so it is taken from the NIC verbatim and never inherited from the
+    // host defaults.
+    resolved.open_file = networking.open_file.clone();
+    if !networking.open_file.is_empty() && networking.netdev.is_empty() {
+        // The chardev generates this NIC's netdev from the fd number later, so
+        // drop any inherited host netdev rather than leak a stale default.
+        resolved.netdev = String::new();
+    } else if !networking.netdev.is_empty() {
+        // An explicit NIC netdev overrides the host default. A netdev set
+        // together with open_file is kept here, not dropped, so that
+        // validate_resolved_network rejects the conflicting pair.
         resolved.netdev = networking.netdev.clone();
     }
+    // Neither set: inherit the host netdev unchanged.
     resolved
 }
 
@@ -47,6 +61,17 @@ pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec<Net
 }
 
 pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> {
+    if !networking.open_file.is_empty() {
+        validate_open_file("networking.open_file", &networking.open_file)?;
+        if networking.mode != NetworkingMode::Custom {
+            bail!("networking.open_file requires mode = \"custom\"");
+        }
+        if !networking.netdev.is_empty() {
+            // The netdev string is generated from the inherited fd number,
+            // which only the process manager knows.
+            bail!("networking.open_file and networking.netdev are mutually exclusive");
+        }
+    }
     if networking.mode != NetworkingMode::Bridge {
         return Ok(());
     }
@@ -69,6 +94,32 @@ pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()>
     Ok(())
 }
 
+/// Chardev paths the process manager must open before exec, in NIC order.
+///
+/// The order is the contract: systemd hands the files to the service in
+/// declaration order, so entry `i` of this list arrives as fd
+/// `SD_LISTEN_FDS_START + i`.
+pub(crate) fn open_files(networks: &[Networking]) -> Vec<String> {
+    networks
+        .iter()
+        .filter(|networking| !networking.open_file.is_empty())
+        .map(|networking| networking.open_file.clone())
+        .collect()
+}
+
+/// File descriptor the NIC at `index` receives, or `None` if it does not use a
+/// pre-opened chardev.
+pub(crate) fn open_file_fd(networks: &[Networking], index: usize) -> Option<u32> {
+    if networks.get(index)?.open_file.is_empty() {
+        return None;
+    }
+    let preceding = networks[..index]
+        .iter()
+        .filter(|networking| !networking.open_file.is_empty())
+        .count();
+    Some(SD_LISTEN_FDS_START + preceding as u32)
+}
+
 /// Derives a deterministic, locally administered unicast MAC address.
 ///
 /// Index zero preserves the legacy single-NIC derivation. Later interfaces
@@ -95,7 +146,84 @@ pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize)
 
 #[cfg(test)]
 mod tests {
-    use super::mac_address_for_vm_index;
+    use super::{
+        mac_address_for_vm_index, open_file_fd, open_files, validate_resolved_network, Networking,
+        NetworkingMode,
+    };
+
+    fn open_file_network(path: &str) -> Networking {
+        Networking {
+            mode: NetworkingMode::Custom,
+            bridge: String::new(),
+            mac_prefix: String::new(),
+            net: String::new(),
+            dhcp_start: String::new(),
+            restrict: false,
+            netdev: String::new(),
+            open_file: path.into(),
+        }
+    }
+
+    #[test]
+    fn open_file_descriptors_are_numbered_in_nic_order() {
+        let mut networks = vec![
+            open_file_network(""),
+            open_file_network("/dev/tap10"),
+            open_file_network(""),
+            open_file_network("/dev/tap11"),
+        ];
+        networks[0].mode = NetworkingMode::User;
+        networks[2].mode = NetworkingMode::Bridge;
+
+        assert_eq!(open_files(&networks), ["/dev/tap10", "/dev/tap11"]);
+        assert_eq!(open_file_fd(&networks, 0), None);
+        assert_eq!(open_file_fd(&networks, 1), Some(3));
+        assert_eq!(open_file_fd(&networks, 2), None);
+        assert_eq!(open_file_fd(&networks, 3), Some(4));
+        assert_eq!(open_file_fd(&networks, 4), None);
+    }
+
+    #[test]
+    fn open_file_networks_are_validated() {
+        validate_resolved_network(&open_file_network("/dev/tap7498")).unwrap();
+
+        for path in [
+            "dev/tap7498",
+            "/dev/tap 7498",
+            "/dev/tap7498:foo",
+            "/dev/tap7498,vhost=on",
+            "/dev/%i/tap7498",
+        ] {
+            validate_resolved_network(&open_file_network(path)).unwrap_err();
+        }
+
+        let mut wrong_mode = open_file_network("/dev/tap7498");
+        wrong_mode.mode = NetworkingMode::Bridge;
+        wrong_mode.bridge = "br0".into();
+        validate_resolved_network(&wrong_mode).unwrap_err();
+
+        let mut with_netdev = open_file_network("/dev/tap7498");
+        with_netdev.netdev = "tap,id=net0,fd=3".into();
+        validate_resolved_network(&with_netdev).unwrap_err();
+    }
+
+    #[test]
+    fn open_file_does_not_inherit_host_custom_netdev() {
+        use rocket::figment::{providers::Format, providers::Toml, Figment};
+
+        let mut cfg: crate::config::Config =
+            Figment::from(Toml::string(crate::config::DEFAULT_CONFIG))
+                .extract()
+                .unwrap();
+        cfg.cvm.networking.mode = NetworkingMode::Custom;
+        cfg.cvm.networking.netdev = "tap,id=net0,ifname=legacy,script=no".into();
+
+        let nic = open_file_network("/dev/tap7498");
+        let resolved = super::resolve_networking(&nic, &cfg.cvm);
+        assert_eq!(resolved.open_file, "/dev/tap7498");
+        assert!(resolved.netdev.is_empty());
+        validate_resolved_network(&resolved).unwrap();
+    }
 
     #[test]
     fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() {