From ee5fa5bc4d42c059461c02108d38b7add9c66c63 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 16:16:52 +0000 Subject: [PATCH 01/12] feat(proof-vm): firecracker orchestrator client, agent api, wire protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit proof-rlm (additive): InspectOutcome / RunOutcome / VmJobOutput derive serde (adjacently tagged output/body) so an orchestrator answers over the wire with the same types; VmJob::topic_id / deadline_s / requires_firecracker helpers for the hard topic bind and per-job timeouts. proof-vm-proto: the HTTPS contract between the control plane and the proof-vm-orchestrator agent (create / attach / run / teardown, ErrorBody + codes, SisterAttestation) and the vsock framing + messages the agent speaks to the RLM guest (jobs, secret staging) and the sister miner guest (the run, no network). Types only; no challenge content. proof-vm-agent: bearer-from-file auth (constant-time, re-read per request, never logged), one running VM per topic_id, request topic and job topic must both equal the VM's, per-VM job lock, Hypervisor trait, and host stamping of paid outputs: sandboxed and flops_used come from the sister guest the host booted, never from the RLM's report. FakeHypervisor + in-process FakeAgent behind test-fixtures; nothing here spawns a process. proof-vm-fc: FirecrackerOrchestrator, the live TopicVmOrchestrator. Reads PROOF_VM_ORCHESTRATOR_URL / _TOKEN_FILE / PROOF_RLM_VM_IMAGE_DIGEST (4 vCPU / 8192 MiB default), https only (plain http on loopback for tests), from_env is None when unset (host keeps UnwiredVmOrchestrator), missing token or unpinned digest is NotWired naming the env var, agent down or bearer refused is Backend — all 503, no host fallback. Refuses a job for another topic before any request, an echo for another vm, a created vm on another image, and a firecracker_required run without the host's sister attestation. Co-authored-by: Mathis --- Cargo.lock | 49 ++ crates/proof-rlm/src/lib.rs | 10 +- crates/proof-rlm/src/runner.rs | 4 +- crates/proof-rlm/src/vm.rs | 113 ++++- crates/proof-vm-agent/Cargo.toml | 36 ++ crates/proof-vm-agent/src/auth.rs | 155 ++++++ crates/proof-vm-agent/src/fixtures_tests.rs | 305 +++++++++++ crates/proof-vm-agent/src/hypervisor.rs | 79 +++ crates/proof-vm-agent/src/lib.rs | 518 +++++++++++++++++++ crates/proof-vm-agent/src/router.rs | 356 +++++++++++++ crates/proof-vm-agent/src/stamp.rs | 167 ++++++ crates/proof-vm-fc/Cargo.toml | 28 + crates/proof-vm-fc/src/lib.rs | 535 ++++++++++++++++++++ crates/proof-vm-fc/tests/live_agent.rs | 376 ++++++++++++++ crates/proof-vm-proto/Cargo.toml | 24 + crates/proof-vm-proto/src/guest.rs | 371 ++++++++++++++ crates/proof-vm-proto/src/lib.rs | 307 +++++++++++ 17 files changed, 3421 insertions(+), 12 deletions(-) create mode 100644 crates/proof-vm-agent/Cargo.toml create mode 100644 crates/proof-vm-agent/src/auth.rs create mode 100644 crates/proof-vm-agent/src/fixtures_tests.rs create mode 100644 crates/proof-vm-agent/src/hypervisor.rs create mode 100644 crates/proof-vm-agent/src/lib.rs create mode 100644 crates/proof-vm-agent/src/router.rs create mode 100644 crates/proof-vm-agent/src/stamp.rs create mode 100644 crates/proof-vm-fc/Cargo.toml create mode 100644 crates/proof-vm-fc/src/lib.rs create mode 100644 crates/proof-vm-fc/tests/live_agent.rs create mode 100644 crates/proof-vm-proto/Cargo.toml create mode 100644 crates/proof-vm-proto/src/guest.rs create mode 100644 crates/proof-vm-proto/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a0bc953df..77ea09fb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3643,6 +3643,55 @@ dependencies = [ "toml", ] +[[package]] +name = "proof-vm-agent" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "http-body-util", + "proof-canon", + "proof-rlm", + "proof-vm-proto", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 2.0.19", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "proof-vm-fc" +version = "0.1.0" +dependencies = [ + "async-trait", + "proof-rlm", + "proof-vm-agent", + "proof-vm-proto", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "proof-vm-proto" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "proof-rlm", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "proptest" version = "1.11.0" diff --git a/crates/proof-rlm/src/lib.rs b/crates/proof-rlm/src/lib.rs index 2661e11c8..d503e29fd 100644 --- a/crates/proof-rlm/src/lib.rs +++ b/crates/proof-rlm/src/lib.rs @@ -21,10 +21,12 @@ //! default. An unregistered id is [`RunnerError::Unregistered`], which the //! host turns into a 503 before any row or rent. //! 4. [`TopicVmOrchestrator`] — create / attach / run / teardown for topic -//! VMs, with [`VmJob`]s that carry public data only. The shipped -//! implementation is [`UnwiredVmOrchestrator`] (refuses); the generic -//! [`VmBackedRunner`] turns inspect / evaluate into VM jobs and is only -//! ever registered by an operator. +//! VMs, with [`VmJob`]s that carry public data only. This crate ships +//! [`UnwiredVmOrchestrator`] (refuses); the live `FirecrackerOrchestrator` +//! (crate `proof-vm-fc`) is an HTTPS client of the `proof-vm-orchestrator` +//! agent on a dedicated KVM host. The generic [`VmBackedRunner`] turns +//! inspect / evaluate into VM jobs and is only ever registered by an +//! operator. //! 5. [`decide_promote`] — pass + green checklist + relative win over the //! bar, direction from the topic. //! diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs index 7fcd84798..3159efd1b 100644 --- a/crates/proof-rlm/src/runner.rs +++ b/crates/proof-rlm/src/runner.rs @@ -352,7 +352,7 @@ pub struct LogFile { } /// What inspection produced: the ticked checklist and the tree it looked at. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct InspectOutcome { /// One item per rule of the requested version. pub checklist: Checklist, @@ -361,7 +361,7 @@ pub struct InspectOutcome { } /// What a paid run produced. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunOutcome { /// Runner-authored measurement. pub report: CustomRunReport, diff --git a/crates/proof-rlm/src/vm.rs b/crates/proof-rlm/src/vm.rs index 715298a5a..9b6d3bb8c 100644 --- a/crates/proof-rlm/src/vm.rs +++ b/crates/proof-rlm/src/vm.rs @@ -9,10 +9,13 @@ //! never host paths, never keys), reads back documents, and tears the VM down //! or retains it by policy. //! -//! The only orchestrator shipped here is [`UnwiredVmOrchestrator`]: it -//! refuses every call and names the env vars a live one would read. There is -//! no host-local execution path in this crate — a missing orchestrator is a -//! 503, not a fallback. +//! The only orchestrator shipped in this crate is [`UnwiredVmOrchestrator`]: +//! it refuses every call and names the env vars a live one reads. The live +//! implementation (`FirecrackerOrchestrator`, crate `proof-vm-fc`) is a thin +//! HTTPS client of the `proof-vm-orchestrator` agent on a dedicated KVM host, +//! where jailer boots one Firecracker RLM VM per topic and every miner run is +//! a **sister** Firecracker guest. There is no host-local execution path +//! anywhere — a missing orchestrator is a 503, not a fallback. use std::sync::Arc; @@ -190,8 +193,47 @@ pub enum VmJob { }, } -/// What a job produced. -#[derive(Debug, Clone, PartialEq)] +impl VmJob { + /// The topic every job is attributed to. An orchestrator refuses a job + /// whose topic is not the one its VM is bound to. + #[must_use] + pub fn topic_id(&self) -> &str { + match self { + Self::ProposeRules { topic, .. } => &topic.id, + Self::Baseline { request } + | Self::Inspect { request, .. } + | Self::Evaluate { request, .. } => &request.topic_id, + Self::Archive { topic_id } => topic_id, + } + } + + /// Wall-clock budget the job's run request carries, if it carries one. + #[must_use] + pub fn deadline_s(&self) -> Option { + match self { + Self::Baseline { request } + | Self::Inspect { request, .. } + | Self::Evaluate { request, .. } => Some(request.sandbox.deadline_s), + Self::ProposeRules { .. } | Self::Archive { .. } => None, + } + } + + /// Whether the job runs miner code and the topic demands the guest. + #[must_use] + pub fn requires_firecracker(&self) -> bool { + match self { + Self::Baseline { request } | Self::Evaluate { request, .. } => { + request.sandbox.firecracker_required + } + Self::ProposeRules { .. } | Self::Inspect { .. } | Self::Archive { .. } => false, + } + } +} + +/// What a job produced. Serialised adjacently tagged (`output` / `body`) so +/// an orchestrator can answer over the wire with the same type. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "output", content = "body", rename_all = "snake_case")] pub enum VmJobOutput { /// Rules the RLM proposes; the store versions them. Rules(Vec), @@ -494,6 +536,65 @@ mod tests { )); } + /// Every job names its topic (the orchestrator's hard bind), the paid + /// jobs carry the deadline the guest is held to, and outputs round-trip + /// over the wire as the same type. + #[test] + fn jobs_name_their_topic_and_outputs_round_trip() { + let req = request(); + let jobs = [ + VmJob::ProposeRules { + topic: Box::new(crate::fixtures::topic()), + current_version: None, + }, + VmJob::Baseline { + request: req.clone(), + }, + VmJob::Inspect { + request: req.clone(), + rules: rules(), + }, + VmJob::Evaluate { + request: req.clone(), + checklist_digest: "c".into(), + rules_version: 1, + }, + VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + ]; + for job in &jobs { + assert_eq!(job.topic_id(), req.topic_id); + } + assert_eq!(jobs[0].deadline_s(), None); + assert_eq!(jobs[1].deadline_s(), Some(req.sandbox.deadline_s)); + assert_eq!(jobs[3].deadline_s(), Some(req.sandbox.deadline_s)); + assert!(jobs[1].requires_firecracker() && jobs[3].requires_firecracker()); + assert!( + !jobs[2].requires_firecracker(), + "inspection runs no miner code" + ); + let outputs = [ + VmJobOutput::Rules(rules().rules), + VmJobOutput::Baseline(crate::fixtures::report_for(&req, 0.5)), + VmJobOutput::Inspected(crate::runner::InspectOutcome { + checklist: crate::fixtures::green(&rules(), &req.submission_digest), + artifact: vec![], + }), + VmJobOutput::Evaluated(crate::runner::RunOutcome { + report: crate::fixtures::report_for(&req, 0.5), + logs: vec![], + }), + VmJobOutput::Archived, + ]; + for out in outputs { + let json = serde_json::to_string(&out).expect("json"); + assert!(json.contains("\"output\""), "{json}"); + let back: VmJobOutput = serde_json::from_str(&json).expect("round trip"); + assert_eq!(back, out); + } + } + #[test] fn env_names_are_names_only() { assert_eq!(VM_ORCHESTRATOR_URL_ENV, "PROOF_VM_ORCHESTRATOR_URL"); diff --git a/crates/proof-vm-agent/Cargo.toml b/crates/proof-vm-agent/Cargo.toml new file mode 100644 index 000000000..b08e67a2a --- /dev/null +++ b/crates/proof-vm-agent/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "proof-vm-agent" +description = "proof-vm-orchestrator agent library: bearer-authenticated HTTP API (create / attach / run / teardown) with a hard topic_id ↔ VM bind, the Hypervisor trait the Firecracker backend implements, and host stamping of paid outputs (sandboxed / flops_used come from the sister guest the host booted, never from the RLM). The fake hypervisor ships behind test-fixtures; CI never boots a VM." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[features] +# Exposes `proof_vm_agent::fixtures::FakeHypervisor` to sibling crates' tests. +test-fixtures = [] + +[dependencies] +async-trait = "0.1" +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +proof-canon = { path = "../proof-canon" } +proof-rlm = { path = "../proof-rlm" } +proof-vm-proto = { path = "../proof-vm-proto" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +subtle = "2" +thiserror = "2" +tokio = { version = "1", features = ["sync", "rt", "time", "net"] } +tracing = "0.1" + +[dev-dependencies] +http-body-util = "0.1" +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/proof-vm-agent/src/auth.rs b/crates/proof-vm-agent/src/auth.rs new file mode 100644 index 000000000..ff51e5d02 --- /dev/null +++ b/crates/proof-vm-agent/src/auth.rs @@ -0,0 +1,155 @@ +//! Bearer authentication from a token **file**. +//! +//! The file is re-read on every check, so an operator rotates the token by +//! rewriting the file — no restart, no env var, nothing in a process listing. +//! A missing or empty file refuses every request (fail-closed). Tokens are +//! compared as SHA-256 digests in constant time and are never logged. + +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; +use subtle::ConstantTimeEq; + +/// Why a request was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AuthError { + /// No usable token on the host: nothing can authenticate. + #[error("agent has no bearer token file; refusing every request")] + NoToken, + /// Header missing, malformed, or wrong. + #[error("bearer rejected")] + Rejected, +} + +/// Bearer check backed by a token file. +pub struct BearerAuth { + token_file: PathBuf, +} + +impl std::fmt::Debug for BearerAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BearerAuth") + .field("token_file", &self.token_file) + .finish() + } +} + +impl BearerAuth { + /// Authenticate against the trimmed contents of `token_file`. + #[must_use] + pub fn from_file(token_file: &Path) -> Self { + Self { + token_file: token_file.to_path_buf(), + } + } + + /// The file this checks (for logs; never the contents). + #[must_use] + pub fn token_file(&self) -> &Path { + &self.token_file + } + + fn expected_digest(&self) -> Option<[u8; 32]> { + let raw = std::fs::read_to_string(&self.token_file).ok()?; + let token = raw.trim(); + if token.is_empty() { + return None; + } + Some(Sha256::digest(token.as_bytes()).into()) + } + + /// Whether the host has a usable token at all. + #[must_use] + pub fn configured(&self) -> bool { + self.expected_digest().is_some() + } + + /// Check an `Authorization` header value. + /// + /// # Errors + /// + /// [`AuthError::NoToken`] when the file is missing / empty, + /// [`AuthError::Rejected`] otherwise on mismatch. + pub fn accepts(&self, authorization: Option<&str>) -> Result<(), AuthError> { + let expected = self.expected_digest().ok_or(AuthError::NoToken)?; + let presented = authorization + .and_then(|h| { + h.strip_prefix("Bearer ") + .or_else(|| h.strip_prefix("bearer ")) + }) + .map(str::trim) + .filter(|t| !t.is_empty()) + .ok_or(AuthError::Rejected)?; + let got: [u8; 32] = Sha256::digest(presented.as_bytes()).into(); + if bool::from(got.ct_eq(&expected)) { + Ok(()) + } else { + Err(AuthError::Rejected) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("proof-vm-agent-auth-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + dir.join(name) + } + + #[test] + fn a_missing_or_empty_file_refuses_everything() { + let auth = BearerAuth::from_file(&tmp("missing")); + assert!(!auth.configured()); + assert_eq!( + auth.accepts(Some("Bearer anything")), + Err(AuthError::NoToken) + ); + let empty = tmp("empty"); + std::fs::write(&empty, " \n").expect("write"); + let auth = BearerAuth::from_file(&empty); + assert_eq!(auth.accepts(Some("Bearer x")), Err(AuthError::NoToken)); + } + + #[test] + fn debug_shows_the_path_never_the_token() { + let file = tmp("debug"); + std::fs::write(&file, "debug-token-not-a-real-secret\n").expect("write"); + let auth = BearerAuth::from_file(&file); + let dump = format!("{auth:?}"); + assert!(dump.contains("token_file"), "{dump}"); + assert!(!dump.contains("debug-token-not-a-real-secret"), "{dump}"); + assert_eq!(auth.token_file(), file.as_path()); + } + + #[test] + fn the_file_contents_are_the_token_and_rotate_without_restart() { + let file = tmp("token"); + std::fs::write(&file, " first-token-not-a-real-secret \n").expect("write"); + let auth = BearerAuth::from_file(&file); + assert!(auth.configured()); + auth.accepts(Some("Bearer first-token-not-a-real-secret")) + .expect("exact"); + auth.accepts(Some("bearer first-token-not-a-real-secret ")) + .expect("case-insensitive scheme, trimmed"); + assert_eq!(auth.accepts(None), Err(AuthError::Rejected)); + assert_eq!(auth.accepts(Some("Bearer ")), Err(AuthError::Rejected)); + assert_eq!( + auth.accepts(Some("Basic first-token-not-a-real-secret")), + Err(AuthError::Rejected) + ); + assert_eq!( + auth.accepts(Some("Bearer first-token-not-a-real-secre")), + Err(AuthError::Rejected) + ); + std::fs::write(&file, "second-token-not-a-real-secret\n").expect("rotate"); + assert_eq!( + auth.accepts(Some("Bearer first-token-not-a-real-secret")), + Err(AuthError::Rejected) + ); + auth.accepts(Some("Bearer second-token-not-a-real-secret")) + .expect("rotated"); + } +} diff --git a/crates/proof-vm-agent/src/fixtures_tests.rs b/crates/proof-vm-agent/src/fixtures_tests.rs new file mode 100644 index 000000000..a3f62a596 --- /dev/null +++ b/crates/proof-vm-agent/src/fixtures_tests.rs @@ -0,0 +1,305 @@ +//! Test fixtures: a recording fake [`Hypervisor`] and an in-process agent +//! server. No process is spawned, no VM boots — this is what CI runs. +//! Compiled for tests and the `test-fixtures` feature only. + +// Test-only code: never compiled into a host binary (see the cfg in lib.rs). +#![allow( + clippy::missing_panics_doc, + clippy::must_use_candidate, + clippy::expect_used, + clippy::unwrap_used +)] + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use proof_canon::ChecklistRule; +use proof_rlm::fixtures::report_for; +use proof_rlm::{ + ArtifactFile, Checklist, CustomRunRequest, InspectOutcome, LogFile, RetainPolicy, RunOutcome, + TopicVmSpec, VmJob, VmJobOutput, +}; +use proof_vm_proto::SisterAttestation; + +use crate::auth::BearerAuth; +use crate::hypervisor::{BootedVm, HvError, Hypervisor, JobOutcome}; +use crate::router::{agent_router, AgentState}; + +/// Digest of the fake miner-guest image. +pub fn miner_image_digest() -> String { + format!("sha256:{}", "dd".repeat(32)) +} + +/// Records every call; answers with canned documents. +pub struct FakeHypervisor { + ready: AtomicBool, + fail_boot: AtomicBool, + primary: Mutex, + red: Mutex>, + /// Whether paid jobs boot a sister guest (the host's view). + sister: AtomicBool, + /// What that sister measures. + sister_flops: Mutex>, + /// What the RLM writes into its own report before the host stamps it. + rlm_claims_sandboxed: AtomicBool, + rlm_flops: Mutex>, + proposed: Mutex>, + job_delay: Mutex>, + boots: Mutex>, + jobs: Mutex>, + teardowns: Mutex>, +} + +impl FakeHypervisor { + pub fn new(primary: f64) -> Arc { + Arc::new(Self { + ready: AtomicBool::new(true), + fail_boot: AtomicBool::new(false), + primary: Mutex::new(primary), + red: Mutex::new(None), + sister: AtomicBool::new(true), + sister_flops: Mutex::new(Some(1)), + rlm_claims_sandboxed: AtomicBool::new(false), + rlm_flops: Mutex::new(Some(999_999)), + proposed: Mutex::new(vec![ChecklistRule { + id: "rlm_rule".into(), + text: "a rule the fake rlm wrote".into(), + }]), + job_delay: Mutex::new(None), + boots: Mutex::new(Vec::new()), + jobs: Mutex::new(Vec::new()), + teardowns: Mutex::new(Vec::new()), + }) + } + + pub fn set_ready(&self, v: bool) { + self.ready.store(v, Ordering::SeqCst); + } + + pub fn set_fail_boot(&self, v: bool) { + self.fail_boot.store(v, Ordering::SeqCst); + } + + pub fn set_primary(&self, v: f64) { + *self.primary.lock().unwrap() = v; + } + + /// Make inspections fail this rule id (None = green). + pub fn set_red(&self, id: Option<&str>) { + *self.red.lock().unwrap() = id.map(str::to_owned); + } + + /// Whether the host boots a sister for paid jobs. + pub fn set_sister(&self, v: bool) { + self.sister.store(v, Ordering::SeqCst); + } + + pub fn set_sister_flops(&self, v: Option) { + *self.sister_flops.lock().unwrap() = v; + } + + /// What the RLM claims before the host corrects it. + pub fn set_rlm_claims_sandboxed(&self, v: bool) { + self.rlm_claims_sandboxed.store(v, Ordering::SeqCst); + } + + pub fn set_rlm_flops(&self, v: Option) { + *self.rlm_flops.lock().unwrap() = v; + } + + pub fn set_proposed(&self, rules: Vec) { + *self.proposed.lock().unwrap() = rules; + } + + /// Make every job take this long (to exercise `Busy`). + pub fn set_job_delay(&self, d: Option) { + *self.job_delay.lock().unwrap() = d; + } + + pub fn boots(&self) -> Vec { + self.boots.lock().unwrap().clone() + } + + pub fn jobs(&self) -> Vec<(String, VmJob)> { + self.jobs.lock().unwrap().clone() + } + + pub fn teardowns(&self) -> Vec<(String, RetainPolicy)> { + self.teardowns.lock().unwrap().clone() + } + + fn rlm_report(&self, req: &CustomRunRequest) -> proof_rlm::CustomRunReport { + let mut r = report_for(req, *self.primary.lock().unwrap()); + r.sandboxed = self.rlm_claims_sandboxed.load(Ordering::SeqCst); + r.flops_used = *self.rlm_flops.lock().unwrap(); + r + } + + fn sister_for(&self, vm: &BootedVm, req: &CustomRunRequest) -> Option { + if !self.sister.load(Ordering::SeqCst) { + return None; + } + Some(SisterAttestation { + sister_vm_id: format!("{}-s{}", vm.vm_id, req.submission_digest.len()), + image_digest: miner_image_digest(), + sandboxed: true, + network: "none".into(), + flops_used: *self.sister_flops.lock().unwrap(), + wall_ms: 10, + exit_code: Some(0), + }) + } +} + +#[async_trait] +impl Hypervisor for FakeHypervisor { + fn name(&self) -> &'static str { + "fake" + } + + fn ready(&self) -> Result<(), HvError> { + if self.ready.load(Ordering::SeqCst) { + Ok(()) + } else { + Err(HvError::NotReady("fake hypervisor told to refuse".into())) + } + } + + async fn boot(&self, vm_id: &str, spec: &TopicVmSpec) -> Result { + if self.fail_boot.load(Ordering::SeqCst) { + return Err(HvError::Backend("fake boot failure".into())); + } + let vm = BootedVm { + vm_id: vm_id.to_owned(), + topic_id: spec.topic_id.clone(), + image_digest: spec.template.image_digest.clone(), + }; + self.boots.lock().unwrap().push(vm.clone()); + Ok(vm) + } + + async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result { + self.jobs + .lock() + .unwrap() + .push((vm.vm_id.clone(), job.clone())); + let delay = *self.job_delay.lock().unwrap(); + if let Some(d) = delay { + tokio::time::sleep(d).await; + } + Ok(match job { + VmJob::ProposeRules { .. } => JobOutcome { + output: VmJobOutput::Rules(self.proposed.lock().unwrap().clone()), + sister: None, + }, + VmJob::Baseline { request } => JobOutcome { + output: VmJobOutput::Baseline(self.rlm_report(request)), + sister: self.sister_for(vm, request), + }, + VmJob::Inspect { request, rules } => { + let red = self.red.lock().unwrap().clone(); + let mut checklist = + Checklist::new(rules, &request.submission_digest, &request.artifact_digest); + for r in &rules.rules { + checklist.record(&r.id, red.as_deref() != Some(r.id.as_str()), &r.text); + } + JobOutcome { + output: VmJobOutput::Inspected(InspectOutcome { + checklist, + artifact: vec![ArtifactFile { + path: "src/main.rs".into(), + bytes: b"fn main() {}\n".to_vec(), + }], + }), + sister: None, + } + } + VmJob::Evaluate { request, .. } => JobOutcome { + output: VmJobOutput::Evaluated(RunOutcome { + report: self.rlm_report(request), + logs: vec![LogFile { + name: "run.log".into(), + bytes: b"ok\n".to_vec(), + }], + }), + sister: self.sister_for(vm, request), + }, + VmJob::Archive { .. } => JobOutcome { + output: VmJobOutput::Archived, + sister: None, + }, + }) + } + + async fn teardown(&self, vm: &BootedVm, policy: RetainPolicy) -> Result { + self.teardowns + .lock() + .unwrap() + .push((vm.vm_id.clone(), policy)); + Ok(true) + } +} + +/// Write `token` to a fresh temp file and return its path. +pub fn token_file(tag: &str, token: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("proof-vm-agent-{}-{tag}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let path = dir.join("token"); + std::fs::write(&path, format!("{token}\n")).expect("write"); + path +} + +/// A running in-process agent on a loopback port. +pub struct FakeAgent { + /// Where it listens (`http://127.0.0.1:port`). + pub addr: SocketAddr, + /// The fake behind it. + pub hypervisor: Arc, + /// Shared state (for assertions). + pub state: AgentState, + task: tokio::task::JoinHandle<()>, +} + +impl FakeAgent { + /// Serve the agent router over `hypervisor`, authenticating against `token_file`. + pub async fn serve(hypervisor: Arc, token_file: &Path) -> Self { + let state = AgentState::new( + hypervisor.clone(), + Arc::new(BearerAuth::from_file(token_file)), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let addr = listener.local_addr().expect("addr"); + let app = agent_router(state.clone()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Self { + addr, + hypervisor, + state, + task, + } + } + + /// `http://127.0.0.1:port`. + pub fn url(&self) -> String { + format!("http://{}", self.addr) + } + + /// Stop serving (later requests fail to connect). + pub fn stop(&self) { + self.task.abort(); + } +} + +impl Drop for FakeAgent { + fn drop(&mut self) { + self.task.abort(); + } +} diff --git a/crates/proof-vm-agent/src/hypervisor.rs b/crates/proof-vm-agent/src/hypervisor.rs new file mode 100644 index 000000000..9440c10ef --- /dev/null +++ b/crates/proof-vm-agent/src/hypervisor.rs @@ -0,0 +1,79 @@ +//! The backend contract the agent drives. +//! +//! A [`Hypervisor`] boots one RLM VM per topic from a digest-pinned image, +//! runs jobs inside it, boots a **sister** miner guest when a paid job asks +//! for one, and tears the VM down or retains it. The Firecracker + jailer +//! implementation lives in `proof-fc-host`; tests use the fake behind the +//! `test-fixtures` feature. Nothing in this crate spawns a process. + +use async_trait::async_trait; +use proof_rlm::{RetainPolicy, TopicVmSpec, VmJob, VmJobOutput}; +use proof_vm_proto::SisterAttestation; + +/// Why the backend refused or failed. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum HvError { + /// Cannot boot anything right now (binaries, `/dev/kvm`, kernel pin). + #[error("hypervisor not ready: {0}")] + NotReady(String), + /// The spec asks for something this host refuses. + #[error("spec: {0}")] + Spec(String), + /// The pinned image is absent or its bytes do not hash to the digest. + #[error("image {0}: not present or does not verify")] + Image(String), + /// The guest agent failed or answered badly. + #[error("guest: {0}")] + Guest(String), + /// Process / filesystem / network plumbing failed. + #[error("hypervisor: {0}")] + Backend(String), + /// The job's deadline passed before the guest answered. + #[error("job deadline of {0}s passed")] + Deadline(u64), +} + +/// One booted topic VM as the backend tracks it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootedVm { + /// Host VM id (also the jail id). + pub vm_id: String, + /// Topic the VM is bound to. + pub topic_id: String, + /// `sha256:` digest verified before boot. + pub image_digest: String, +} + +/// What one job produced, with the host's view of any sister run. +#[derive(Debug, Clone, PartialEq)] +pub struct JobOutcome { + /// The guest's document (paid outputs are re-stamped by the agent). + pub output: VmJobOutput, + /// Filled by the **host** iff it booted a sister guest for this job. + pub sister: Option, +} + +/// Boot / run / teardown for topic VMs. +#[async_trait] +pub trait Hypervisor: Send + Sync { + /// Backend name shown on `/v1/health`. + fn name(&self) -> &'static str; + + /// Whether a VM could boot right now. + /// + /// # Errors + /// + /// [`HvError::NotReady`] naming what is missing (never a secret). + fn ready(&self) -> Result<(), HvError>; + + /// Boot the RLM VM for `spec` under `vm_id`, verify the image digest + /// first, wait for the guest agent, stage owner key material. + async fn boot(&self, vm_id: &str, spec: &TopicVmSpec) -> Result; + + /// Run one job inside the VM, booting a sister guest if the RLM asks. + async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result; + + /// Stop the VM; keep its scratch under `Retain`. `Ok(true)` only when the + /// requested end state was reached. + async fn teardown(&self, vm: &BootedVm, policy: RetainPolicy) -> Result; +} diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs new file mode 100644 index 000000000..3f7549d56 --- /dev/null +++ b/crates/proof-vm-agent/src/lib.rs @@ -0,0 +1,518 @@ +//! `proof-vm-orchestrator` agent library. +//! +//! The agent runs on a **dedicated KVM host** (never the control-plane +//! droplet, never a Lium pod) and is the only thing that talks to Firecracker. +//! The Proof control plane reaches it over HTTPS with a bearer read from a +//! file ([`BearerAuth`]) and drives four verbs (`proof_vm_proto::paths`): +//! +//! | Verb | Route | Bind | +//! |------|-------|------| +//! | create | `POST /v1/vms` | one running VM per `topic_id`; a second create is 409 | +//! | attach | `GET /v1/vms/by-topic/{topic_id}` | 404 when the topic has no running VM | +//! | run | `POST /v1/vms/{vm_id}/jobs` | request `topic_id` **and** the job's own topic must equal the VM's | +//! | teardown | `DELETE /v1/vms/{vm_id}` | request `topic_id` must equal the VM's; destroy or retain | +//! +//! The agent never mounts a host path into a guest, never receives a key +//! from the control plane, and stamps `sandboxed` / `flops_used` on paid +//! outputs from the sister guest **it** booted ([`stamp_output`]). The +//! [`Hypervisor`] behind it is Firecracker + jailer in production +//! (`proof-fc-host`) and [`fixtures::FakeHypervisor`] in every test — no +//! test here or in CI boots a VM. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +mod auth; +mod hypervisor; +mod router; +mod stamp; + +/// Fake hypervisor + in-process agent. Test builds and the `test-fixtures` +/// feature only; never part of a host binary. +#[cfg(any(test, feature = "test-fixtures"))] +#[path = "fixtures_tests.rs"] +pub mod fixtures; + +pub use auth::{AuthError, BearerAuth}; +pub use hypervisor::{BootedVm, HvError, Hypervisor, JobOutcome}; +pub use router::{agent_router, AgentError, AgentState}; +pub use stamp::{output_matches, stamp_output}; + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use http_body_util::BodyExt; + use proof_rlm::fixtures::{pinned_template, request, rules, token_for}; + use proof_rlm::{RetainPolicy, TopicVmSpec, VmJob, VmJobOutput}; + use proof_vm_proto::{ + paths, AgentHealth, CreateVmRequest, ErrorBody, ErrorCode, RunJobRequest, RunJobResponse, + TeardownRequest, TeardownResponse, VmRecord, VmState, + }; + use tower::ServiceExt; + + use super::fixtures::{token_file, FakeHypervisor}; + use super::*; + + const TOKEN: &str = "agent-test-token-not-a-real-secret"; + + fn app(hv: Arc, tag: &str) -> (axum::Router, AgentState) { + let auth = Arc::new(BearerAuth::from_file(&token_file(tag, TOKEN))); + let state = AgentState::new(hv, auth); + (agent_router(state.clone()), state) + } + + async fn call( + app: &axum::Router, + method: &str, + path: &str, + bearer: Option<&str>, + body: Option, + ) -> (StatusCode, T) { + let mut req = Request::builder().method(method).uri(path); + if let Some(b) = bearer { + req = req.header("authorization", format!("Bearer {b}")); + } + let req = match body { + Some(v) => req + .header("content-type", "application/json") + .body(Body::from(v.to_string())), + None => req.body(Body::empty()), + } + .expect("request"); + let resp = app.clone().oneshot(req).await.expect("response"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let parsed = serde_json::from_slice(&bytes) + .unwrap_or_else(|e| panic!("{status} body {:?}: {e}", String::from_utf8_lossy(&bytes))); + (status, parsed) + } + + fn spec() -> TopicVmSpec { + let req = request(); + TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox) + } + + async fn create(app: &axum::Router) -> VmRecord { + let (status, rec): (StatusCode, VmRecord) = call( + app, + "POST", + paths::VMS, + Some(TOKEN), + Some(serde_json::to_value(CreateVmRequest { spec: spec() }).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + rec + } + + #[tokio::test] + async fn every_route_needs_the_bearer_including_health() { + let (app, _) = app(FakeHypervisor::new(0.5), "auth"); + for (method, path) in [ + ("GET", paths::HEALTH.to_owned()), + ("POST", paths::VMS.to_owned()), + ("GET", paths::vm_by_topic("topic-a")), + ("POST", paths::vm_jobs("x")), + ("DELETE", paths::vm("x")), + ] { + let (status, err): (StatusCode, ErrorBody) = + call(&app, method, &path, None, None).await; + assert_eq!(status, StatusCode::UNAUTHORIZED, "{method} {path}"); + assert_eq!(err.code, ErrorCode::Unauthorized); + let (status, _): (StatusCode, ErrorBody) = + call(&app, method, &path, Some("wrong-token"), None).await; + assert_eq!(status, StatusCode::UNAUTHORIZED, "{method} {path}"); + } + let (status, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!(status, StatusCode::OK); + assert!(health.ready); + assert_eq!(health.hypervisor, "fake"); + assert_eq!(health.vms, 0); + } + + #[tokio::test] + async fn one_vm_per_topic_then_attach_run_and_destroy() { + let hv = FakeHypervisor::new(0.8); + let (app, state) = app(hv.clone(), "flow"); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(err.code, ErrorCode::NotFound); + + let rec = create(&app).await; + assert_eq!(rec.handle.topic_id, "topic-a"); + assert!( + rec.handle.vm_id.starts_with("topic-a-"), + "{}", + rec.handle.vm_id + ); + assert_eq!(rec.state, VmState::Running); + assert_eq!(rec.image_digest, pinned_template().image_digest); + assert_eq!((rec.vcpus, rec.mem_mib), (2, 4_096)); + assert_eq!(hv.boots().len(), 1); + + let (status, dup): (StatusCode, ErrorBody) = call( + &app, + "POST", + paths::VMS, + Some(TOKEN), + Some(serde_json::to_value(CreateVmRequest { spec: spec() }).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(dup.code, ErrorCode::AlreadyExists); + assert_eq!(hv.boots().len(), 1, "no second boot"); + + let (status, attached): (StatusCode, VmRecord) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(attached, rec); + assert_eq!(state.running().await.len(), 1); + } + + #[tokio::test] + async fn a_job_runs_on_the_bound_vm_and_destroy_removes_it() { + let hv = FakeHypervisor::new(0.8); + let (app, _) = app(hv.clone(), "destroy"); + let rec = create(&app).await; + let req = request(); + let job = VmJob::Inspect { + request: req.clone(), + rules: rules(), + }; + let (status, out): (StatusCode, RunJobResponse) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(out.vm_id, rec.handle.vm_id); + assert!(out.sister.is_none(), "inspection boots no sister"); + assert!(matches!(out.output, VmJobOutput::Inspected(_))); + + let (status, down): (StatusCode, TeardownResponse) = call( + &app, + "DELETE", + &paths::vm(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(TeardownRequest { + topic_id: "topic-a".into(), + policy: RetainPolicy::Destroy, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(down.confirmed); + assert_eq!(down.state, VmState::Destroyed); + assert_eq!( + hv.teardowns(), + vec![(rec.handle.vm_id.clone(), RetainPolicy::Destroy)] + ); + let (status, _): (StatusCode, ErrorBody) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "destroyed vms are gone"); + let (status, _): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + } + + /// A job or teardown that names another topic than the VM's never reaches + /// the hypervisor — whether the mismatch is in the envelope or the job. + #[tokio::test] + async fn the_topic_bind_is_hard_on_both_the_envelope_and_the_job() { + let hv = FakeHypervisor::new(0.8); + let (app, _) = app(hv.clone(), "bind"); + let rec = create(&app).await; + let req = request(); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: "topic-b".into(), + job: VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(err.code, ErrorCode::TopicMismatch); + let mut other = req.clone(); + other.topic_id = "topic-b".into(); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Baseline { request: other }, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(err.code, ErrorCode::TopicMismatch); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "DELETE", + &paths::vm(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(TeardownRequest { + topic_id: "topic-b".into(), + policy: RetainPolicy::Destroy, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(err.code, ErrorCode::TopicMismatch); + assert!(hv.jobs().is_empty(), "nothing reached the hypervisor"); + assert!(hv.teardowns().is_empty()); + } + + /// Paid outputs carry the host's facts: the RLM's `sandboxed` claim is + /// replaced by whether a sister booted, and `flops_used` by what the + /// sister measured. + #[tokio::test] + async fn paid_outputs_are_host_stamped_and_carry_the_attestation() { + let hv = FakeHypervisor::new(0.8); + hv.set_rlm_claims_sandboxed(true); + hv.set_rlm_flops(Some(123)); + hv.set_sister_flops(Some(7)); + let (app, _) = app(hv.clone(), "stamp"); + let rec = create(&app).await; + let req = request(); + let evaluate = |req: &proof_rlm::CustomRunRequest| RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Evaluate { + request: req.clone(), + checklist_digest: token_for(req).checklist_digest().to_owned(), + rules_version: 1, + }, + }; + let (status, out): (StatusCode, RunJobResponse) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(serde_json::to_value(evaluate(&req)).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::OK); + let sister = out.sister.expect("sister attested"); + assert!(sister.sandboxed); + assert_eq!(sister.network, "none"); + let VmJobOutput::Evaluated(run) = out.output else { + panic!("shape"); + }; + assert!(run.report.sandboxed); + assert_eq!(run.report.flops_used, Some(7), "sister measurement wins"); + run.report.verify(&req).expect("bound"); + + hv.set_sister(false); + let (status, out): (StatusCode, RunJobResponse) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(serde_json::to_value(evaluate(&req)).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(out.sister.is_none()); + let VmJobOutput::Evaluated(run) = out.output else { + panic!("shape"); + }; + assert!( + !run.report.sandboxed, + "no sister, the RLM's claim is overruled" + ); + assert_eq!(run.report.flops_used, Some(123), "the RLM ran it itself"); + assert!(matches!( + run.report.verify(&req), + Err(proof_rlm::ReportError::NotSandboxed) + )); + } + + #[tokio::test] + async fn a_busy_vm_refuses_a_second_job_and_retain_keeps_the_record() { + let hv = FakeHypervisor::new(0.8); + hv.set_job_delay(Some(std::time::Duration::from_millis(300))); + let (app, state) = app(hv.clone(), "busy"); + let rec = create(&app).await; + let req = request(); + let body = serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + }) + .expect("json"); + let slow = { + let app = app.clone(); + let body = body.clone(); + let path = paths::vm_jobs(&rec.handle.vm_id); + tokio::spawn(async move { + let (status, _): (StatusCode, RunJobResponse) = + call(&app, "POST", &path, Some(TOKEN), Some(body)).await; + status + }) + }; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(body), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(err.code, ErrorCode::Busy); + assert_eq!(slow.await.expect("join"), StatusCode::OK); + + let (status, down): (StatusCode, TeardownResponse) = call( + &app, + "DELETE", + &paths::vm(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(TeardownRequest { + topic_id: "topic-a".into(), + policy: RetainPolicy::Retain, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(down.state, VmState::Retained); + assert!(state.running().await.is_empty(), "retained is not running"); + let (status, _): (StatusCode, ErrorBody) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a retained vm is not attachable" + ); + let rec2 = create(&app).await; + assert_ne!( + rec2.handle.vm_id, rec.handle.vm_id, + "fresh vm id after retain" + ); + } + + #[tokio::test] + async fn bad_specs_and_an_unready_hypervisor_never_boot() { + let hv = FakeHypervisor::new(0.8); + let (app, _) = app(hv.clone(), "spec"); + let mut unpinned = spec(); + unpinned.template.image_digest.clear(); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + paths::VMS, + Some(TOKEN), + Some(serde_json::to_value(CreateVmRequest { spec: unpinned }).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(err.code, ErrorCode::BadSpec); + hv.set_ready(false); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + paths::VMS, + Some(TOKEN), + Some(serde_json::to_value(CreateVmRequest { spec: spec() }).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(err.code, ErrorCode::NotReady); + let (_, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert!(!health.ready); + assert!(health.reason.contains("refuse")); + hv.set_ready(true); + hv.set_fail_boot(true); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + paths::VMS, + Some(TOKEN), + Some(serde_json::to_value(CreateVmRequest { spec: spec() }).expect("json")), + ) + .await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(err.code, ErrorCode::Backend); + assert!(hv.boots().is_empty()); + } +} diff --git a/crates/proof-vm-agent/src/router.rs b/crates/proof-vm-agent/src/router.rs new file mode 100644 index 000000000..be53e875c --- /dev/null +++ b/crates/proof-vm-agent/src/router.rs @@ -0,0 +1,356 @@ +//! The agent's HTTP surface: create / attach / run / teardown, bearer-gated, +//! with the topic ↔ VM bind enforced on every call that names a VM. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use axum::extract::{Path, Request, State}; +use axum::http::{header, StatusCode}; +use axum::middleware::{self, Next}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use proof_rlm::{RetainPolicy, VmHandle}; +use proof_vm_proto::{ + paths, AgentHealth, CreateVmRequest, ErrorBody, ErrorCode, RunJobRequest, RunJobResponse, + TeardownRequest, TeardownResponse, VmRecord, VmState, API_VERSION, +}; +use tokio::sync::{Mutex, RwLock}; + +use crate::auth::BearerAuth; +use crate::hypervisor::{BootedVm, HvError, Hypervisor}; +use crate::stamp::{output_matches, stamp_output}; + +/// Longest `vm_id` the agent mints (jailer ids are capped at 64 chars). +const MAX_VM_ID_LEN: usize = 63; + +/// One JSON error answer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentError { + /// Class (also the status). + pub code: ErrorCode, + /// Detail. Never a secret. + pub error: String, +} + +impl AgentError { + fn new(code: ErrorCode, error: impl Into) -> Self { + Self { + code, + error: error.into(), + } + } +} + +impl From for AgentError { + fn from(e: HvError) -> Self { + let code = match e { + HvError::NotReady(_) | HvError::Image(_) => ErrorCode::NotReady, + HvError::Spec(_) => ErrorCode::BadSpec, + HvError::Guest(_) | HvError::Backend(_) | HvError::Deadline(_) => ErrorCode::Backend, + }; + Self::new(code, e.to_string()) + } +} + +impl IntoResponse for AgentError { + fn into_response(self) -> Response { + let status = + StatusCode::from_u16(self.code.status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + ( + status, + Json(ErrorBody { + code: self.code, + error: self.error, + }), + ) + .into_response() + } +} + +struct VmEntry { + record: VmRecord, + booted: BootedVm, + /// One job (or teardown) at a time per VM. + lock: Arc>, +} + +struct Inner { + hypervisor: Arc, + auth: Arc, + vms: RwLock>, + create_lock: Mutex<()>, + next_id: AtomicU64, +} + +/// Shared agent state. +#[derive(Clone)] +pub struct AgentState { + inner: Arc, +} + +impl AgentState { + /// State over `hypervisor`, gated by `auth`. + #[must_use] + pub fn new(hypervisor: Arc, auth: Arc) -> Self { + Self { + inner: Arc::new(Inner { + hypervisor, + auth, + vms: RwLock::new(BTreeMap::new()), + create_lock: Mutex::new(()), + next_id: AtomicU64::new(1), + }), + } + } + + /// Running VMs, by id. + pub async fn running(&self) -> Vec { + self.inner + .vms + .read() + .await + .values() + .filter(|e| e.record.state == VmState::Running) + .map(|e| e.record.clone()) + .collect() + } + + fn mint_vm_id(&self, topic_id: &str) -> String { + let n = self.inner.next_id.fetch_add(1, Ordering::SeqCst); + let suffix = format!("-{n:04}"); + let keep = MAX_VM_ID_LEN.saturating_sub(suffix.len()); + let mut prefix = topic_id.to_owned(); + prefix.truncate(keep); + format!("{prefix}{suffix}") + } + + async fn entry(&self, vm_id: &str) -> Result<(VmRecord, BootedVm, Arc>), AgentError> { + let vms = self.inner.vms.read().await; + let e = vms + .get(vm_id) + .ok_or_else(|| AgentError::new(ErrorCode::NotFound, format!("no vm {vm_id}")))?; + Ok((e.record.clone(), e.booted.clone(), e.lock.clone())) + } +} + +fn bind(record: &VmRecord, topic_id: &str, what: &str) -> Result<(), AgentError> { + if record.handle.topic_id == topic_id.trim() { + Ok(()) + } else { + Err(AgentError::new( + ErrorCode::TopicMismatch, + format!( + "vm {} is bound to topic {:?}, {what} names {:?}", + record.handle.vm_id, record.handle.topic_id, topic_id + ), + )) + } +} + +async fn require_bearer(State(state): State, req: Request, next: Next) -> Response { + let presented = req + .headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + match state.inner.auth.accepts(presented) { + Ok(()) => next.run(req).await, + Err(e) => AgentError::new(ErrorCode::Unauthorized, e.to_string()).into_response(), + } +} + +async fn health(State(state): State) -> Json { + let (ready, reason) = match state.inner.hypervisor.ready() { + Ok(()) => (true, String::new()), + Err(e) => (false, e.to_string()), + }; + Json(AgentHealth { + api_version: API_VERSION, + ready, + reason, + hypervisor: state.inner.hypervisor.name().to_owned(), + vms: state.inner.vms.read().await.len(), + }) +} + +async fn create_vm( + State(state): State, + Json(body): Json, +) -> Result<(StatusCode, Json), AgentError> { + let spec = body.spec; + spec.validate() + .map_err(|e| AgentError::new(ErrorCode::BadSpec, e.to_string()))?; + state.inner.hypervisor.ready()?; + // Serialise creates: the topic ↔ VM check and the insert must be one step. + let _create = state.inner.create_lock.lock().await; + if let Some(existing) = + state.inner.vms.read().await.values().find(|e| { + e.record.handle.topic_id == spec.topic_id && e.record.state == VmState::Running + }) + { + return Err(AgentError::new( + ErrorCode::AlreadyExists, + format!( + "topic {:?} already has vm {}", + spec.topic_id, existing.record.handle.vm_id + ), + )); + } + let vm_id = state.mint_vm_id(&spec.topic_id); + let booted = state.inner.hypervisor.boot(&vm_id, &spec).await?; + if booted.topic_id != spec.topic_id || booted.vm_id != vm_id { + return Err(AgentError::new( + ErrorCode::Backend, + "hypervisor booted a vm under another binding", + )); + } + let record = VmRecord { + handle: VmHandle { + topic_id: spec.topic_id.clone(), + vm_id: vm_id.clone(), + }, + image_digest: booted.image_digest.clone(), + vcpus: spec.template.vcpus, + mem_mib: spec.template.mem_mib, + sandbox: spec.sandbox.clone(), + retain: spec.retain, + state: VmState::Running, + }; + tracing::info!(topic_id = %spec.topic_id, %vm_id, image = %booted.image_digest, "topic vm booted"); + state.inner.vms.write().await.insert( + vm_id, + VmEntry { + record: record.clone(), + booted, + lock: Arc::new(Mutex::new(())), + }, + ); + Ok((StatusCode::CREATED, Json(record))) +} + +async fn attach( + State(state): State, + Path(topic_id): Path, +) -> Result, AgentError> { + state + .inner + .vms + .read() + .await + .values() + .find(|e| e.record.handle.topic_id == topic_id && e.record.state == VmState::Running) + .map(|e| Json(e.record.clone())) + .ok_or_else(|| { + AgentError::new( + ErrorCode::NotFound, + format!("no running vm for topic {topic_id:?}"), + ) + }) +} + +async fn run_job( + State(state): State, + Path(vm_id): Path, + Json(body): Json, +) -> Result, AgentError> { + let (record, booted, lock) = state.entry(&vm_id).await?; + if record.state != VmState::Running { + return Err(AgentError::new( + ErrorCode::NotFound, + format!("vm {vm_id} is {:?}, not running", record.state), + )); + } + bind(&record, &body.topic_id, "the request")?; + bind(&record, body.job.topic_id(), "the job")?; + let _guard = lock + .try_lock_owned() + .map_err(|_| AgentError::new(ErrorCode::Busy, format!("vm {vm_id} is running a job")))?; + let outcome = state.inner.hypervisor.run_job(&booted, &body.job).await?; + if !output_matches(&body.job, &outcome.output) { + return Err(AgentError::new( + ErrorCode::WrongOutput, + "guest answered with another output shape", + )); + } + if let Some(s) = &outcome.sister { + tracing::info!( + %vm_id, sister = %s.sister_vm_id, sandboxed = s.sandboxed, + flops_used = ?s.flops_used, wall_ms = s.wall_ms, "sister guest run attested" + ); + } + let output = stamp_output(outcome.output, outcome.sister.as_ref()); + Ok(Json(RunJobResponse { + topic_id: record.handle.topic_id, + vm_id, + output, + sister: outcome.sister, + })) +} + +async fn teardown( + State(state): State, + Path(vm_id): Path, + Json(body): Json, +) -> Result, AgentError> { + let (record, booted, lock) = state.entry(&vm_id).await?; + bind(&record, &body.topic_id, "the teardown")?; + if record.state != VmState::Running { + return Ok(Json(TeardownResponse { + topic_id: record.handle.topic_id, + vm_id, + state: record.state, + confirmed: true, + })); + } + let _guard = lock + .try_lock_owned() + .map_err(|_| AgentError::new(ErrorCode::Busy, format!("vm {vm_id} is running a job")))?; + let confirmed = state + .inner + .hypervisor + .teardown(&booted, body.policy) + .await?; + let end = match body.policy { + RetainPolicy::Destroy => VmState::Destroyed, + RetainPolicy::Retain => VmState::Retained, + }; + let state_now = if confirmed { + let mut vms = state.inner.vms.write().await; + match end { + VmState::Destroyed => { + vms.remove(&vm_id); + } + VmState::Retained | VmState::Running => { + if let Some(e) = vms.get_mut(&vm_id) { + e.record.state = end; + } + } + } + tracing::info!(%vm_id, topic_id = %record.handle.topic_id, ?end, "topic vm torn down"); + end + } else { + VmState::Running + }; + Ok(Json(TeardownResponse { + topic_id: record.handle.topic_id, + vm_id, + state: state_now, + confirmed, + })) +} + +/// The agent router. Every route, health included, needs the bearer. +pub fn agent_router(state: AgentState) -> Router { + Router::new() + .route(paths::HEALTH, get(health)) + .route(paths::VMS, post(create_vm)) + .route("/v1/vms/by-topic/{topic_id}", get(attach)) + .route("/v1/vms/{vm_id}/jobs", post(run_job)) + .route("/v1/vms/{vm_id}", axum::routing::delete(teardown)) + .layer(middleware::from_fn_with_state( + state.clone(), + require_bearer, + )) + .with_state(state) +} diff --git a/crates/proof-vm-agent/src/stamp.rs b/crates/proof-vm-agent/src/stamp.rs new file mode 100644 index 000000000..681f45871 --- /dev/null +++ b/crates/proof-vm-agent/src/stamp.rs @@ -0,0 +1,167 @@ +//! Host authority over paid outputs. +//! +//! The RLM guest authors the report, but two fields are **facts about the +//! host**, so the agent overwrites them from what it booted and observed: +//! +//! - `sandboxed` is `true` only when the host booted a sister Firecracker +//! guest for this job and the run happened inside it. An RLM that claims +//! `sandboxed: true` without a sister is corrected to `false`, and the +//! control plane then refuses the report for a `firecracker_required` +//! topic (`ReportError::NotSandboxed`). +//! - `flops_used` is the sister guest's measurement when a sister ran. A +//! sister that measured nothing yields `None`, which the control plane +//! refuses against a budget (`ReportError::FlopsMissing`, 503, no row) — +//! the RLM's own figure is never substituted for a run it did not perform. +//! +//! Inspection and rule proposals run no miner code and are passed through. + +use proof_rlm::{CustomRunReport, VmJob, VmJobOutput}; +use proof_vm_proto::SisterAttestation; + +/// Whether `output` is the shape `job` asks for. +#[must_use] +pub fn output_matches(job: &VmJob, output: &VmJobOutput) -> bool { + matches!( + (job, output), + (VmJob::ProposeRules { .. }, VmJobOutput::Rules(_)) + | (VmJob::Baseline { .. }, VmJobOutput::Baseline(_)) + | (VmJob::Inspect { .. }, VmJobOutput::Inspected(_)) + | (VmJob::Evaluate { .. }, VmJobOutput::Evaluated(_)) + | (VmJob::Archive { .. }, VmJobOutput::Archived) + ) +} + +fn stamp_report(report: &mut CustomRunReport, sister: Option<&SisterAttestation>) { + match sister { + Some(s) => { + report.sandboxed = s.sandboxed; + report.flops_used = s.flops_used; + } + None => report.sandboxed = false, + } +} + +/// Apply the host's view to a paid output. Non-paid outputs are unchanged. +#[must_use] +pub fn stamp_output(mut output: VmJobOutput, sister: Option<&SisterAttestation>) -> VmJobOutput { + match &mut output { + VmJobOutput::Baseline(report) => stamp_report(report, sister), + VmJobOutput::Evaluated(run) => stamp_report(&mut run.report, sister), + VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => {} + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_rlm::fixtures::{report_for, request, rules}; + use proof_rlm::RunOutcome; + + fn sister(sandboxed: bool, flops: Option) -> SisterAttestation { + SisterAttestation { + sister_vm_id: "topic-a-0001-s1".into(), + image_digest: format!("sha256:{}", "dd".repeat(32)), + sandboxed, + network: "none".into(), + flops_used: flops, + wall_ms: 10, + exit_code: Some(0), + } + } + + #[test] + fn the_rlm_cannot_claim_a_sandbox_the_host_did_not_boot() { + let req = request(); + let mut claimed = report_for(&req, 0.9); + claimed.sandboxed = true; + claimed.flops_used = Some(5); + let out = stamp_output( + VmJobOutput::Evaluated(RunOutcome { + report: claimed.clone(), + logs: vec![], + }), + None, + ); + let VmJobOutput::Evaluated(run) = out else { + panic!("shape"); + }; + assert!(!run.report.sandboxed, "no sister, no sandbox"); + assert_eq!( + run.report.flops_used, + Some(5), + "the RLM ran it itself; its own measurement stands" + ); + assert!( + run.report.verify(&req).is_err(), + "firecracker_required topic refuses the corrected report" + ); + } + + #[test] + fn a_sister_run_stamps_sandboxed_and_the_guest_measurement() { + let req = request(); + let mut lied = report_for(&req, 0.9); + lied.sandboxed = false; + lied.flops_used = Some(1); + let out = stamp_output(VmJobOutput::Baseline(lied), Some(&sister(true, Some(42)))); + let VmJobOutput::Baseline(report) = out else { + panic!("shape"); + }; + assert!(report.sandboxed); + assert_eq!(report.flops_used, Some(42), "host-relayed guest figure"); + let none = stamp_output( + VmJobOutput::Baseline(report_for(&req, 0.9)), + Some(&sister(true, None)), + ); + let VmJobOutput::Baseline(report) = none else { + panic!("shape"); + }; + assert_eq!( + report.flops_used, None, + "a sister that measured nothing is not the RLM's number" + ); + assert!(matches!( + report.verify(&req), + Err(proof_rlm::ReportError::FlopsMissing { .. }) + )); + } + + #[test] + fn non_paid_outputs_pass_through_and_shapes_are_checked() { + let rules = rules(); + let out = stamp_output( + VmJobOutput::Rules(rules.rules.clone()), + Some(&sister(true, Some(1))), + ); + assert_eq!(out, VmJobOutput::Rules(rules.rules.clone())); + let req = request(); + let inspect = VmJob::Inspect { + request: req.clone(), + rules: rules.clone(), + }; + assert!(output_matches( + &inspect, + &VmJobOutput::Inspected(proof_rlm::InspectOutcome { + checklist: proof_rlm::fixtures::green(&rules, &req.submission_digest), + artifact: vec![], + }) + )); + assert!(!output_matches(&inspect, &VmJobOutput::Archived)); + assert!(output_matches( + &VmJob::Archive { + topic_id: req.topic_id.clone() + }, + &VmJobOutput::Archived + )); + assert!(!output_matches( + &VmJob::Baseline { + request: req.clone() + }, + &VmJobOutput::Evaluated(RunOutcome { + report: report_for(&req, 0.1), + logs: vec![], + }) + )); + } +} diff --git a/crates/proof-vm-fc/Cargo.toml b/crates/proof-vm-fc/Cargo.toml new file mode 100644 index 000000000..d5c74f178 --- /dev/null +++ b/crates/proof-vm-fc/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "proof-vm-fc" +description = "FirecrackerOrchestrator: the live TopicVmOrchestrator. A thin HTTPS client (bearer from a file, never logged) of the proof-vm-orchestrator agent on a dedicated KVM host. Fail-closed: unset URL → unwired, missing token / unpinned RLM image digest / agent down → 503, never a host-local fallback." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +async-trait = "0.1" +proof-rlm = { path = "../proof-rlm" } +proof-vm-proto = { path = "../proof-vm-proto" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +tracing = "0.1" +url = "2" + +[dev-dependencies] +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +proof-vm-agent = { path = "../proof-vm-agent", features = ["test-fixtures"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "time"] } + +[lints] +workspace = true diff --git a/crates/proof-vm-fc/src/lib.rs b/crates/proof-vm-fc/src/lib.rs new file mode 100644 index 000000000..d22c3704f --- /dev/null +++ b/crates/proof-vm-fc/src/lib.rs @@ -0,0 +1,535 @@ +//! `FirecrackerOrchestrator` — the live [`TopicVmOrchestrator`]. +//! +//! The control plane never touches Firecracker. It talks HTTPS to the +//! `proof-vm-orchestrator` agent on a **dedicated KVM host**, which boots one +//! jailed Firecracker RLM VM per topic from the digest the control plane +//! pins (`PROOF_RLM_VM_IMAGE_DIGEST`) and runs every miner artefact in a +//! **sister** Firecracker guest. Jobs cross the wire as [`VmJob`]s — public +//! topic data, digests, rule versions — never a host path, a key, or an +//! origin. Owner key material is staged by the agent from the host's own +//! files; this crate only ever sends the bearer it reads from +//! `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`, and never logs it. +//! +//! Fail-closed, in this order: +//! +//! - `PROOF_VM_ORCHESTRATOR_URL` unset → [`FirecrackerOrchestrator::from_env`] +//! is `None` and the host keeps `UnwiredVmOrchestrator` (503). +//! - URL set but not `https://` (plain `http://` is accepted on loopback +//! only, for tests) → configuration error at boot, nothing wired. +//! - Token file missing / empty, RLM image digest unpinned → `ready()` is +//! [`VmError::NotWired`] naming the env var → 503 with the root cause. +//! - Agent unreachable, bearer refused, hypervisor not ready → +//! [`VmError::Backend`] → 503. There is no host-local execution path. +//! +//! Hard binds the client enforces on top of the agent's: a job must name +//! the handle's topic before any request leaves; the agent must echo the +//! same topic and VM; a created VM must report the digest that was pinned; +//! and a `firecracker_required` run must come back with the host's sister +//! attestation (`sandboxed: true`) or the output is not evidence. + +#![forbid(unsafe_code)] +#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use async_trait::async_trait; +use proof_rlm::{ + RetainPolicy, TopicVmOrchestrator, TopicVmSpec, VmError, VmHandle, VmJob, VmJobOutput, + VmTemplate, RLM_VM_IMAGE_DIGEST_ENV, VM_ORCHESTRATOR_TOKEN_FILE_ENV, VM_ORCHESTRATOR_URL_ENV, +}; +use proof_vm_proto::{ + paths, AgentHealth, CreateVmRequest, ErrorBody, RunJobRequest, RunJobResponse, TeardownRequest, + TeardownResponse, VmRecord, API_VERSION, +}; +use reqwest::{Method, StatusCode}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use url::Url; + +/// Optional extra PEM root the agent's certificate chains to (private CA). +pub const VM_ORCHESTRATOR_CA_FILE_ENV: &str = "PROOF_VM_ORCHESTRATOR_CA_FILE"; +/// RLM VM vCPUs (default [`DEFAULT_RLM_VCPUS`]). +pub const RLM_VM_VCPUS_ENV: &str = "PROOF_RLM_VM_VCPUS"; +/// RLM VM memory in MiB (default [`DEFAULT_RLM_MEM_MIB`]). +pub const RLM_VM_MEM_MIB_ENV: &str = "PROOF_RLM_VM_MEM_MIB"; +/// Comma-separated custom ids the generic `VmBackedRunner` serves over this +/// orchestrator. Registration is an operator action; unset = nothing registered. +pub const VM_RUNNER_CUSTOM_IDS_ENV: &str = "PROOF_VM_RUNNER_CUSTOM_IDS"; + +/// Locked RLM VM shape: 4 vCPU. +pub const DEFAULT_RLM_VCPUS: u32 = 4; +/// Locked RLM VM shape: 8192 MiB. +pub const DEFAULT_RLM_MEM_MIB: u32 = 8_192; +/// Wall-clock for a `create` (image verify + boot + guest hello + staging). +pub const DEFAULT_CREATE_TIMEOUT_S: u64 = 600; +/// Wall-clock for a job that carries no deadline (`ProposeRules`, `Archive`). +pub const DEFAULT_JOB_TIMEOUT_S: u64 = 3_600; +/// Added to a job's own deadline before the client gives up on the agent. +pub const JOB_GRACE_S: u64 = 60; + +/// Why the orchestrator could not be configured. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FcConfigError { + /// Not a URL. + #[error("{VM_ORCHESTRATOR_URL_ENV} is not a URL: {0}")] + BadUrl(String), + /// Plain `http://` to a non-loopback host. + #[error("{VM_ORCHESTRATOR_URL_ENV} must be https:// (plain http only on loopback): {0}")] + Insecure(String), + /// URL set, token file env missing. + #[error("{VM_ORCHESTRATOR_TOKEN_FILE_ENV} is not set (the bearer is a file, never a value)")] + NoTokenFile, + /// A sizing knob did not parse or is out of range. + #[error("{0} is not a number in range: {1:?}")] + BadNumber(&'static str, String), + /// The CA file is unreadable or not PEM. + #[error("{VM_ORCHESTRATOR_CA_FILE_ENV}: {0}")] + BadCa(String), + /// The HTTP client could not be built. + #[error("http client: {0}")] + Client(String), +} + +/// Operator configuration. Holds paths, never secrets. +#[derive(Debug, Clone)] +pub struct FcConfig { + /// Agent base URL (`https://kvm-host:8200`). + pub url: Url, + /// Bearer file, re-read per request. + pub token_file: PathBuf, + /// RLM VM image pin + size. + pub template: VmTemplate, + /// Extra PEM root, if the agent uses a private CA. + pub ca_file: Option, + /// TCP + TLS connect budget. + pub connect_timeout: Duration, + /// `create` budget. + pub create_timeout: Duration, + /// Budget for jobs with no deadline of their own. + pub default_job_timeout: Duration, +} + +fn env_trimmed(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +fn env_u32(name: &'static str, default: u32) -> Result { + match env_trimmed(name) { + None => Ok(default), + Some(raw) => raw + .parse::() + .map_err(|_| FcConfigError::BadNumber(name, raw)), + } +} + +fn is_loopback(url: &Url) -> bool { + matches!( + url.host(), + Some( + url::Host::Domain("localhost") + | url::Host::Ipv4(std::net::Ipv4Addr::LOCALHOST) + | url::Host::Ipv6(std::net::Ipv6Addr::LOCALHOST) + ) + ) +} + +impl FcConfig { + /// Config for `url` + `token_file` with the locked RLM shape. + #[must_use] + pub fn new(url: Url, token_file: &Path, image_digest: &str) -> Self { + Self { + url, + token_file: token_file.to_path_buf(), + template: VmTemplate { + image_digest: image_digest.trim().to_owned(), + vcpus: DEFAULT_RLM_VCPUS, + mem_mib: DEFAULT_RLM_MEM_MIB, + }, + ca_file: None, + connect_timeout: Duration::from_secs(10), + create_timeout: Duration::from_secs(DEFAULT_CREATE_TIMEOUT_S), + default_job_timeout: Duration::from_secs(DEFAULT_JOB_TIMEOUT_S), + } + } + + /// Read the operator env. `Ok(None)` when the URL is unset (unwired). + pub fn from_env() -> Result, FcConfigError> { + let Some(raw) = env_trimmed(VM_ORCHESTRATOR_URL_ENV) else { + return Ok(None); + }; + let url = Url::parse(&raw).map_err(|e| FcConfigError::BadUrl(format!("{raw}: {e}")))?; + let token_file = + env_trimmed(VM_ORCHESTRATOR_TOKEN_FILE_ENV).ok_or(FcConfigError::NoTokenFile)?; + let digest = env_trimmed(RLM_VM_IMAGE_DIGEST_ENV).unwrap_or_default(); + let mut cfg = Self::new(url, Path::new(&token_file), &digest); + cfg.template.vcpus = env_u32(RLM_VM_VCPUS_ENV, DEFAULT_RLM_VCPUS)?; + cfg.template.mem_mib = env_u32(RLM_VM_MEM_MIB_ENV, DEFAULT_RLM_MEM_MIB)?; + cfg.ca_file = env_trimmed(VM_ORCHESTRATOR_CA_FILE_ENV).map(PathBuf::from); + cfg.validate()?; + Ok(Some(cfg)) + } + + /// `https://`, or `http://` on loopback only. Sizes are checked by the + /// template at `ready()` so an unpinned digest is a 503, not a boot error. + pub fn validate(&self) -> Result<(), FcConfigError> { + match self.url.scheme() { + "https" => Ok(()), + "http" if is_loopback(&self.url) => Ok(()), + _ => Err(FcConfigError::Insecure(self.url.to_string())), + } + } +} + +/// Live orchestrator client. +pub struct FirecrackerOrchestrator { + config: FcConfig, + http: reqwest::Client, +} + +impl std::fmt::Debug for FirecrackerOrchestrator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FirecrackerOrchestrator") + .field("url", &self.config.url.as_str()) + .field("token_file", &self.config.token_file) + .field("template", &self.config.template) + .finish_non_exhaustive() + } +} + +fn backend(msg: impl Into) -> VmError { + VmError::Backend(msg.into()) +} + +impl FirecrackerOrchestrator { + /// Build the client. Reads the CA file (if any) now; the token per request. + pub fn new(config: FcConfig) -> Result { + config.validate()?; + let mut builder = reqwest::Client::builder() + .connect_timeout(config.connect_timeout) + .user_agent(format!("proof-vm-fc/{API_VERSION}")); + if let Some(ca) = &config.ca_file { + let pem = std::fs::read(ca) + .map_err(|e| FcConfigError::BadCa(format!("{}: {e}", ca.display())))?; + let cert = reqwest::Certificate::from_pem(&pem) + .map_err(|e| FcConfigError::BadCa(format!("{}: {e}", ca.display())))?; + builder = builder.add_root_certificate(cert); + } + let http = builder + .build() + .map_err(|e| FcConfigError::Client(e.to_string()))?; + Ok(Self { config, http }) + } + + /// From the operator env. `Ok(None)` = unwired (keep `UnwiredVmOrchestrator`). + pub fn from_env() -> Result, FcConfigError> { + FcConfig::from_env()?.map(Self::new).transpose() + } + + /// The RLM VM template this client asks the agent to boot. + #[must_use] + pub fn template(&self) -> &VmTemplate { + &self.config.template + } + + /// Agent base URL. + #[must_use] + pub fn url(&self) -> &Url { + &self.config.url + } + + /// The bearer, read fresh. Never logged. + fn token(&self) -> Result { + std::fs::read_to_string(&self.config.token_file) + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + VmError::NotWired(format!( + "{VM_ORCHESTRATOR_TOKEN_FILE_ENV} ({}) missing or empty", + self.config.token_file.display() + )) + }) + } + + fn endpoint(&self, path: &str) -> Result { + self.config + .url + .join(path.trim_start_matches('/')) + .map_err(|e| backend(format!("orchestrator url: {e}"))) + } + + /// One authenticated call. `Ok(None)` for 404 (callers decide if that is + /// an answer); every other non-2xx is a [`VmError::Backend`] carrying the + /// agent's error code — never the bearer. + async fn call( + &self, + method: Method, + path: &str, + body: Option<&B>, + timeout: Duration, + ) -> Result, VmError> { + let token = self.token()?; + let mut req = self + .http + .request(method.clone(), self.endpoint(path)?) + .bearer_auth(token) + .timeout(timeout); + if let Some(b) = body { + req = req.json(b); + } + let resp = req + .send() + .await + .map_err(|e| backend(format!("orchestrator unreachable ({method} {path}): {e}")))?; + let status = resp.status(); + if status == StatusCode::NOT_FOUND { + return Ok(None); + } + if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN { + return Err(backend(format!( + "orchestrator refused the bearer ({status}); rotate {VM_ORCHESTRATOR_TOKEN_FILE_ENV} on both sides" + ))); + } + let bytes = resp + .bytes() + .await + .map_err(|e| backend(format!("orchestrator body ({method} {path}): {e}")))?; + if !status.is_success() { + let detail = serde_json::from_slice::(&bytes).map_or_else( + |_| { + let text = String::from_utf8_lossy(&bytes); + text.chars().take(240).collect::() + }, + |e| format!("{:?}: {}", e.code, e.error), + ); + return Err(backend(format!( + "orchestrator {status} on {method} {path}: {detail}" + ))); + } + serde_json::from_slice::(&bytes).map(Some).map_err(|e| { + backend(format!( + "orchestrator answer ({method} {path}) did not parse: {e}" + )) + }) + } + + /// `GET /v1/health`. + pub async fn health(&self) -> Result { + self.call::<(), AgentHealth>( + Method::GET, + paths::HEALTH, + None, + self.config.connect_timeout, + ) + .await? + .ok_or_else(|| backend("orchestrator has no health route")) + } + + fn job_timeout(&self, job: &VmJob) -> Duration { + job.deadline_s() + .map_or(self.config.default_job_timeout, |d| { + Duration::from_secs(d.saturating_add(JOB_GRACE_S)) + }) + } +} + +fn check_echo(handle: &VmHandle, topic_id: &str, vm_id: &str) -> Result<(), VmError> { + if handle.topic_id == topic_id && handle.vm_id == vm_id { + Ok(()) + } else { + Err(backend(format!( + "orchestrator answered for {topic_id}/{vm_id}, asked about {}/{}", + handle.topic_id, handle.vm_id + ))) + } +} + +#[async_trait] +impl TopicVmOrchestrator for FirecrackerOrchestrator { + fn ready(&self) -> Result<(), VmError> { + self.token()?; + self.config + .template + .validate() + .map_err(|e| VmError::NotWired(format!("{RLM_VM_IMAGE_DIGEST_ENV}: {e}"))) + } + + async fn create(&self, spec: &TopicVmSpec) -> Result { + self.ready()?; + spec.validate()?; + let record: VmRecord = self + .call( + Method::POST, + paths::VMS, + Some(&CreateVmRequest { spec: spec.clone() }), + self.config.create_timeout, + ) + .await? + .ok_or_else(|| backend("orchestrator has no create route"))?; + if record.handle.topic_id != spec.topic_id { + return Err(backend(format!( + "orchestrator bound the vm to {:?}, asked for {:?}", + record.handle.topic_id, spec.topic_id + ))); + } + if !record + .image_digest + .eq_ignore_ascii_case(&spec.template.image_digest) + { + return Err(backend(format!( + "orchestrator booted image {} instead of the pinned {}", + record.image_digest, spec.template.image_digest + ))); + } + tracing::info!(topic_id = %spec.topic_id, vm_id = %record.handle.vm_id, "topic vm created"); + Ok(record.handle) + } + + async fn attach(&self, topic_id: &str) -> Result, VmError> { + self.ready()?; + let found: Option = self + .call::<(), VmRecord>( + Method::GET, + &paths::vm_by_topic(topic_id.trim()), + None, + self.config.connect_timeout, + ) + .await?; + match found { + Some(r) if r.handle.topic_id == topic_id.trim() => Ok(Some(r.handle)), + Some(r) => Err(backend(format!( + "orchestrator returned vm {} of topic {:?} for topic {topic_id:?}", + r.handle.vm_id, r.handle.topic_id + ))), + None => Ok(None), + } + } + + async fn run(&self, handle: &VmHandle, job: VmJob) -> Result { + self.ready()?; + if job.topic_id() != handle.topic_id { + return Err(VmError::Spec("topic_id")); + } + let needs_sister = job.requires_firecracker(); + let timeout = self.job_timeout(&job); + let resp: RunJobResponse = self + .call( + Method::POST, + &paths::vm_jobs(&handle.vm_id), + Some(&RunJobRequest { + topic_id: handle.topic_id.clone(), + job, + }), + timeout, + ) + .await? + .ok_or_else(|| backend(format!("orchestrator knows no vm {}", handle.vm_id)))?; + check_echo(handle, &resp.topic_id, &resp.vm_id)?; + let attested = resp.sister.as_ref().is_some_and(|s| s.sandboxed); + if needs_sister && !attested { + return Err(backend( + "firecracker_required run came back without the host's sister-guest attestation", + )); + } + let claims_sandbox = match &resp.output { + VmJobOutput::Baseline(r) => r.sandboxed, + VmJobOutput::Evaluated(run) => run.report.sandboxed, + VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => false, + }; + if claims_sandbox && !attested { + return Err(backend( + "report claims a sandbox the orchestrator did not attest", + )); + } + Ok(resp.output) + } + + async fn teardown(&self, handle: &VmHandle, policy: RetainPolicy) -> Result { + self.ready()?; + let resp: TeardownResponse = self + .call( + Method::DELETE, + &paths::vm(&handle.vm_id), + Some(&TeardownRequest { + topic_id: handle.topic_id.clone(), + policy, + }), + self.config.create_timeout, + ) + .await? + .ok_or_else(|| backend(format!("orchestrator knows no vm {}", handle.vm_id)))?; + check_echo(handle, &resp.topic_id, &resp.vm_id)?; + tracing::info!( + topic_id = %handle.topic_id, vm_id = %handle.vm_id, ?policy, + state = ?resp.state, confirmed = resp.confirmed, "topic vm teardown" + ); + Ok(resp.confirmed) + } +} + +/// Parse [`VM_RUNNER_CUSTOM_IDS_ENV`]-style lists: comma-separated, trimmed, +/// empty entries dropped, order kept, duplicates removed. +#[must_use] +pub fn parse_custom_ids(raw: &str) -> Vec { + let mut out: Vec = Vec::new(); + for id in raw.split(',').map(str::trim).filter(|s| !s.is_empty()) { + if !out.iter().any(|o| o == id) { + out.push(id.to_owned()); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scheme_rule_is_https_or_loopback_http() { + let tok = Path::new("/nonexistent/token"); + let ok = |u: &str| FcConfig::new(Url::parse(u).expect("url"), tok, "").validate(); + ok("https://kvm.example.invalid:8200").expect("https anywhere"); + ok("http://127.0.0.1:8200").expect("loopback http"); + ok("http://localhost:8200").expect("localhost http"); + ok("http://[::1]:8200").expect("v6 loopback http"); + assert!(matches!( + ok("http://10.0.0.5:8200"), + Err(FcConfigError::Insecure(_)) + )); + assert!(matches!( + ok("http://kvm.example.invalid:8200"), + Err(FcConfigError::Insecure(_)) + )); + assert!(FirecrackerOrchestrator::new(FcConfig::new( + Url::parse("http://10.0.0.5:8200").expect("url"), + tok, + "" + )) + .is_err()); + } + + #[test] + fn defaults_are_the_locked_rlm_shape_and_ids_parse() { + let cfg = FcConfig::new( + Url::parse("https://kvm.example.invalid").expect("url"), + Path::new("/x"), + " sha256:abc ", + ); + assert_eq!(cfg.template.vcpus, 4); + assert_eq!(cfg.template.mem_mib, 8_192); + assert_eq!(cfg.template.image_digest, "sha256:abc"); + assert_eq!( + parse_custom_ids(" a_metric, b-metric ,,a_metric, "), + vec!["a_metric".to_owned(), "b-metric".to_owned()] + ); + assert!(parse_custom_ids("").is_empty()); + assert_eq!(VM_RUNNER_CUSTOM_IDS_ENV, "PROOF_VM_RUNNER_CUSTOM_IDS"); + assert_eq!(VM_ORCHESTRATOR_CA_FILE_ENV, "PROOF_VM_ORCHESTRATOR_CA_FILE"); + assert_eq!(RLM_VM_VCPUS_ENV, "PROOF_RLM_VM_VCPUS"); + assert_eq!(RLM_VM_MEM_MIB_ENV, "PROOF_RLM_VM_MEM_MIB"); + } +} diff --git a/crates/proof-vm-fc/tests/live_agent.rs b/crates/proof-vm-fc/tests/live_agent.rs new file mode 100644 index 000000000..bd2c8d2d1 --- /dev/null +++ b/crates/proof-vm-fc/tests/live_agent.rs @@ -0,0 +1,376 @@ +//! `FirecrackerOrchestrator` against an in-process `proof-vm-orchestrator` +//! agent over a **fake** hypervisor. No Firecracker, no VM, no network beyond +//! loopback — exactly what CI runs. + +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use proof_rlm::fixtures::{pinned_template, request, rules, token_for, FakeOrchestrator}; +use proof_rlm::{ + CustomRunner, RetainPolicy, RunnerError, TopicVmOrchestrator, TopicVmSpec, VmBackedRunner, + VmError, VmHandle, VmJob, VmJobOutput, VmTemplate, RLM_VM_IMAGE_DIGEST_ENV, + VM_ORCHESTRATOR_TOKEN_FILE_ENV, VM_ORCHESTRATOR_URL_ENV, +}; +use proof_vm_agent::fixtures::{token_file, FakeAgent, FakeHypervisor}; +use proof_vm_fc::{ + FcConfig, FcConfigError, FirecrackerOrchestrator, DEFAULT_RLM_MEM_MIB, DEFAULT_RLM_VCPUS, + RLM_VM_MEM_MIB_ENV, RLM_VM_VCPUS_ENV, +}; +use url::Url; + +const TOKEN: &str = "fc-client-test-token-not-a-real-secret"; + +static ENV: Mutex<()> = Mutex::new(()); + +fn client(agent: &FakeAgent, token: &Path, digest: &str) -> FirecrackerOrchestrator { + let mut cfg = FcConfig::new(Url::parse(&agent.url()).expect("url"), token, digest); + cfg.template.vcpus = pinned_template().vcpus; + cfg.template.mem_mib = pinned_template().mem_mib; + FirecrackerOrchestrator::new(cfg).expect("client") +} + +async fn live(tag: &str) -> (FakeAgent, PathBuf) { + let token = token_file(tag, TOKEN); + let agent = FakeAgent::serve(FakeHypervisor::new(0.8), &token).await; + (agent, token) +} + +fn spec(template: VmTemplate) -> TopicVmSpec { + let req = request(); + TopicVmSpec::for_topic(&req.topic_id, template, req.sandbox) +} + +#[tokio::test] +async fn one_topic_one_vm_inspect_then_paid_run_with_sister_attestation() { + let (agent, token) = live("flow").await; + let orch = Arc::new(client(&agent, &token, &pinned_template().image_digest)); + orch.ready().expect("token + pin present"); + let health = orch.health().await.expect("health"); + assert!(health.ready); + assert_eq!(health.hypervisor, "fake"); + + let runner = VmBackedRunner::new(orch.clone(), pinned_template()); + runner.ready().expect("runner ready"); + let req = request(); + let inspected = runner.inspect(&req, &rules()).await.expect("inspect"); + assert!(inspected.checklist.is_green(&rules())); + let run = runner + .evaluate(&req, &token_for(&req)) + .await + .expect("evaluate"); + assert!((run.report.primary_value - 0.8).abs() < 1e-12); + assert!(run.report.sandboxed, "host attested the sister guest"); + assert_eq!( + run.report.flops_used, + Some(1), + "the sister's measurement, not the RLM's" + ); + let hv = &agent.hypervisor; + assert_eq!(hv.boots().len(), 1, "second job attached, no second boot"); + assert_eq!(hv.boots()[0].topic_id, req.topic_id); + assert_eq!(hv.boots()[0].image_digest, pinned_template().image_digest); + let jobs = hv.jobs(); + assert_eq!(jobs.len(), 2); + assert!(matches!(jobs[0].1, VmJob::Inspect { .. })); + assert!(matches!(jobs[1].1, VmJob::Evaluate { .. })); + for (_, job) in &jobs { + let dump = serde_json::to_string(job).expect("json"); + for forbidden in [ + "/run/base", + "/opt/base", + "api_key", + "127.0.0.1", + "base_url", + TOKEN, + ] { + assert!(!dump.contains(forbidden), "job leaked {forbidden}: {dump}"); + } + } + + let handle = orch + .attach(&req.topic_id) + .await + .expect("attach") + .expect("exists"); + assert_eq!(handle.topic_id, req.topic_id); + assert!(orch + .teardown(&handle, RetainPolicy::Destroy) + .await + .expect("teardown")); + assert_eq!( + hv.teardowns(), + vec![(handle.vm_id.clone(), RetainPolicy::Destroy)] + ); + assert_eq!(orch.attach(&req.topic_id).await.expect("attach"), None); + assert_eq!( + orch.teardown(&handle, RetainPolicy::Destroy).await, + Err(VmError::Backend(format!( + "orchestrator knows no vm {}", + handle.vm_id + ))), + "a destroyed vm is gone" + ); +} + +#[tokio::test] +async fn missing_token_or_unpinned_digest_is_not_wired_and_never_calls_out() { + let (agent, token) = live("unwired").await; + let no_token = client( + &agent, + Path::new("/nonexistent/vm_orchestrator_token"), + &pinned_template().image_digest, + ); + let err = no_token.ready().expect_err("no token file"); + assert!(matches!(err, VmError::NotWired(_)), "{err}"); + assert!( + err.to_string().contains(VM_ORCHESTRATOR_TOKEN_FILE_ENV), + "{err}" + ); + let runner = VmBackedRunner::new(Arc::new(no_token), pinned_template()); + assert!(matches!( + runner.inspect(&request(), &rules()).await, + Err(RunnerError::NotWired(_)) + )); + + let unpinned = client(&agent, &token, ""); + let err = unpinned.ready().expect_err("unpinned"); + assert!(matches!(err, VmError::NotWired(_)), "{err}"); + assert!(err.to_string().contains(RLM_VM_IMAGE_DIGEST_ENV), "{err}"); + let err = unpinned + .create(&spec(VmTemplate::unpinned())) + .await + .expect_err("create refuses before any request"); + assert!(matches!(err, VmError::NotWired(_)), "{err}"); + assert!( + agent.hypervisor.boots().is_empty(), + "nothing reached the agent" + ); + assert!(agent.hypervisor.jobs().is_empty()); +} + +#[tokio::test] +async fn a_wrong_bearer_or_a_dead_agent_is_a_backend_refusal_without_the_token() { + let (agent, _) = live("bearer").await; + let wrong = token_file("bearer-wrong", "another-token-not-a-real-secret"); + let orch = client(&agent, &wrong, &pinned_template().image_digest); + orch.ready().expect("configured"); + let err = orch + .create(&spec(pinned_template())) + .await + .expect_err("refused"); + let text = err.to_string(); + assert!(matches!(err, VmError::Backend(_)), "{text}"); + assert!(text.contains("bearer"), "{text}"); + assert!(!text.contains("another-token"), "token leaked: {text}"); + assert!(!text.contains(TOKEN), "token leaked: {text}"); + let runner = VmBackedRunner::new(Arc::new(orch), pinned_template()); + assert!(matches!( + runner.inspect(&request(), &rules()).await, + Err(RunnerError::Backend(_)) + )); + + let (dead, token) = live("dead").await; + let orch = client(&dead, &token, &pinned_template().image_digest); + dead.stop(); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let err = orch + .attach("topic-a") + .await + .expect_err("agent down is not None"); + assert!(matches!(err, VmError::Backend(_)), "{err}"); + assert!(err.to_string().contains("unreachable"), "{err}"); +} + +#[tokio::test] +async fn jobs_are_bound_to_the_handle_topic_before_any_request() { + let (agent, token) = live("bind").await; + let orch = client(&agent, &token, &pinned_template().image_digest); + let handle = orch.create(&spec(pinned_template())).await.expect("create"); + let err = orch + .run( + &handle, + VmJob::Archive { + topic_id: "topic-b".into(), + }, + ) + .await + .expect_err("other topic"); + assert_eq!(err, VmError::Spec("topic_id")); + assert!(agent.hypervisor.jobs().is_empty(), "never left the client"); + let forged = VmHandle { + topic_id: "topic-b".into(), + vm_id: handle.vm_id.clone(), + }; + let err = orch + .run( + &forged, + VmJob::Archive { + topic_id: "topic-b".into(), + }, + ) + .await + .expect_err("agent refuses the mismatch"); + assert!(matches!(err, VmError::Backend(_)), "{err}"); + assert!(err.to_string().contains("TopicMismatch"), "{err}"); + let err = orch + .teardown(&forged, RetainPolicy::Retain) + .await + .expect_err("teardown bound too"); + assert!(err.to_string().contains("TopicMismatch"), "{err}"); + assert!(agent.hypervisor.teardowns().is_empty()); + let out = orch + .run( + &handle, + VmJob::Archive { + topic_id: handle.topic_id.clone(), + }, + ) + .await + .expect("bound job runs"); + assert_eq!(out, VmJobOutput::Archived); +} + +#[tokio::test] +async fn a_firecracker_required_run_without_the_sister_attestation_is_not_evidence() { + let (agent, token) = live("sister").await; + agent.hypervisor.set_sister(false); + agent.hypervisor.set_rlm_claims_sandboxed(true); + let orch = Arc::new(client(&agent, &token, &pinned_template().image_digest)); + let runner = VmBackedRunner::new(orch.clone(), pinned_template()); + let req = request(); + assert!(req.sandbox.firecracker_required); + let err = runner + .evaluate(&req, &token_for(&req)) + .await + .expect_err("no sister"); + assert!(matches!(err, RunnerError::Backend(_)), "{err}"); + assert!(err.to_string().contains("sister"), "{err}"); + + let mut relaxed = req.clone(); + relaxed.sandbox.firecracker_required = false; + let handle = orch + .attach(&req.topic_id) + .await + .expect("attach") + .expect("vm"); + let out = orch + .run( + &handle, + VmJob::Baseline { + request: relaxed.clone(), + }, + ) + .await + .expect("a topic that does not require the guest may run in the RLM VM"); + let VmJobOutput::Baseline(report) = out else { + panic!("shape"); + }; + assert!(!report.sandboxed, "the host overruled the RLM's claim"); + report + .verify(&relaxed) + .expect("not required, so still evidence"); + assert!(report.verify(&req).is_err()); +} + +#[tokio::test] +async fn the_created_vm_must_run_the_pinned_image_and_a_fake_answer_is_refused() { + let (agent, token) = live("pin").await; + let mut other = pinned_template(); + other.image_digest = format!("sha256:{}", "ee".repeat(32)); + let orch = client(&agent, &token, &other.image_digest); + let handle = orch.create(&spec(other.clone())).await.expect("create"); + assert_eq!(agent.hypervisor.boots()[0].image_digest, other.image_digest); + // A second create for the same topic is the agent's one-VM-per-topic rule. + let err = orch.create(&spec(other)).await.expect_err("duplicate"); + assert!(err.to_string().contains("AlreadyExists"), "{err}"); + assert!(orch + .teardown(&handle, RetainPolicy::Retain) + .await + .expect("retain")); + assert_eq!( + orch.attach(&handle.topic_id).await.expect("attach"), + None, + "a retained vm is not attachable; the next run creates a fresh one" + ); + // The reference fake orchestrator and the live client agree on the contract. + let reference = FakeOrchestrator::new(0.8); + reference.ready().expect("reference"); +} + +#[tokio::test] +async fn from_env_is_none_when_unset_and_reads_the_locked_shape() { + let _guard = ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for name in [ + VM_ORCHESTRATOR_URL_ENV, + VM_ORCHESTRATOR_TOKEN_FILE_ENV, + RLM_VM_IMAGE_DIGEST_ENV, + RLM_VM_VCPUS_ENV, + RLM_VM_MEM_MIB_ENV, + ] { + std::env::remove_var(name); + } + assert!(FirecrackerOrchestrator::from_env() + .expect("unset is fine") + .is_none()); + + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "https://kvm.example.invalid:8200"); + assert_eq!( + FcConfig::from_env().expect_err("token file env required"), + FcConfigError::NoTokenFile + ); + std::env::set_var( + VM_ORCHESTRATOR_TOKEN_FILE_ENV, + "/run/base/proof/vm_orchestrator_token", + ); + let cfg = FcConfig::from_env().expect("config").expect("some"); + assert_eq!(cfg.template.vcpus, DEFAULT_RLM_VCPUS); + assert_eq!(cfg.template.mem_mib, DEFAULT_RLM_MEM_MIB); + assert!( + cfg.template.image_digest.is_empty(), + "unpinned until the operator sets it" + ); + let orch = FirecrackerOrchestrator::from_env() + .expect("builds") + .expect("some"); + let err = orch.ready().expect_err("no token file on this box, no pin"); + assert!(matches!(err, VmError::NotWired(_)), "{err}"); + assert!(!format!("{orch:?}").contains("Bearer")); + + std::env::set_var( + RLM_VM_IMAGE_DIGEST_ENV, + format!("sha256:{}", "ab".repeat(32)), + ); + std::env::set_var(RLM_VM_VCPUS_ENV, "8"); + std::env::set_var(RLM_VM_MEM_MIB_ENV, "16384"); + let cfg = FcConfig::from_env().expect("config").expect("some"); + assert_eq!((cfg.template.vcpus, cfg.template.mem_mib), (8, 16_384)); + cfg.template.validate().expect("pinned"); + std::env::set_var(RLM_VM_VCPUS_ENV, "many"); + assert!(matches!( + FcConfig::from_env(), + Err(FcConfigError::BadNumber(RLM_VM_VCPUS_ENV, _)) + )); + std::env::remove_var(RLM_VM_VCPUS_ENV); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "http://10.0.0.9:8200"); + assert!(matches!( + FcConfig::from_env(), + Err(FcConfigError::Insecure(_)) + )); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "not a url"); + assert!(matches!( + FcConfig::from_env(), + Err(FcConfigError::BadUrl(_)) + )); + for name in [ + VM_ORCHESTRATOR_URL_ENV, + VM_ORCHESTRATOR_TOKEN_FILE_ENV, + RLM_VM_IMAGE_DIGEST_ENV, + RLM_VM_MEM_MIB_ENV, + ] { + std::env::remove_var(name); + } +} diff --git a/crates/proof-vm-proto/Cargo.toml b/crates/proof-vm-proto/Cargo.toml new file mode 100644 index 000000000..8ee110e93 --- /dev/null +++ b/crates/proof-vm-proto/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "proof-vm-proto" +description = "Wire protocol between the Proof control plane (FirecrackerOrchestrator client) and the proof-vm-orchestrator agent on the KVM host, plus the vsock framing the agent speaks to the RLM and miner guests. Types only; no challenge content." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +base64 = "0.22" +proof-rlm = { path = "../proof-rlm" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +tokio = { version = "1", features = ["io-util"] } + +[dev-dependencies] +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +tokio = { version = "1", features = ["io-util", "macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/proof-vm-proto/src/guest.rs b/crates/proof-vm-proto/src/guest.rs new file mode 100644 index 000000000..b6fe890bb --- /dev/null +++ b/crates/proof-vm-proto/src/guest.rs @@ -0,0 +1,371 @@ +//! Agent ↔ guest protocol over Firecracker vsock. +//! +//! Every message is one **frame**: a 4-byte big-endian length followed by +//! that many bytes of JSON ([`write_frame`] / [`read_frame`]). Three +//! channels exist, each on its own vsock port: +//! +//! | Port | Direction | Purpose | +//! |------|-----------|---------| +//! | [`RLM_JOB_PORT`] | host → RLM guest | [`HostToRlm`] / [`RlmToHost`]: hello, secret staging, jobs | +//! | [`SISTER_PORT`] | RLM guest → host | [`SisterRequest`] / [`SisterResult`]: "run this artefact in a sister guest" | +//! | [`MINER_PORT`] | host → miner guest | [`HostToMiner`] / [`MinerToHost`]: the run itself | +//! +//! The RLM guest never talks to the miner guest; the host relays the +//! artefact bytes it already inspected and relays the result back. The miner +//! guest has no network interface at all. Guest agents live in the pinned +//! images (`PROOF_RLM_VM_IMAGE_DIGEST`, the agent's miner image), not in this +//! repository; this module is the contract they implement. + +use std::collections::BTreeMap; + +use base64::Engine; +use proof_rlm::{VmJob, VmJobOutput}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +use crate::{ProtoError, API_VERSION}; + +/// vsock port the RLM guest agent listens on for host jobs. +pub const RLM_JOB_PORT: u32 = 5000; +/// Host-side vsock port the RLM guest connects to for a sister run. +pub const SISTER_PORT: u32 = 5001; +/// vsock port the miner (sister) guest agent listens on. +pub const MINER_PORT: u32 = 5002; +/// Firecracker's guest CID for every guest (the host is always CID 2). +pub const GUEST_CID: u32 = 3; +/// Largest frame either side accepts (artefact tarballs travel inside one). +pub const MAX_FRAME_BYTES: u32 = 256 * 1024 * 1024; + +/// One file staged into a guest (owner key material, artefact members, outputs). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StagedFile { + /// Single path segment or relative path inside the guest's staging dir. + pub name: String, + /// Standard base64 of the bytes. + pub bytes_b64: String, +} + +impl StagedFile { + /// Encode `bytes` under `name`. + #[must_use] + pub fn new(name: &str, bytes: &[u8]) -> Self { + Self { + name: name.to_owned(), + bytes_b64: base64::engine::general_purpose::STANDARD.encode(bytes), + } + } + + /// Decode the bytes. + /// + /// # Errors + /// + /// [`ProtoError::Decode`] on bad base64. + pub fn bytes(&self) -> Result, ProtoError> { + base64::engine::general_purpose::STANDARD + .decode(&self.bytes_b64) + .map_err(|e| ProtoError::Decode(format!("{}: {e}", self.name))) + } +} + +/// Host → RLM guest. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostToRlm { + /// First message after boot: version + the VM's binding. + Hello { + /// [`API_VERSION`]. + api_version: u32, + /// Topic this VM is bound to. + topic_id: String, + /// Host VM id. + vm_id: String, + }, + /// Owner key material the control plane only ever probed for presence. + /// The guest keeps it in memory / tmpfs; it never appears in any output. + StageSecrets { + /// Files by name. + files: Vec, + }, + /// Run one job. Answered by [`RlmToHost::Done`] or [`RlmToHost::Failed`]. + Run { + /// The work (public data only). + job: Box, + }, +} + +/// RLM guest → host (answers on [`RLM_JOB_PORT`]). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RlmToHost { + /// Answer to `Hello`. + Ready { + /// Guest agent name/version (informational). + agent: String, + /// [`API_VERSION`] the guest speaks. + api_version: u32, + }, + /// Answer to `StageSecrets`. + Staged { + /// Files accepted. + count: usize, + }, + /// Job finished. + Done { + /// The document. Paid outputs are re-stamped by the host. + output: VmJobOutput, + }, + /// Job failed inside the guest. + Failed { + /// Why (never a secret). + error: String, + }, +} + +/// RLM guest → host on [`SISTER_PORT`]: run a miner artefact in a sister +/// Firecracker guest. The RLM already fetched and inspected the tree; it +/// ships the bytes so the sister needs no network. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SisterRequest { + /// Must equal the RLM VM's bound topic. + pub topic_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// sha256 hex of `artifact_tar`; the host re-hashes before boot. + pub artifact_digest: String, + /// The artefact tree as a tarball (`StagedFile` named `artifact.tar`). + pub artifact_tar: StagedFile, + /// Command run inside the sister (relative to the unpacked tree). + pub entrypoint: Vec, + /// Wall-clock the run is held to. + pub deadline_s: u64, + /// Cap the guest enforces on measured FLOPs (the miner's declaration). + pub declared_flops: u64, + /// Seed every run uses. + pub seed: u64, + /// Opaque topic params (`constraints.params`), exported to the run. + pub params: BTreeMap, +} + +/// Host → RLM guest: what happened in the sister. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SisterResult { + /// Host id of the sister VM. + pub sister_vm_id: String, + /// `sha256:` digest of the miner-guest image the host booted. + pub image_digest: String, + /// Exit code (`None` = killed at the deadline). + pub exit_code: Option, + /// The deadline cut the run. + pub timed_out: bool, + /// Last bytes of stdout/stderr (bounded). + pub stdout_tail: String, + /// FLOPs the guest measured (`None` = the guest measured nothing). + pub flops_used: Option, + /// Wall-clock of the run. + pub wall_ms: u64, + /// Output files the run left in its output dir (bounded). + pub outputs: Vec, +} + +/// Host → miner guest on [`MINER_PORT`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostToMiner { + /// The run. Answered by [`MinerToHost::Done`] or [`MinerToHost::Failed`]. + Run { + /// [`API_VERSION`]. + api_version: u32, + /// Artefact tarball. + artifact_tar: StagedFile, + /// Command. + entrypoint: Vec, + /// Deadline the guest itself also enforces. + deadline_s: u64, + /// FLOP cap. + declared_flops: u64, + /// Seed. + seed: u64, + /// Opaque params exported to the run's environment. + params: BTreeMap, + }, +} + +/// Miner guest → host. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MinerToHost { + /// Sent once after boot. + Ready { + /// Guest agent name/version. + agent: String, + /// [`API_VERSION`]. + api_version: u32, + }, + /// The run finished (or was cut by the guest's own deadline). + Done { + /// Exit code. + exit_code: Option, + /// Guest deadline hit. + timed_out: bool, + /// Bounded tail. + stdout_tail: String, + /// Measured FLOPs. + flops_used: Option, + /// Output files (bounded). + outputs: Vec, + }, + /// The guest could not run at all. + Failed { + /// Why. + error: String, + }, +} + +/// Refuse a peer that speaks another version. +/// +/// # Errors +/// +/// [`ProtoError::WrongVersion`]. +pub fn check_version(got: u32) -> Result<(), ProtoError> { + if got == API_VERSION { + Ok(()) + } else { + Err(ProtoError::WrongVersion { got }) + } +} + +/// Encode one frame. +/// +/// # Errors +/// +/// [`ProtoError::Decode`] when the value does not serialise, +/// [`ProtoError::FrameTooLarge`] over the cap. +pub fn encode_frame(value: &T) -> Result, ProtoError> { + let body = serde_json::to_vec(value).map_err(|e| ProtoError::Decode(e.to_string()))?; + let len = u32::try_from(body.len()).map_err(|_| ProtoError::FrameTooLarge(u32::MAX))?; + if len > MAX_FRAME_BYTES { + return Err(ProtoError::FrameTooLarge(len)); + } + let mut out = Vec::with_capacity(body.len().saturating_add(4)); + out.extend_from_slice(&len.to_be_bytes()); + out.extend_from_slice(&body); + Ok(out) +} + +/// Write one frame. +/// +/// # Errors +/// +/// See [`encode_frame`]; [`ProtoError::Io`] on the channel. +pub async fn write_frame( + w: &mut W, + value: &T, +) -> Result<(), ProtoError> { + let bytes = encode_frame(value)?; + w.write_all(&bytes) + .await + .map_err(|e| ProtoError::Io(e.to_string()))?; + w.flush().await.map_err(|e| ProtoError::Io(e.to_string())) +} + +/// Read one frame. +/// +/// # Errors +/// +/// [`ProtoError::FrameTooLarge`], [`ProtoError::Decode`], [`ProtoError::Io`]. +pub async fn read_frame Deserialize<'de>>( + r: &mut R, +) -> Result { + let mut len = [0u8; 4]; + r.read_exact(&mut len) + .await + .map_err(|e| ProtoError::Io(e.to_string()))?; + let len = u32::from_be_bytes(len); + if len > MAX_FRAME_BYTES { + return Err(ProtoError::FrameTooLarge(len)); + } + let mut body = vec![0u8; len as usize]; + r.read_exact(&mut body) + .await + .map_err(|e| ProtoError::Io(e.to_string()))?; + serde_json::from_slice(&body).map_err(|e| ProtoError::Decode(e.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_rlm::fixtures::request; + + #[tokio::test] + async fn frames_round_trip_and_oversized_frames_refuse() { + let msg = HostToRlm::Hello { + api_version: API_VERSION, + topic_id: "topic-a".into(), + vm_id: "vm-0".into(), + }; + let bytes = encode_frame(&msg).expect("encode"); + let body_len = u32::try_from(bytes.len() - 4).expect("fits"); + assert_eq!(&bytes[..4], &body_len.to_be_bytes()); + let mut cursor = std::io::Cursor::new(bytes); + let back: HostToRlm = read_frame(&mut cursor).await.expect("decode"); + assert_eq!(back, msg); + + let (mut a, mut b) = tokio::io::duplex(1 << 16); + let job = HostToRlm::Run { + job: Box::new(VmJob::Archive { + topic_id: "topic-a".into(), + }), + }; + write_frame(&mut a, &job).await.expect("write"); + let back: HostToRlm = read_frame(&mut b).await.expect("read"); + assert_eq!(back, job); + + let mut huge = std::io::Cursor::new((MAX_FRAME_BYTES + 1).to_be_bytes().to_vec()); + let err = read_frame::<_, HostToRlm>(&mut huge) + .await + .expect_err("too large"); + assert!(matches!(err, ProtoError::FrameTooLarge(_)), "{err}"); + assert!(check_version(API_VERSION).is_ok()); + assert_eq!(check_version(2), Err(ProtoError::WrongVersion { got: 2 })); + } + + #[test] + fn staged_files_round_trip_and_sister_documents_are_public() { + let f = StagedFile::new("artifact.tar", b"\x00\x01binary"); + assert_eq!(f.bytes().expect("decode"), b"\x00\x01binary"); + let mut bad = f.clone(); + bad.bytes_b64 = "!!".into(); + assert!(bad.bytes().is_err()); + let req = request(); + let sister = SisterRequest { + topic_id: req.topic_id.clone(), + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.clone(), + artifact_tar: f, + entrypoint: vec!["./run.sh".into()], + deadline_s: req.sandbox.deadline_s, + declared_flops: req.declared_flops, + seed: req.seed, + params: req.constraints.params.clone(), + }; + let json = serde_json::to_string(&sister).expect("json"); + for forbidden in ["/run/base", "api_key", "127.0.0.1", "base_url"] { + assert!(!json.contains(forbidden), "{json}"); + } + let back: SisterRequest = serde_json::from_str(&json).expect("round trip"); + assert_eq!(back, sister); + let done = MinerToHost::Done { + exit_code: Some(0), + timed_out: false, + stdout_tail: "ok".into(), + flops_used: Some(7), + outputs: vec![], + }; + assert!(serde_json::to_string(&done) + .expect("json") + .contains("\"type\":\"done\"")); + assert_eq!(RLM_JOB_PORT, 5000); + assert_eq!(SISTER_PORT, 5001); + assert_eq!(MINER_PORT, 5002); + } +} diff --git a/crates/proof-vm-proto/src/lib.rs b/crates/proof-vm-proto/src/lib.rs new file mode 100644 index 000000000..13aa45e78 --- /dev/null +++ b/crates/proof-vm-proto/src/lib.rs @@ -0,0 +1,307 @@ +//! Wire protocol of the Proof topic-VM orchestrator. +//! +//! Two boundaries share these types: +//! +//! 1. **Control plane ↔ agent** (HTTPS, bearer): `FirecrackerOrchestrator` +//! in `proof-vm-fc` calls the `proof-vm-orchestrator` agent on a dedicated +//! KVM host. Requests carry a [`TopicVmSpec`] or a [`VmJob`] — public topic +//! data, digests, rule versions — never a host path, a key, or an origin. +//! Every VM is bound to exactly one `topic_id`; every job and teardown +//! names that topic again and the agent refuses a mismatch. +//! 2. **Agent ↔ guests** (Firecracker vsock, length-prefixed JSON, +//! [`guest`]): the agent hands the RLM guest its jobs, stages owner key +//! material the control plane never reads, and boots a **sister** miner +//! guest (no network, no host filesystem) when the RLM asks for a run. +//! The host — not the RLM — stamps [`SisterAttestation`] and the report's +//! `sandboxed` / `flops_used`. +//! +//! Nothing here names a benchmark, a model, or a repository. + +#![forbid(unsafe_code)] +#![allow(clippy::module_name_repetitions)] + +use proof_rlm::{RetainPolicy, SandboxPolicy, TopicVmSpec, VmHandle, VmJob, VmJobOutput}; +use serde::{Deserialize, Serialize}; + +pub mod guest; + +/// Only accepted `api_version` on both boundaries. +pub const API_VERSION: u32 = 1; + +/// Port the agent listens on by default (HTTPS on the KVM host). +pub const DEFAULT_AGENT_PORT: u16 = 8200; + +/// Agent HTTP paths. +pub mod paths { + /// `GET` readiness (no auth beyond the bearer). + pub const HEALTH: &str = "/v1/health"; + /// `POST` create a topic VM. + pub const VMS: &str = "/v1/vms"; + + /// `DELETE` teardown / retain one VM. + #[must_use] + pub fn vm(vm_id: &str) -> String { + format!("/v1/vms/{vm_id}") + } + + /// `POST` run one job inside the VM. + #[must_use] + pub fn vm_jobs(vm_id: &str) -> String { + format!("/v1/vms/{vm_id}/jobs") + } + + /// `GET` the VM bound to a topic (404 = none). + #[must_use] + pub fn vm_by_topic(topic_id: &str) -> String { + format!("/v1/vms/by-topic/{topic_id}") + } +} + +/// `GET /v1/health`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentHealth { + /// Equals [`API_VERSION`]. + pub api_version: u32, + /// Whether the hypervisor could boot a VM right now. + pub ready: bool, + /// Why not (empty when ready). Never a secret. + pub reason: String, + /// Backend name (`firecracker`, or `fake` in tests). + pub hypervisor: String, + /// VMs currently bound (running or retained). + pub vms: usize, +} + +/// `POST /v1/vms` body. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CreateVmRequest { + /// What to boot, for which topic. + pub spec: TopicVmSpec, +} + +/// Where a VM is in its life. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VmState { + /// Booted and taking jobs. + Running, + /// Torn down under [`RetainPolicy::Retain`]: process gone, scratch kept for audit. + Retained, + /// Torn down under [`RetainPolicy::Destroy`]: nothing left on the host. + Destroyed, +} + +/// One topic VM as the agent knows it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmRecord { + /// Topic ↔ VM binding. + pub handle: VmHandle, + /// `sha256:` digest of the RLM image the host verified before boot. + pub image_digest: String, + /// vCPUs booted. + pub vcpus: u32, + /// Guest memory booted. + pub mem_mib: u32, + /// Sandbox policy the VM was created under. + pub sandbox: SandboxPolicy, + /// What teardown does by default. + pub retain: RetainPolicy, + /// Lifecycle state. + pub state: VmState, +} + +/// `POST /v1/vms/{vm_id}/jobs` body. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunJobRequest { + /// Must equal the VM's bound topic **and** the job's own topic. + pub topic_id: String, + /// The work (public data only). + pub job: VmJob, +} + +/// What the host attests about the sister miner guest a job used. +/// +/// Written by the agent from what it booted and observed, never copied from +/// the RLM guest. The control plane cross-checks it against the report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SisterAttestation { + /// Host id of the sister VM (destroyed after the run). + pub sister_vm_id: String, + /// `sha256:` digest of the miner-guest image the host verified and booted. + pub image_digest: String, + /// The host booted the sister and the run happened inside it. + pub sandboxed: bool, + /// Network the sister had. Always `none`: the artefact travels over vsock. + pub network: String, + /// FLOPs the guest measured for the run (host-relayed, never RLM-authored). + pub flops_used: Option, + /// Wall-clock of the sister run. + pub wall_ms: u64, + /// Exit code of the run inside the guest (`None` = killed at the deadline). + pub exit_code: Option, +} + +/// `POST /v1/vms/{vm_id}/jobs` response. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunJobResponse { + /// Echo of the bound topic. + pub topic_id: String, + /// Echo of the VM. + pub vm_id: String, + /// The job's output, host-stamped for paid runs. + pub output: VmJobOutput, + /// Present iff the host booted a sister guest for this job. + pub sister: Option, +} + +/// `DELETE /v1/vms/{vm_id}` body. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TeardownRequest { + /// Must equal the VM's bound topic. + pub topic_id: String, + /// Destroy or retain. + pub policy: RetainPolicy, +} + +/// `DELETE /v1/vms/{vm_id}` response. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TeardownResponse { + /// Echo of the bound topic. + pub topic_id: String, + /// Echo of the VM. + pub vm_id: String, + /// End state. + pub state: VmState, + /// `true` only when the host reached the requested end state. + pub confirmed: bool, +} + +/// Machine-readable error class. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorCode { + /// Missing or wrong bearer. + Unauthorized, + /// Hypervisor cannot boot right now (binaries, `/dev/kvm`, image). + NotReady, + /// Spec failed validation on the host. + BadSpec, + /// Request topic ≠ VM's bound topic (or ≠ the job's topic). + TopicMismatch, + /// The topic already has a VM. + AlreadyExists, + /// No such VM. + NotFound, + /// The VM is running another job. + Busy, + /// The hypervisor or guest failed. + Backend, + /// The guest answered with the wrong output shape. + WrongOutput, +} + +impl ErrorCode { + /// HTTP status the agent answers with. + #[must_use] + pub fn status(self) -> u16 { + match self { + Self::Unauthorized => 401, + Self::NotReady => 503, + Self::BadSpec => 400, + Self::TopicMismatch | Self::AlreadyExists | Self::Busy => 409, + Self::NotFound => 404, + Self::Backend | Self::WrongOutput => 502, + } + } +} + +/// Error body every non-2xx answer carries. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ErrorBody { + /// Class. + pub code: ErrorCode, + /// Human detail. Never a secret, never a host path the CP could act on. + pub error: String, +} + +/// Why a wire document is not usable. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ProtoError { + /// A frame was longer than [`guest::MAX_FRAME_BYTES`]. + #[error("frame of {0} bytes exceeds the {max} byte cap", max = guest::MAX_FRAME_BYTES)] + FrameTooLarge(u32), + /// JSON did not parse. + #[error("decode: {0}")] + Decode(String), + /// I/O on the channel. + #[error("channel: {0}")] + Io(String), + /// The peer speaks another `api_version`. + #[error("peer api_version {got}, this build speaks {API_VERSION}")] + WrongVersion { + /// What the peer said. + got: u32, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_rlm::fixtures::{pinned_template, request}; + + #[test] + fn paths_are_stable_and_codes_map_to_statuses() { + assert_eq!(paths::HEALTH, "/v1/health"); + assert_eq!(paths::VMS, "/v1/vms"); + assert_eq!(paths::vm("vm-1"), "/v1/vms/vm-1"); + assert_eq!(paths::vm_jobs("vm-1"), "/v1/vms/vm-1/jobs"); + assert_eq!(paths::vm_by_topic("topic-a"), "/v1/vms/by-topic/topic-a"); + assert_eq!(ErrorCode::Unauthorized.status(), 401); + assert_eq!(ErrorCode::NotReady.status(), 503); + assert_eq!(ErrorCode::BadSpec.status(), 400); + assert_eq!(ErrorCode::TopicMismatch.status(), 409); + assert_eq!(ErrorCode::NotFound.status(), 404); + assert_eq!(ErrorCode::Backend.status(), 502); + assert_eq!(DEFAULT_AGENT_PORT, 8200); + assert_eq!(API_VERSION, 1); + } + + #[test] + fn documents_round_trip_and_carry_public_data_only() { + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + let create = CreateVmRequest { spec }; + let json = serde_json::to_string(&create).expect("json"); + let back: CreateVmRequest = serde_json::from_str(&json).expect("round trip"); + assert_eq!(back, create); + let run = RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Evaluate { + request: req.clone(), + checklist_digest: "c".into(), + rules_version: 1, + }, + }; + let json = serde_json::to_string(&run).expect("json"); + for forbidden in ["/run/base", "/opt/base", "api_key", "127.0.0.1", "base_url"] { + assert!(!json.contains(forbidden), "leaked {forbidden}: {json}"); + } + let back: RunJobRequest = serde_json::from_str(&json).expect("round trip"); + assert_eq!(back.job.topic_id(), req.topic_id); + let err = ErrorBody { + code: ErrorCode::TopicMismatch, + error: "x".into(), + }; + let json = serde_json::to_string(&err).expect("json"); + assert!(json.contains("topic_mismatch"), "{json}"); + let resp = TeardownResponse { + topic_id: "t".into(), + vm_id: "v".into(), + state: VmState::Retained, + confirmed: true, + }; + assert!(serde_json::to_string(&resp) + .expect("json") + .contains("\"retained\"")); + } +} From 2ecc65a1328b9dac5236c2252c29a7a3d1516380 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 16:46:18 +0000 Subject: [PATCH 02/12] feat(proof-fc-host): firecracker + jailer backend with sister miner guest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Hypervisor the proof-vm-orchestrator agent drives on a dedicated KVM host. Per topic VM: resolve /sha256-.ext4 for the digest the control plane pinned and re-verify its bytes (cached by len+mtime), build the jail (kernel copy, read-only rootfs copy, fresh ext4 scratch, vm-config.json with jail-relative paths, vsock at /v.sock), a TAP on its own /30 plus a per-VM nftables table that forwards only the operator's egress allowlist (empty list = no egress) and masquerades out the uplink, exec Firecracker through the jailer (no --daemonize / --new-pid-ns so the child handle is the VM), Hello over vsock, stage owner key material read from the host's own owner_key_dir (the control plane never sees it). Paid jobs (Baseline / Evaluate) listen on v.sock_5001 for the RLM's sister request: topic must match the VM's, the artefact tar must hash to the stated digest, then a second microVM boots from the pinned sister image with no network interface, receives the bytes and the run over vsock, is held to the deadline (+grace, host kill as backstop), and is destroyed. The host writes the SisterAttestation (sandboxed, network none, guest-measured flops_used, wall, exit) that the agent stamps onto the report. One sister per job; none for inspection or rule proposals. Teardown kills the VM, drops the table + TAP, then rm -rf (Destroy) or moves the jail under retain_dir (Retain). Every host command goes through a Shell trait; tests assert the exact argv with a recording shell and prove ready() refuses on a host without firecracker/jailer//dev/kvm — nothing in CI boots a VM. proto: SisterAnswer (result | refused) for the sister channel. Co-authored-by: Mathis --- Cargo.lock | 17 + crates/proof-fc-host/Cargo.toml | 29 ++ crates/proof-fc-host/src/config.rs | 265 +++++++++++++++ crates/proof-fc-host/src/images.rs | 163 ++++++++++ crates/proof-fc-host/src/jail.rs | 375 +++++++++++++++++++++ crates/proof-fc-host/src/lib.rs | 501 +++++++++++++++++++++++++++++ crates/proof-fc-host/src/net.rs | 214 ++++++++++++ crates/proof-fc-host/src/shell.rs | 113 +++++++ crates/proof-fc-host/src/sister.rs | 284 ++++++++++++++++ crates/proof-fc-host/src/vsock.rs | 227 +++++++++++++ crates/proof-vm-proto/src/guest.rs | 19 +- 11 files changed, 2206 insertions(+), 1 deletion(-) create mode 100644 crates/proof-fc-host/Cargo.toml create mode 100644 crates/proof-fc-host/src/config.rs create mode 100644 crates/proof-fc-host/src/images.rs create mode 100644 crates/proof-fc-host/src/jail.rs create mode 100644 crates/proof-fc-host/src/lib.rs create mode 100644 crates/proof-fc-host/src/net.rs create mode 100644 crates/proof-fc-host/src/shell.rs create mode 100644 crates/proof-fc-host/src/sister.rs create mode 100644 crates/proof-fc-host/src/vsock.rs diff --git a/Cargo.lock b/Cargo.lock index 77ea09fb2..ff8b1bd7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3499,6 +3499,23 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "proof-fc-host" +version = "0.1.0" +dependencies = [ + "async-trait", + "hex", + "proof-rlm", + "proof-vm-agent", + "proof-vm-proto", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", + "tracing", +] + [[package]] name = "proof-harvest" version = "0.1.0" diff --git a/crates/proof-fc-host/Cargo.toml b/crates/proof-fc-host/Cargo.toml new file mode 100644 index 000000000..ad7dac5b1 --- /dev/null +++ b/crates/proof-fc-host/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "proof-fc-host" +description = "Firecracker + jailer backend of the proof-vm-orchestrator agent: digest-verified images, one jailed RLM microVM per topic, vsock job channel, per-VM egress allowlist (nftables), sister miner guest with no network for every paid run, destroy-or-retain teardown. Every host action is rendered through a Shell trait so tests never spawn firecracker." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +async-trait = "0.1" +hex = "0.4" +proof-rlm = { path = "../proof-rlm" } +proof-vm-agent = { path = "../proof-vm-agent" } +proof-vm-proto = { path = "../proof-vm-proto" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" +tokio = { version = "1", features = ["fs", "io-util", "net", "process", "rt", "sync", "time", "macros"] } +tracing = "0.1" + +[dev-dependencies] +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/proof-fc-host/src/config.rs b/crates/proof-fc-host/src/config.rs new file mode 100644 index 000000000..de12cca2a --- /dev/null +++ b/crates/proof-fc-host/src/config.rs @@ -0,0 +1,265 @@ +//! Operator configuration of the Firecracker host. Paths and pins only — +//! no secret ever lives here; owner key material is read from +//! `owner_key_dir` at boot time and staged over vsock. + +use std::net::Ipv4Addr; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use proof_vm_agent::HvError; + +/// Locked sister (miner guest) shape: vCPUs. +pub const DEFAULT_SISTER_VCPUS: u32 = 2; +/// Locked sister (miner guest) shape: memory. +pub const DEFAULT_SISTER_MEM_MIB: u32 = 4_096; +/// RLM VM writable scratch drive. +pub const DEFAULT_SCRATCH_MIB: u32 = 8_192; +/// Sister scratch drive (outputs + unpacked artefact). +pub const DEFAULT_SISTER_SCRATCH_MIB: u32 = 2_048; +/// Largest artefact tarball the host relays into a sister. +pub const MAX_ARTIFACT_TAR_BYTES: usize = 64 * 1024 * 1024; + +/// L4 protocol of one allowlist entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Proto { + /// TCP only. + Tcp, + /// UDP only. + Udp, + /// Any protocol (no port). + Any, +} + +/// One egress allowlist entry: `CIDR[:port[/tcp|udp]]`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EgressAllow { + /// Destination network. + pub net: Ipv4Addr, + /// Prefix length. + pub prefix: u8, + /// Destination port (`None` = any). + pub port: Option, + /// Protocol. + pub proto: Proto, +} + +impl EgressAllow { + /// Parse `1.2.3.4/32:443`, `10.0.0.0/8`, `1.1.1.1/32:53/udp`, `1.2.3.4:443`. + /// + /// # Errors + /// + /// [`HvError::Spec`] naming the entry. + pub fn parse(raw: &str) -> Result { + let bad = |why: &str| HvError::Spec(format!("egress allow {raw:?}: {why}")); + let s = raw.trim(); + let (addr_part, rest) = match s.split_once(':') { + Some((a, r)) => (a, Some(r)), + None => (s, None), + }; + let (net_s, prefix_s) = addr_part.split_once('/').unwrap_or((addr_part, "32")); + let net: Ipv4Addr = net_s.parse().map_err(|_| bad("not an IPv4 address"))?; + let prefix: u8 = prefix_s.parse().map_err(|_| bad("bad prefix"))?; + if prefix > 32 { + return Err(bad("prefix over 32")); + } + let (port, proto) = match rest { + None => (None, Proto::Any), + Some(r) => { + let (p, proto) = match r.split_once('/') { + Some((p, "tcp")) => (p, Proto::Tcp), + Some((p, "udp")) => (p, Proto::Udp), + Some(_) => return Err(bad("protocol must be tcp or udp")), + None => (r, Proto::Tcp), + }; + let port: u16 = p.parse().map_err(|_| bad("bad port"))?; + if port == 0 { + return Err(bad("port 0")); + } + (Some(port), proto) + } + }; + Ok(Self { + net, + prefix, + port, + proto, + }) + } + + /// `1.2.3.4/32`. + #[must_use] + pub fn cidr(&self) -> String { + format!("{}/{}", self.net, self.prefix) + } +} + +/// Everything the backend needs. Built by the agent binary from flags / env. +#[derive(Debug, Clone)] +pub struct HostConfig { + /// Statically linked `firecracker` binary the jailer execs. + pub firecracker_bin: PathBuf, + /// `jailer` binary (same release as `firecracker_bin`). + pub jailer_bin: PathBuf, + /// `--chroot-base-dir` (jails land under `/firecracker//root`). + pub chroot_base: PathBuf, + /// Root filesystem images named `sha256-.ext4`. + pub image_dir: PathBuf, + /// Guest kernel (`vmlinux`), pinned by `kernel_digest`. + pub kernel: PathBuf, + /// `sha256:` of `kernel`. + pub kernel_digest: String, + /// `sha256:` of the miner-guest (sister) rootfs in `image_dir`. + pub sister_image_digest: String, + /// uid / gid the jailer drops to. + pub jail_uid: u32, + /// See `jail_uid`. + pub jail_gid: u32, + /// RLM VM scratch drive size. + pub scratch_mib: u32, + /// Sister vCPUs (sized by the host, never by the RLM). + pub sister_vcpus: u32, + /// Sister memory. + pub sister_mem_mib: u32, + /// Sister scratch drive. + pub sister_scratch_mib: u32, + /// How long a guest agent may take to say `Ready`. + pub boot_timeout: Duration, + /// Slack added to a job's own deadline before the host kills it. + pub deadline_grace: Duration, + /// Budget for jobs that carry no deadline (`ProposeRules`, `Archive`). + pub default_job_timeout: Duration, + /// Directory whose files are staged into the RLM VM over vsock. The + /// control plane never reads them. + pub owner_key_dir: Option, + /// Uplink the RLM VMs are masqueraded through. + pub uplink: String, + /// First /30 of the host↔guest point-to-point pool. + pub net_base: Ipv4Addr, + /// What the RLM VM may reach. Empty = no egress at all. + pub egress_allow: Vec, + /// Where retained jails are moved on `Retain`. + pub retain_dir: PathBuf, +} + +impl HostConfig { + /// Defaults for a host laid out like the runbook. + #[must_use] + pub fn defaults() -> Self { + Self { + firecracker_bin: PathBuf::from("/usr/local/bin/firecracker"), + jailer_bin: PathBuf::from("/usr/local/bin/jailer"), + chroot_base: PathBuf::from("/srv/jailer"), + image_dir: PathBuf::from("/var/lib/proof-vm/images"), + kernel: PathBuf::from("/var/lib/proof-vm/vmlinux"), + kernel_digest: String::new(), + sister_image_digest: String::new(), + jail_uid: 65534, + jail_gid: 65534, + scratch_mib: DEFAULT_SCRATCH_MIB, + sister_vcpus: DEFAULT_SISTER_VCPUS, + sister_mem_mib: DEFAULT_SISTER_MEM_MIB, + sister_scratch_mib: DEFAULT_SISTER_SCRATCH_MIB, + boot_timeout: Duration::from_mins(2), + deadline_grace: Duration::from_secs(30), + default_job_timeout: Duration::from_hours(1), + owner_key_dir: None, + uplink: "eth0".into(), + net_base: Ipv4Addr::new(172, 16, 0, 0), + egress_allow: Vec::new(), + retain_dir: PathBuf::from("/var/lib/proof-vm/retained"), + } + } + + /// Shape check of the pins and sizes (paths are checked by `ready()`). + /// + /// # Errors + /// + /// [`HvError::Spec`]. + pub fn validate(&self) -> Result<(), HvError> { + if crate::images::digest_hex(&self.kernel_digest).is_none() { + return Err(HvError::Spec( + "kernel_digest must be sha256:<64 hex> (do not invent one)".into(), + )); + } + if crate::images::digest_hex(&self.sister_image_digest).is_none() { + return Err(HvError::Spec( + "sister_image_digest must be sha256:<64 hex> (do not invent one)".into(), + )); + } + if !(1..=64).contains(&self.sister_vcpus) || !(512..=131_072).contains(&self.sister_mem_mib) + { + return Err(HvError::Spec("sister vcpus / mem_mib out of range".into())); + } + if self.uplink.trim().is_empty() || self.uplink.len() > 15 { + return Err(HvError::Spec("uplink must be an interface name".into())); + } + Ok(()) + } + + /// Jail root for `vm_id`: `///root`. + #[must_use] + pub fn jail_root(&self, vm_id: &str) -> PathBuf { + let exec_name = self.firecracker_bin.file_name().map_or_else( + || "firecracker".into(), + |n| n.to_string_lossy().into_owned(), + ); + self.chroot_base.join(exec_name).join(vm_id).join("root") + } + + /// `//` (what teardown removes or retains). + #[must_use] + pub fn jail_dir(&self, vm_id: &str) -> PathBuf { + self.jail_root(vm_id) + .parent() + .map_or_else(|| self.chroot_base.join(vm_id), Path::to_path_buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn egress_entries_parse_cidr_port_and_proto() { + let a = EgressAllow::parse("1.2.3.4/32:443").expect("parse"); + assert_eq!(a.cidr(), "1.2.3.4/32"); + assert_eq!((a.port, a.proto), (Some(443), Proto::Tcp)); + let b = EgressAllow::parse("10.0.0.0/8").expect("parse"); + assert_eq!((b.port, b.proto), (None, Proto::Any)); + let c = EgressAllow::parse(" 1.1.1.1:53/udp ").expect("parse"); + assert_eq!( + (c.cidr(), c.port, c.proto), + ("1.1.1.1/32".into(), Some(53), Proto::Udp) + ); + for bad in [ + "nope", + "1.2.3.4/33", + "1.2.3.4:0", + "1.2.3.4:443/sctp", + "1.2.3.4:x", + ] { + assert!(EgressAllow::parse(bad).is_err(), "{bad}"); + } + } + + #[test] + fn defaults_validate_only_with_real_pins_and_jail_paths_follow_the_jailer() { + let mut cfg = HostConfig::defaults(); + assert!(cfg.validate().is_err(), "no invented digests"); + cfg.kernel_digest = format!("sha256:{}", "aa".repeat(32)); + assert!(cfg.validate().is_err()); + cfg.sister_image_digest = format!("sha256:{}", "bb".repeat(32)); + cfg.validate().expect("pinned"); + assert_eq!((cfg.sister_vcpus, cfg.sister_mem_mib), (2, 4_096)); + assert_eq!( + cfg.jail_root("topic-a-0001"), + PathBuf::from("/srv/jailer/firecracker/topic-a-0001/root") + ); + assert_eq!( + cfg.jail_dir("topic-a-0001"), + PathBuf::from("/srv/jailer/firecracker/topic-a-0001") + ); + cfg.uplink = "a-very-long-interface-name".into(); + assert!(cfg.validate().is_err()); + } +} diff --git a/crates/proof-fc-host/src/images.rs b/crates/proof-fc-host/src/images.rs new file mode 100644 index 000000000..fdd2c917c --- /dev/null +++ b/crates/proof-fc-host/src/images.rs @@ -0,0 +1,163 @@ +//! Digest-pinned images. An image boots only if the bytes on disk hash to +//! the `sha256:` the control plane (RLM image) or the operator (kernel, +//! sister image) pinned. Verified files are remembered by `(len, mtime)` so +//! a multi-GiB rootfs is hashed once per change, not once per boot. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::SystemTime; + +use proof_vm_agent::HvError; +use sha2::{Digest, Sha256}; + +/// The 64 hex chars of a `sha256:` pin (case-insensitive, trimmed). +#[must_use] +pub fn digest_hex(pin: &str) -> Option { + let hex = pin + .trim() + .strip_prefix("sha256:")? + .trim() + .to_ascii_lowercase(); + (hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit())).then_some(hex) +} + +/// `/sha256-.ext4` for a pin. +/// +/// # Errors +/// +/// [`HvError::Image`] when the pin is malformed or the file is absent. +pub fn image_path(image_dir: &Path, pin: &str) -> Result { + let hex = + digest_hex(pin).ok_or_else(|| HvError::Image(format!("{pin:?} is not sha256:")))?; + let path = image_dir.join(format!("sha256-{hex}.ext4")); + if path.is_file() { + Ok(path) + } else { + Err(HvError::Image(format!( + "sha256:{hex} (no {} on this host)", + path.display() + ))) + } +} + +/// sha256 hex of a file, streamed. +/// +/// # Errors +/// +/// [`HvError::Backend`] on I/O. +pub fn sha256_file(path: &Path) -> Result { + let mut f = std::fs::File::open(path) + .map_err(|e| HvError::Backend(format!("open {}: {e}", path.display())))?; + let mut h = Sha256::new(); + std::io::copy(&mut f, &mut h) + .map_err(|e| HvError::Backend(format!("read {}: {e}", path.display())))?; + Ok(hex::encode(h.finalize())) +} + +fn stamp(path: &Path) -> Option<(u64, SystemTime)> { + let m = std::fs::metadata(path).ok()?; + Some((m.len(), m.modified().ok()?)) +} + +/// Remembers which files verified against which digest. +#[derive(Default)] +pub struct ImageCache { + verified: Mutex>, +} + +impl ImageCache { + /// Verify `path` hashes to `pin`, hashing only when the file changed. + /// + /// # Errors + /// + /// [`HvError::Image`] on mismatch, [`HvError::Backend`] on I/O. + pub async fn verify(&self, path: &Path, pin: &str) -> Result<(), HvError> { + let want = digest_hex(pin) + .ok_or_else(|| HvError::Image(format!("{pin:?} is not sha256:")))?; + let now = stamp(path) + .ok_or_else(|| HvError::Image(format!("{} is not readable", path.display())))?; + { + let cache = self + .verified + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((hex, len, mtime)) = cache.get(path) { + if *hex == want && (*len, *mtime) == now { + return Ok(()); + } + } + } + let p = path.to_path_buf(); + let got = tokio::task::spawn_blocking(move || sha256_file(&p)) + .await + .map_err(|e| HvError::Backend(format!("hash task: {e}")))??; + if got != want { + return Err(HvError::Image(format!( + "{} hashes to sha256:{got}, pin is sha256:{want}", + path.display() + ))); + } + self.verified + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(path.to_path_buf(), (want, now.0, now.1)); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("proof-fc-images-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("dir"); + d + } + + #[test] + fn pins_are_sha256_hex_and_images_are_named_by_them() { + let hex = "ab".repeat(32); + assert_eq!( + digest_hex(&format!(" sha256:{} ", hex.to_uppercase())), + Some(hex.clone()) + ); + assert_eq!(digest_hex(&hex), None, "prefix required"); + assert_eq!(digest_hex("sha256:abc"), None); + assert_eq!(digest_hex(""), None); + let d = dir("paths"); + let err = image_path(&d, &format!("sha256:{hex}")).expect_err("absent"); + assert!(matches!(err, HvError::Image(_)), "{err}"); + std::fs::write(d.join(format!("sha256-{hex}.ext4")), b"x").expect("write"); + assert_eq!( + image_path(&d, &format!("sha256:{hex}")).expect("present"), + d.join(format!("sha256-{hex}.ext4")) + ); + assert!(image_path(&d, "not-a-pin").is_err()); + } + + #[tokio::test] + async fn verify_hashes_once_per_change_and_refuses_a_mismatch() { + let d = dir("verify"); + let file = d.join("rootfs.ext4"); + std::fs::write(&file, b"rootfs bytes").expect("write"); + let good = format!("sha256:{}", sha256_file(&file).expect("hash")); + let cache = ImageCache::default(); + cache.verify(&file, &good).await.expect("matches"); + cache.verify(&file, &good).await.expect("cached"); + let bad = format!("sha256:{}", "00".repeat(32)); + let err = cache.verify(&file, &bad).await.expect_err("mismatch"); + assert!(matches!(err, HvError::Image(_)), "{err}"); + assert!(err.to_string().contains("pin is"), "{err}"); + std::fs::write(&file, b"tampered after verification").expect("rewrite"); + let err = cache + .verify(&file, &good) + .await + .expect_err("changed bytes are re-hashed"); + assert!(matches!(err, HvError::Image(_)), "{err}"); + assert!(cache.verify(&d.join("missing"), &good).await.is_err()); + assert!(cache.verify(&file, "junk").await.is_err()); + } +} diff --git a/crates/proof-fc-host/src/jail.rs b/crates/proof-fc-host/src/jail.rs new file mode 100644 index 000000000..03d34097a --- /dev/null +++ b/crates/proof-fc-host/src/jail.rs @@ -0,0 +1,375 @@ +//! Jail layout and process control for one Firecracker microVM. +//! +//! The jailer chroots Firecracker into `/firecracker//root`; +//! everything the VM needs is placed there first (kernel copy, read-only +//! rootfs copy, fresh scratch drive, `vm-config.json`) and referenced by +//! jail-relative paths. Without `--daemonize` / `--new-pid-ns` the jailer +//! `exec`s into Firecracker, so the child handle we hold **is** the VM. + +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use proof_vm_agent::HvError; +use proof_vm_proto::guest::GUEST_CID; +use serde_json::{json, Value}; + +use crate::config::HostConfig; +use crate::net::NetPlan; +use crate::shell::{sh, Shell}; + +/// Jail-relative names Firecracker sees. +pub const KERNEL_IN_JAIL: &str = "vmlinux"; +/// See [`KERNEL_IN_JAIL`]. +pub const ROOTFS_IN_JAIL: &str = "rootfs.ext4"; +/// See [`KERNEL_IN_JAIL`]. +pub const SCRATCH_IN_JAIL: &str = "scratch.ext4"; +/// See [`KERNEL_IN_JAIL`]. +pub const CONFIG_IN_JAIL: &str = "vm-config.json"; +/// Firecracker vsock UDS (host connects here with `CONNECT `). +pub const VSOCK_IN_JAIL: &str = "v.sock"; +/// Firecracker API socket (unused by the agent; kept for operators). +pub const API_SOCK_IN_JAIL: &str = "run/firecracker.socket"; +/// Serial console capture beside the jail. +pub const CONSOLE_LOG: &str = "console.log"; + +/// What one microVM boots with. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VmBoot { + /// Jail id (also the VM id). + pub id: String, + /// vCPUs. + pub vcpus: u32, + /// Memory. + pub mem_mib: u32, + /// Verified rootfs on the host to copy in. + pub rootfs: PathBuf, + /// Scratch drive size. + pub scratch_mib: u32, + /// Network (RLM VM) or none (sister guest). + pub net: Option, +} + +/// Firecracker `--config-file` document for `boot`. +#[must_use] +pub fn vm_config(boot: &VmBoot) -> Value { + let mut boot_args = String::from("console=ttyS0 reboot=k panic=1 pci=off"); + let mut ifaces = Vec::new(); + if let Some(net) = &boot.net { + boot_args.push(' '); + boot_args.push_str(&net.boot_arg()); + ifaces.push(json!({ + "iface_id": "eth0", + "guest_mac": net.guest_mac, + "host_dev_name": net.tap, + })); + } + json!({ + "boot-source": { + "kernel_image_path": format!("/{KERNEL_IN_JAIL}"), + "boot_args": boot_args, + "initrd_path": null, + }, + "drives": [ + { + "drive_id": "rootfs", + "path_on_host": format!("/{ROOTFS_IN_JAIL}"), + "is_root_device": true, + "is_read_only": true, + }, + { + "drive_id": "scratch", + "path_on_host": format!("/{SCRATCH_IN_JAIL}"), + "is_root_device": false, + "is_read_only": false, + } + ], + "machine-config": { + "vcpu_count": boot.vcpus, + "mem_size_mib": boot.mem_mib, + "smt": false, + }, + "vsock": { + "guest_cid": GUEST_CID, + "uds_path": format!("/{VSOCK_IN_JAIL}"), + }, + "network-interfaces": ifaces, + }) +} + +/// The jailer argv (program excluded). +#[must_use] +pub fn jailer_args(cfg: &HostConfig, id: &str) -> Vec { + vec![ + "--id".into(), + id.into(), + "--exec-file".into(), + cfg.firecracker_bin.display().to_string(), + "--uid".into(), + cfg.jail_uid.to_string(), + "--gid".into(), + cfg.jail_gid.to_string(), + "--chroot-base-dir".into(), + cfg.chroot_base.display().to_string(), + "--".into(), + "--config-file".into(), + format!("/{CONFIG_IN_JAIL}"), + "--api-sock".into(), + format!("/{API_SOCK_IN_JAIL}"), + ] +} + +/// Build the jail root for `boot`: copies (reflink when the filesystem can), +/// a fresh ext4 scratch drive, ownership for the jail uid, the config file, +/// and the per-VM nftables ruleset beside the root (never inside it). +/// +/// # Errors +/// +/// [`HvError::Backend`] from the first failing step. +pub async fn prepare( + cfg: &HostConfig, + shell: &dyn Shell, + boot: &VmBoot, +) -> Result { + let root = cfg.jail_root(&boot.id); + let root_s = root.display().to_string(); + if root.exists() { + return Err(HvError::Backend(format!("jail {root_s} already exists"))); + } + sh(shell, "mkdir", &["-p", &format!("{root_s}/run")]).await?; + let kernel_src = cfg.kernel.display().to_string(); + sh( + shell, + "cp", + &[ + "--reflink=auto", + &kernel_src, + &format!("{root_s}/{KERNEL_IN_JAIL}"), + ], + ) + .await?; + let rootfs_src = boot.rootfs.display().to_string(); + sh( + shell, + "cp", + &[ + "--reflink=auto", + &rootfs_src, + &format!("{root_s}/{ROOTFS_IN_JAIL}"), + ], + ) + .await?; + let scratch = format!("{root_s}/{SCRATCH_IN_JAIL}"); + sh( + shell, + "truncate", + &["-s", &format!("{}M", boot.scratch_mib), &scratch], + ) + .await?; + sh(shell, "mkfs.ext4", &["-q", "-F", &scratch]).await?; + let config = serde_json::to_string_pretty(&vm_config(boot)) + .map_err(|e| HvError::Backend(format!("render vm config: {e}")))?; + write(&root.join(CONFIG_IN_JAIL), config.as_bytes())?; + if let Some(net) = &boot.net { + write( + &cfg.jail_dir(&boot.id).join("net.nft"), + net.ruleset().as_bytes(), + )?; + } + let owner = format!("{}:{}", cfg.jail_uid, cfg.jail_gid); + sh(shell, "chown", &["-R", &owner, &root_s]).await?; + Ok(root) +} + +fn write(path: &Path, bytes: &[u8]) -> Result<(), HvError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| HvError::Backend(format!("mkdir {}: {e}", parent.display())))?; + } + std::fs::write(path, bytes) + .map_err(|e| HvError::Backend(format!("write {}: {e}", path.display()))) +} + +/// Spawn the jailer (which execs Firecracker). Console goes to +/// `/console.log`. +/// +/// # Errors +/// +/// [`HvError::Backend`]. +pub fn spawn(cfg: &HostConfig, id: &str) -> Result { + let log_path = cfg.jail_dir(id).join(CONSOLE_LOG); + let log = std::fs::File::create(&log_path) + .map_err(|e| HvError::Backend(format!("console log {}: {e}", log_path.display())))?; + let err = log + .try_clone() + .map_err(|e| HvError::Backend(format!("console log clone: {e}")))?; + tokio::process::Command::new(&cfg.jailer_bin) + .args(jailer_args(cfg, id)) + .stdin(Stdio::null()) + .stdout(Stdio::from(log)) + .stderr(Stdio::from(err)) + .kill_on_drop(true) + .spawn() + .map_err(|e| HvError::Backend(format!("spawn {}: {e}", cfg.jailer_bin.display()))) +} + +/// Kill the VM process and reap it. +pub async fn kill(child: &mut tokio::process::Child) { + let _ = child.start_kill(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await; +} + +/// Remove the jail entirely. +/// +/// # Errors +/// +/// [`HvError::Backend`]. +pub async fn destroy(cfg: &HostConfig, shell: &dyn Shell, id: &str) -> Result<(), HvError> { + let dir = cfg.jail_dir(id).display().to_string(); + sh(shell, "rm", &["-rf", &dir]).await?; + Ok(()) +} + +/// Move the jail under `retain_dir` for audit (scratch, console, config). +/// +/// # Errors +/// +/// [`HvError::Backend`]. +pub async fn retain(cfg: &HostConfig, shell: &dyn Shell, id: &str) -> Result { + let dest = cfg.retain_dir.join(id); + sh( + shell, + "mkdir", + &["-p", &cfg.retain_dir.display().to_string()], + ) + .await?; + let src = cfg.jail_dir(id).display().to_string(); + sh(shell, "mv", &[&src, &dest.display().to_string()]).await?; + Ok(dest) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shell::RecordingShell; + + fn cfg(tag: &str) -> HostConfig { + let mut c = HostConfig::defaults(); + c.chroot_base = + std::env::temp_dir().join(format!("proof-fc-jail-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&c.chroot_base); + c.kernel = PathBuf::from("/var/lib/proof-vm/vmlinux"); + c.uplink = "eno1".into(); + c + } + + fn boot(net: Option) -> VmBoot { + VmBoot { + id: "topic-a-0001".into(), + vcpus: 4, + mem_mib: 8_192, + rootfs: PathBuf::from("/var/lib/proof-vm/images/sha256-aa.ext4"), + scratch_mib: 8_192, + net, + } + } + + #[test] + fn the_config_file_references_jail_relative_paths_and_a_read_only_root() { + let c = cfg("config"); + let plan = NetPlan::for_index(&c, 3); + let v = vm_config(&boot(Some(plan.clone()))); + assert_eq!(v["boot-source"]["kernel_image_path"], "/vmlinux"); + let args = v["boot-source"]["boot_args"].as_str().expect("args"); + assert!( + args.starts_with("console=ttyS0 reboot=k panic=1 pci=off ip=172.16.0.14::172.16.0.13:"), + "{args}" + ); + assert_eq!(v["drives"][0]["path_on_host"], "/rootfs.ext4"); + assert_eq!(v["drives"][0]["is_read_only"], true); + assert_eq!(v["drives"][0]["is_root_device"], true); + assert_eq!(v["drives"][1]["path_on_host"], "/scratch.ext4"); + assert_eq!(v["drives"][1]["is_read_only"], false); + assert_eq!(v["machine-config"]["vcpu_count"], 4); + assert_eq!(v["machine-config"]["mem_size_mib"], 8_192); + assert_eq!(v["vsock"]["guest_cid"], 3); + assert_eq!(v["vsock"]["uds_path"], "/v.sock"); + assert_eq!(v["network-interfaces"][0]["host_dev_name"], "pfc3"); + assert_eq!(v["network-interfaces"][0]["guest_mac"], plan.guest_mac); + let sister = vm_config(&boot(None)); + assert_eq!( + sister["network-interfaces"] + .as_array() + .expect("array") + .len(), + 0, + "sister has no nic" + ); + assert!(!sister["boot-source"]["boot_args"] + .as_str() + .expect("args") + .contains("ip=")); + } + + #[test] + fn jailer_argv_pins_id_uid_gid_chroot_and_the_config_file() { + let c = cfg("argv"); + let args = jailer_args(&c, "topic-a-0001"); + let joined = args.join(" "); + assert!(joined.starts_with("--id topic-a-0001 --exec-file /usr/local/bin/firecracker --uid 65534 --gid 65534 --chroot-base-dir "), "{joined}"); + assert!( + joined.ends_with("-- --config-file /vm-config.json --api-sock /run/firecracker.socket"), + "{joined}" + ); + assert!( + !joined.contains("--daemonize") && !joined.contains("--new-pid-ns"), + "the child handle must be the vm" + ); + } + + #[tokio::test] + async fn prepare_renders_copy_scratch_chown_and_writes_config_plus_rules() { + let c = cfg("prepare"); + let shell = RecordingShell::default(); + let plan = NetPlan::for_index(&c, 0); + let root = prepare(&c, &shell, &boot(Some(plan))) + .await + .expect("prepare"); + assert_eq!(root, c.jail_root("topic-a-0001")); + let flat: Vec = shell.calls().iter().map(|a| a.join(" ")).collect(); + let r = root.display().to_string(); + assert_eq!(flat[0], format!("mkdir -p {r}/run")); + assert_eq!( + flat[1], + format!("cp --reflink=auto /var/lib/proof-vm/vmlinux {r}/vmlinux") + ); + assert_eq!( + flat[2], + format!("cp --reflink=auto /var/lib/proof-vm/images/sha256-aa.ext4 {r}/rootfs.ext4") + ); + assert_eq!(flat[3], format!("truncate -s 8192M {r}/scratch.ext4")); + assert_eq!(flat[4], format!("mkfs.ext4 -q -F {r}/scratch.ext4")); + assert_eq!(flat[5], format!("chown -R 65534:65534 {r}")); + let config = std::fs::read_to_string(root.join(CONFIG_IN_JAIL)).expect("config written"); + assert!(config.contains("\"vcpu_count\": 4")); + let rules = std::fs::read_to_string(c.jail_dir("topic-a-0001").join("net.nft")) + .expect("rules written"); + assert!(rules.starts_with("table inet proof_vm_pfc0 {")); + assert!( + !root.join("net.nft").exists(), + "rules stay outside the chroot" + ); + let err = prepare(&c, &shell, &boot(None)) + .await + .expect_err("second jail with the same id"); + assert!(err.to_string().contains("already exists"), "{err}"); + destroy(&c, &shell, "topic-a-0001").await.expect("destroy"); + let dest = retain(&c, &shell, "topic-a-0001").await.expect("retain"); + assert_eq!(dest, c.retain_dir.join("topic-a-0001")); + let flat: Vec = shell.calls().iter().map(|a| a.join(" ")).collect(); + assert!(flat + .iter() + .any(|l| l == &format!("rm -rf {}", c.jail_dir("topic-a-0001").display()))); + assert!(flat.iter().any(|l| l.starts_with("mv "))); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } +} diff --git a/crates/proof-fc-host/src/lib.rs b/crates/proof-fc-host/src/lib.rs new file mode 100644 index 000000000..7e0a8c815 --- /dev/null +++ b/crates/proof-fc-host/src/lib.rs @@ -0,0 +1,501 @@ +//! Firecracker + jailer backend of the `proof-vm-orchestrator` agent. +//! +//! One **dedicated KVM host** runs this. For every topic the control plane +//! asks about, the host: +//! +//! 1. resolves the pinned RLM image (`/sha256-.ext4`) and +//! re-verifies its digest ([`images`]); +//! 2. builds a jail (kernel copy, read-only rootfs copy, fresh scratch +//! drive, `vm-config.json`) and a TAP on its own /30 with an nftables +//! table that allows **only** the operator's egress list ([`net`]); +//! 3. execs Firecracker through the jailer ([`jail`]), waits for the RLM +//! guest agent on vsock, and stages the owner key material from the +//! host's own directory — the control plane never sees it ([`vsock`]); +//! 4. runs jobs over vsock; while a **paid** job runs it listens for the +//! RLM's sister request and boots a second microVM with **no network** +//! for the miner artefact, then attests that run ([`sister`]); +//! 5. tears the VM down: `Destroy` removes the jail, `Retain` moves it under +//! `retain_dir` for audit. +//! +//! Every host command goes through [`Shell`], so the tests in this crate +//! assert the exact argv without spawning anything. Nothing in CI boots a +//! VM: [`FirecrackerHypervisor::ready`] refuses on a host without +//! `firecracker`, `jailer`, and `/dev/kvm`, and the tests check that refusal. + +#![forbid(unsafe_code)] +#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] + +pub mod config; +pub mod images; +pub mod jail; +pub mod net; +pub mod shell; +pub mod sister; +pub mod vsock; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use proof_rlm::{RetainPolicy, TopicVmSpec, VmJob}; +use proof_vm_agent::{BootedVm, HvError, Hypervisor, JobOutcome}; +use proof_vm_proto::guest::{ + check_version, HostToRlm, RlmToHost, SisterAnswer, SisterRequest, StagedFile, RLM_JOB_PORT, + SISTER_PORT, +}; +use proof_vm_proto::{SisterAttestation, API_VERSION}; +use tokio::net::UnixListener; +use tokio::sync::Mutex; + +pub use config::{EgressAllow, HostConfig, Proto}; +pub use images::ImageCache; +pub use net::NetPlan; +pub use shell::{RecordingShell, Shell, SystemShell}; +pub use sister::SisterCtx; + +struct LiveVm { + child: tokio::process::Child, + root: PathBuf, + net: NetPlan, +} + +/// The production [`Hypervisor`]. +pub struct FirecrackerHypervisor { + ctx: SisterCtx, + kernel_verified: AtomicBool, + vms: Mutex>, + net_index: AtomicU32, + sister_seq: AtomicU64, +} + +fn executable(path: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path).is_ok_and(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) +} + +impl FirecrackerHypervisor { + /// Over the real shell. + /// + /// # Errors + /// + /// [`HvError::Spec`] when the config's pins / sizes are malformed. + pub fn new(cfg: HostConfig) -> Result { + Self::with_shell(cfg, Arc::new(SystemShell)) + } + + /// Over any [`Shell`] (tests record instead of running). + /// + /// # Errors + /// + /// [`HvError::Spec`]. + pub fn with_shell(cfg: HostConfig, shell: Arc) -> Result { + cfg.validate()?; + Ok(Self { + ctx: SisterCtx { + cfg: Arc::new(cfg), + shell, + images: Arc::new(ImageCache::default()), + }, + kernel_verified: AtomicBool::new(false), + vms: Mutex::new(HashMap::new()), + net_index: AtomicU32::new(0), + sister_seq: AtomicU64::new(1), + }) + } + + /// The config in force. + #[must_use] + pub fn config(&self) -> &HostConfig { + &self.ctx.cfg + } + + /// Owner key material from `owner_key_dir`, to stage over vsock. Read + /// here, sent to the guest, never logged, never returned to the control + /// plane. Absent dir = nothing to stage. + fn owner_files(&self) -> Result, HvError> { + let Some(dir) = &self.ctx.cfg.owner_key_dir else { + return Ok(Vec::new()); + }; + let mut out = Vec::new(); + let entries = std::fs::read_dir(dir) + .map_err(|e| HvError::Backend(format!("owner key dir {}: {e}", dir.display())))?; + for entry in entries { + let entry = entry.map_err(|e| HvError::Backend(format!("owner key dir: {e}")))?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + let bytes = std::fs::read(&path) + .map_err(|e| HvError::Backend(format!("owner key {name}: {e}")))?; + out.push(StagedFile::new(&name, &bytes)); + } + Ok(out) + } + + async fn verify_kernel(&self) -> Result<(), HvError> { + let cfg = &self.ctx.cfg; + self.ctx + .images + .verify(&cfg.kernel, &cfg.kernel_digest) + .await?; + self.kernel_verified.store(true, Ordering::SeqCst); + Ok(()) + } + + fn job_budget(&self, job: &VmJob) -> Duration { + job.deadline_s() + .map_or(self.ctx.cfg.default_job_timeout, |d| { + Duration::from_secs(d).saturating_add(self.ctx.cfg.deadline_grace) + }) + } + + /// Accept sister requests from the RLM guest for one job; at most one + /// sister per job, none for jobs that run no miner code. + async fn serve_sisters( + ctx: Arc, + listener: UnixListener, + vm: BootedVm, + seq: Arc, + slot: Arc>>, + paid: bool, + ) { + loop { + let Ok((stream, _)) = listener.accept().await else { + return; + }; + let mut ch = vsock::GuestChannel::from_stream(stream); + let answer = match ch + .recv_within::(Duration::from_mins(1)) + .await + { + Err(e) => SisterAnswer::Refused { + error: e.to_string(), + }, + Ok(_) if !paid => SisterAnswer::Refused { + error: "this job runs no miner code; no sister".into(), + }, + Ok(req) => { + let mut taken = slot.lock().await; + if taken.is_some() { + SisterAnswer::Refused { + error: "one sister per job".into(), + } + } else { + let n = seq.fetch_add(1, Ordering::SeqCst); + match sister::run(&ctx, &vm, n, &req).await { + Ok((result, attestation)) => { + *taken = Some(attestation); + SisterAnswer::Result { result } + } + Err(e) => SisterAnswer::Refused { + error: e.to_string(), + }, + } + } + } + }; + if let Err(e) = ch.send(&answer).await { + tracing::warn!(vm_id = %vm.vm_id, "sister answer not delivered: {e}"); + } + } + } +} + +#[async_trait] +impl Hypervisor for FirecrackerHypervisor { + fn name(&self) -> &'static str { + "firecracker" + } + + fn ready(&self) -> Result<(), HvError> { + let cfg = &self.ctx.cfg; + let checks: [(&str, bool); 5] = [ + ("firecracker binary", executable(&cfg.firecracker_bin)), + ("jailer binary", executable(&cfg.jailer_bin)), + ("/dev/kvm", Path::new("/dev/kvm").exists()), + ("image dir", cfg.image_dir.is_dir()), + ("kernel", cfg.kernel.is_file()), + ]; + if let Some((what, _)) = checks.iter().find(|(_, ok)| !ok) { + return Err(HvError::NotReady(format!("{what} missing on this host"))); + } + if cfg.egress_allow.is_empty() { + tracing::debug!("egress allowlist empty: topic vms get no egress"); + } + Ok(()) + } + + async fn boot(&self, vm_id: &str, spec: &TopicVmSpec) -> Result { + self.ready()?; + spec.validate().map_err(|e| HvError::Spec(e.to_string()))?; + let cfg = self.ctx.cfg.clone(); + if !self.kernel_verified.load(Ordering::SeqCst) { + self.verify_kernel().await?; + } + let image = images::image_path(&cfg.image_dir, &spec.template.image_digest)?; + self.ctx + .images + .verify(&image, &spec.template.image_digest) + .await?; + let owner_files = self.owner_files()?; + let net = NetPlan::for_index(&cfg, self.net_index.fetch_add(1, Ordering::SeqCst)); + let boot = jail::VmBoot { + id: vm_id.to_owned(), + vcpus: spec.template.vcpus, + mem_mib: spec.template.mem_mib, + rootfs: image, + scratch_mib: cfg.scratch_mib, + net: Some(net.clone()), + }; + let shell = self.ctx.shell.as_ref(); + let root = jail::prepare(&cfg, shell, &boot).await?; + net.up(shell, cfg.jail_uid).await?; + let rules = cfg.jail_dir(vm_id).join("net.nft").display().to_string(); + net.load_rules(shell, &rules).await?; + let mut child = jail::spawn(&cfg, vm_id)?; + let hello = async { + let mut ch = + vsock::GuestChannel::connect_within(&root, RLM_JOB_PORT, cfg.boot_timeout).await?; + ch.send(&HostToRlm::Hello { + api_version: API_VERSION, + topic_id: spec.topic_id.clone(), + vm_id: vm_id.to_owned(), + }) + .await?; + match ch.recv_within::(cfg.boot_timeout).await? { + RlmToHost::Ready { api_version, agent } => { + check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; + tracing::info!(%vm_id, %agent, "rlm guest ready"); + } + other => return Err(HvError::Guest(format!("rlm guest answered {other:?} to hello"))), + } + if !owner_files.is_empty() { + let count = owner_files.len(); + ch.send(&HostToRlm::StageSecrets { files: owner_files }).await?; + match ch.recv_within::(cfg.boot_timeout).await? { + RlmToHost::Staged { count: got } if got == count => { + tracing::info!(%vm_id, count, "owner key material staged (contents not logged)"); + } + other => { + return Err(HvError::Guest(format!("staging answered {other:?}"))); + } + } + } + Ok::<(), HvError>(()) + } + .await; + if let Err(e) = hello { + jail::kill(&mut child).await; + net.down(shell).await; + let _ = jail::destroy(&cfg, shell, vm_id).await; + return Err(e); + } + self.vms + .lock() + .await + .insert(vm_id.to_owned(), LiveVm { child, root, net }); + Ok(BootedVm { + vm_id: vm_id.to_owned(), + topic_id: spec.topic_id.clone(), + image_digest: spec.template.image_digest.clone(), + }) + } + + async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result { + let root = { + let vms = self.vms.lock().await; + let live = vms + .get(&vm.vm_id) + .ok_or_else(|| HvError::Backend(format!("vm {} is not running here", vm.vm_id)))?; + live.root.clone() + }; + let paid = matches!(job, VmJob::Baseline { .. } | VmJob::Evaluate { .. }); + let listener = vsock::listen(&root, SISTER_PORT)?; + let slot = Arc::new(Mutex::new(None)); + let sisters = tokio::spawn(Self::serve_sisters( + Arc::new(SisterCtx { + cfg: self.ctx.cfg.clone(), + shell: self.ctx.shell.clone(), + images: self.ctx.images.clone(), + }), + listener, + vm.clone(), + Arc::new(AtomicU64::new( + self.sister_seq.fetch_add(1_000, Ordering::SeqCst), + )), + slot.clone(), + paid, + )); + let budget = self.job_budget(job); + let answer = async { + let mut ch = vsock::GuestChannel::connect(&root, RLM_JOB_PORT).await?; + ch.send(&HostToRlm::Run { + job: Box::new(job.clone()), + }) + .await?; + ch.recv_within::(budget).await + } + .await; + sisters.abort(); + let _ = std::fs::remove_file(vsock::listener_path(&root, SISTER_PORT)); + let sister = slot.lock().await.take(); + match answer? { + RlmToHost::Done { output } => Ok(JobOutcome { output, sister }), + RlmToHost::Failed { error } => Err(HvError::Guest(error)), + other => Err(HvError::Guest(format!( + "rlm guest answered {other:?} to a job" + ))), + } + } + + async fn teardown(&self, vm: &BootedVm, policy: RetainPolicy) -> Result { + let Some(mut live) = self.vms.lock().await.remove(&vm.vm_id) else { + return Err(HvError::Backend(format!( + "vm {} is not running here", + vm.vm_id + ))); + }; + jail::kill(&mut live.child).await; + let shell = self.ctx.shell.as_ref(); + for e in live.net.down(shell).await { + tracing::warn!(vm_id = %vm.vm_id, "network teardown: {e}"); + } + match policy { + RetainPolicy::Destroy => jail::destroy(&self.ctx.cfg, shell, &vm.vm_id).await?, + RetainPolicy::Retain => { + let dest = jail::retain(&self.ctx.cfg, shell, &vm.vm_id).await?; + tracing::info!(vm_id = %vm.vm_id, retained = %dest.display(), "topic vm retained"); + } + } + Ok(true) + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + use proof_rlm::fixtures::{pinned_template, request}; + + fn cfg(tag: &str) -> HostConfig { + let mut c = HostConfig::defaults(); + let base = std::env::temp_dir().join(format!("proof-fc-host-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + std::fs::create_dir_all(base.join("images")).expect("dir"); + c.firecracker_bin = base.join("firecracker"); + c.jailer_bin = base.join("jailer"); + c.chroot_base = base.join("jailer-root"); + c.image_dir = base.join("images"); + c.kernel = base.join("vmlinux"); + c.kernel_digest = format!("sha256:{}", "aa".repeat(32)); + c.sister_image_digest = format!("sha256:{}", "bb".repeat(32)); + c + } + + /// CI has no Firecracker: `ready()` names what is missing and nothing + /// is ever spawned. This is the "zero live FC in GitHub runners" gate. + #[tokio::test] + async fn without_firecracker_the_backend_refuses_and_never_spawns() { + let shell = Arc::new(RecordingShell::default()); + let hv = FirecrackerHypervisor::with_shell(cfg("unready"), shell.clone()).expect("config"); + assert_eq!(hv.name(), "firecracker"); + let err = hv.ready().expect_err("no binaries"); + assert!(matches!(err, HvError::NotReady(_)), "{err}"); + assert!(err.to_string().contains("firecracker binary"), "{err}"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + let err = hv + .boot("topic-a-0001", &spec) + .await + .expect_err("boot refused"); + assert!(matches!(err, HvError::NotReady(_)), "{err}"); + let vm = BootedVm { + vm_id: "topic-a-0001".into(), + topic_id: req.topic_id.clone(), + image_digest: pinned_template().image_digest, + }; + let err = hv + .run_job( + &vm, + &VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + ) + .await + .expect_err("unknown vm"); + assert!(err.to_string().contains("not running here"), "{err}"); + assert!(hv.teardown(&vm, RetainPolicy::Destroy).await.is_err()); + assert!(shell.calls().is_empty(), "no host command ran"); + let mut bad = cfg("badpin"); + bad.kernel_digest = "latest".into(); + assert!(FirecrackerHypervisor::with_shell(bad, shell).is_err()); + } + + /// With fake binaries present, the pin gate runs next: a kernel whose + /// bytes do not hash to the pin, or an RLM image absent from the image + /// dir, refuses before any jail is prepared. + #[tokio::test] + async fn pins_gate_the_boot_before_any_jail_is_built() { + let mut c = cfg("pins"); + for bin in [&c.firecracker_bin, &c.jailer_bin] { + std::fs::write(bin, b"#!/bin/sh\nexit 0\n").expect("write"); + std::fs::set_permissions(bin, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + } + std::fs::write(&c.kernel, b"not the pinned kernel").expect("kernel"); + c.kernel_digest = format!("sha256:{}", images::sha256_file(&c.kernel).expect("hash")); + let shell = Arc::new(RecordingShell::default()); + let hv = FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config"); + if !Path::new("/dev/kvm").exists() { + let err = hv.ready().expect_err("no kvm on this box"); + assert!(err.to_string().contains("/dev/kvm"), "{err}"); + return; + } + hv.ready().expect("binaries + kvm present"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + let err = hv + .boot("topic-a-0001", &spec) + .await + .expect_err("rlm image absent"); + assert!(matches!(err, HvError::Image(_)), "{err}"); + assert!(shell.calls().is_empty(), "refused before the jail"); + let owner = hv.owner_files().expect("no dir = nothing"); + assert!(owner.is_empty()); + } + + #[tokio::test] + async fn owner_key_material_is_read_from_the_host_dir_only() { + let mut c = cfg("owner"); + let dir = c.image_dir.join("../owner-keys"); + std::fs::create_dir_all(&dir).expect("dir"); + std::fs::write(dir.join("inference_key"), b"owner-key-not-a-real-secret").expect("write"); + std::fs::create_dir_all(dir.join("subdir")).expect("subdir ignored"); + c.owner_key_dir = Some(dir); + let hv = FirecrackerHypervisor::with_shell(c, Arc::new(RecordingShell::default())) + .expect("config"); + let files = hv.owner_files().expect("read"); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, "inference_key"); + assert_eq!( + files[0].bytes().expect("decode"), + b"owner-key-not-a-real-secret" + ); + let budget = hv.job_budget(&VmJob::Archive { + topic_id: "t".into(), + }); + assert_eq!(budget, hv.config().default_job_timeout); + let req = request(); + let paid = hv.job_budget(&VmJob::Baseline { + request: req.clone(), + }); + assert_eq!( + paid, + Duration::from_secs(req.sandbox.deadline_s) + hv.config().deadline_grace + ); + } +} diff --git a/crates/proof-fc-host/src/net.rs b/crates/proof-fc-host/src/net.rs new file mode 100644 index 000000000..bdff5e70d --- /dev/null +++ b/crates/proof-fc-host/src/net.rs @@ -0,0 +1,214 @@ +//! Per-VM networking: a TAP on a /30, NAT through the uplink, and an +//! nftables table that lets the RLM VM reach **only** the operator's egress +//! allowlist. The sister miner guest gets no interface at all. + +use std::net::Ipv4Addr; + +use proof_vm_agent::HvError; + +use crate::config::{EgressAllow, HostConfig, Proto}; +use crate::shell::{sh, Shell}; + +/// One RLM VM's network plan. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NetPlan { + /// Host TAP device (`pfc`, ≤15 chars). + pub tap: String, + /// Host side of the /30. + pub host_ip: Ipv4Addr, + /// Guest side of the /30. + pub guest_ip: Ipv4Addr, + /// Guest MAC (`AA:FC:...`, derived from the index). + pub guest_mac: String, + /// Uplink for masquerade. + pub uplink: String, + /// Allowlist. + pub allow: Vec, +} + +impl NetPlan { + /// Plan for the `index`-th VM: /30 number `index` of the pool. + #[must_use] + pub fn for_index(cfg: &HostConfig, index: u32) -> Self { + let base = u32::from(cfg.net_base) & !0b11; + let net = base.wrapping_add(index.wrapping_mul(4)); + let [a, b] = [(index >> 8) & 0xff, index & 0xff]; + Self { + tap: format!("pfc{index}"), + host_ip: Ipv4Addr::from(net.wrapping_add(1)), + guest_ip: Ipv4Addr::from(net.wrapping_add(2)), + guest_mac: format!("AA:FC:00:00:{a:02X}:{b:02X}"), + uplink: cfg.uplink.clone(), + allow: cfg.egress_allow.clone(), + } + } + + /// nftables table name (one per VM so teardown is one `delete table`). + #[must_use] + pub fn table(&self) -> String { + format!("proof_vm_{}", self.tap) + } + + /// Kernel `ip=` argument giving the guest its address statically. + #[must_use] + pub fn boot_arg(&self) -> String { + format!( + "ip={}::{}:255.255.255.252::eth0:off", + self.guest_ip, self.host_ip + ) + } + + /// The per-VM nftables ruleset: forward from the TAP only to the + /// allowlist (established replies back in), masquerade out the uplink, + /// drop everything else the guest sends. Empty allowlist = no egress. + #[must_use] + pub fn ruleset(&self) -> String { + use std::fmt::Write as _; + let t = self.table(); + let tap = &self.tap; + let mut out = format!( + "table inet {t} {{\n chain forward {{\n type filter hook forward priority 0; policy accept;\n oifname \"{tap}\" ct state established,related accept\n oifname \"{tap}\" drop\n iifname \"{tap}\" ct state established,related accept\n" + ); + for a in &self.allow { + let l4 = match (a.proto, a.port) { + (Proto::Tcp, Some(p)) => format!(" tcp dport {p}"), + (Proto::Udp, Some(p)) => format!(" udp dport {p}"), + (Proto::Tcp, None) => " meta l4proto tcp".into(), + (Proto::Udp, None) => " meta l4proto udp".into(), + (Proto::Any, _) => String::new(), + }; + let _ = writeln!( + out, + " iifname \"{tap}\" ip daddr {}{l4} accept", + a.cidr() + ); + } + let _ = write!( + out, + " iifname \"{tap}\" drop\n }}\n chain postrouting {{\n type nat hook postrouting priority 100; policy accept;\n ip saddr {} oifname \"{}\" masquerade\n }}\n}}\n", + self.guest_ip, self.uplink + ); + out + } + + /// Create the TAP (owned by the jail uid so jailed Firecracker can open + /// it), address it, enable forwarding, load the ruleset. + /// + /// # Errors + /// + /// [`HvError::Backend`] from the first failing command. + pub async fn up(&self, shell: &dyn Shell, jail_uid: u32) -> Result<(), HvError> { + let uid = jail_uid.to_string(); + sh( + shell, + "ip", + &[ + "tuntap", "add", "dev", &self.tap, "mode", "tap", "user", &uid, + ], + ) + .await?; + let cidr = format!("{}/30", self.host_ip); + sh(shell, "ip", &["addr", "add", &cidr, "dev", &self.tap]).await?; + sh(shell, "ip", &["link", "set", &self.tap, "up"]).await?; + sh(shell, "sysctl", &["-q", "-w", "net.ipv4.ip_forward=1"]).await?; + Ok(()) + } + + /// Load the ruleset from `ruleset_path` (written beside the jail by the + /// caller; `nft` reads files, the [`Shell`] carries no stdin). + /// + /// # Errors + /// + /// [`HvError::Backend`]. + pub async fn load_rules(&self, shell: &dyn Shell, ruleset_path: &str) -> Result<(), HvError> { + sh(shell, "nft", &["-f", ruleset_path]).await?; + Ok(()) + } + + /// Delete the table and the TAP. Errors are reported, not fatal: a + /// half-torn network must not leave the VM record alive. + pub async fn down(&self, shell: &dyn Shell) -> Vec { + let mut errs = Vec::new(); + if let Err(e) = sh(shell, "nft", &["delete", "table", "inet", &self.table()]).await { + errs.push(e); + } + if let Err(e) = sh(shell, "ip", &["link", "del", &self.tap]).await { + errs.push(e); + } + errs + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shell::RecordingShell; + + fn cfg() -> HostConfig { + let mut c = HostConfig::defaults(); + c.egress_allow = vec![ + EgressAllow::parse("203.0.113.10/32:443").expect("allow"), + EgressAllow::parse("198.51.100.0/24").expect("allow"), + EgressAllow::parse("1.1.1.1:53/udp").expect("allow"), + ]; + c + } + + #[test] + fn plans_carve_the_pool_into_p2p_slash_30s() { + let p0 = NetPlan::for_index(&cfg(), 0); + assert_eq!(p0.tap, "pfc0"); + assert_eq!(p0.host_ip, Ipv4Addr::new(172, 16, 0, 1)); + assert_eq!(p0.guest_ip, Ipv4Addr::new(172, 16, 0, 2)); + assert_eq!(p0.guest_mac, "AA:FC:00:00:00:00"); + assert_eq!( + p0.boot_arg(), + "ip=172.16.0.2::172.16.0.1:255.255.255.252::eth0:off" + ); + let p300 = NetPlan::for_index(&cfg(), 300); + assert_eq!(p300.host_ip, Ipv4Addr::new(172, 16, 4, 177)); + assert_eq!(p300.guest_ip, Ipv4Addr::new(172, 16, 4, 178)); + assert_eq!(p300.guest_mac, "AA:FC:00:00:01:2C"); + assert!(p300.tap.len() <= 15); + assert_eq!(p300.table(), "proof_vm_pfc300"); + } + + #[test] + fn the_ruleset_allows_only_the_list_and_drops_the_rest() { + let p = NetPlan::for_index(&cfg(), 7); + let rules = p.ruleset(); + let want = "table inet proof_vm_pfc7 {\n chain forward {\n type filter hook forward priority 0; policy accept;\n oifname \"pfc7\" ct state established,related accept\n oifname \"pfc7\" drop\n iifname \"pfc7\" ct state established,related accept\n iifname \"pfc7\" ip daddr 203.0.113.10/32 tcp dport 443 accept\n iifname \"pfc7\" ip daddr 198.51.100.0/24 accept\n iifname \"pfc7\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"pfc7\" drop\n }\n chain postrouting {\n type nat hook postrouting priority 100; policy accept;\n ip saddr 172.16.0.30 oifname \"eth0\" masquerade\n }\n}\n"; + assert_eq!(rules, want); + let mut none = cfg(); + none.egress_allow.clear(); + let closed = NetPlan::for_index(&none, 0).ruleset(); + assert!(!closed.contains("daddr"), "no allow → no accept: {closed}"); + assert!(closed.contains("iifname \"pfc0\" drop")); + } + + #[tokio::test] + async fn up_and_down_render_the_expected_commands() { + let shell = RecordingShell::default(); + let p = NetPlan::for_index(&cfg(), 2); + p.up(&shell, 65534).await.expect("up"); + p.load_rules(&shell, "/srv/jailer/firecracker/x/net.nft") + .await + .expect("rules"); + assert!(p.down(&shell).await.is_empty()); + let calls = shell.calls(); + let flat: Vec = calls.iter().map(|c| c.join(" ")).collect(); + assert!( + flat.contains(&"ip tuntap add dev pfc2 mode tap user 65534".to_owned()), + "{flat:?}" + ); + assert!( + flat.contains(&"ip addr add 172.16.0.9/30 dev pfc2".to_owned()), + "{flat:?}" + ); + assert!(flat.contains(&"ip link set pfc2 up".to_owned())); + assert!(flat.contains(&"sysctl -q -w net.ipv4.ip_forward=1".to_owned())); + assert!(flat.contains(&"nft -f /srv/jailer/firecracker/x/net.nft".to_owned())); + assert!(flat.contains(&"nft delete table inet proof_vm_pfc2".to_owned())); + assert!(flat.contains(&"ip link del pfc2".to_owned())); + } +} diff --git a/crates/proof-fc-host/src/shell.rs b/crates/proof-fc-host/src/shell.rs new file mode 100644 index 000000000..b7d347222 --- /dev/null +++ b/crates/proof-fc-host/src/shell.rs @@ -0,0 +1,113 @@ +//! Host commands behind a trait, so every jail / network / cleanup step is +//! rendered as an argv the tests can assert on without running anything. + +use std::sync::Mutex; + +use async_trait::async_trait; +use proof_vm_agent::HvError; + +/// One finished command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CmdOutput { + /// Exit status (`None` = killed by signal). + pub code: Option, + /// Stdout, lossy UTF-8. + pub stdout: String, + /// Stderr, lossy UTF-8. + pub stderr: String, +} + +impl CmdOutput { + /// `Ok` iff the command exited 0. + /// + /// # Errors + /// + /// [`HvError::Backend`] naming the program and the stderr tail. + pub fn ok(self, program: &str) -> Result { + if self.code == Some(0) { + Ok(self) + } else { + let tail: String = self + .stderr + .chars() + .rev() + .take(300) + .collect::>() + .into_iter() + .rev() + .collect(); + Err(HvError::Backend(format!( + "{program} exited {:?}: {}", + self.code, + tail.trim() + ))) + } + } +} + +/// Runs host commands. Production: [`SystemShell`]. Tests: [`RecordingShell`]. +#[async_trait] +pub trait Shell: Send + Sync { + /// Run `program` with `args` to completion. + async fn run(&self, program: &str, args: &[String]) -> Result; +} + +/// `tokio::process::Command`. +pub struct SystemShell; + +#[async_trait] +impl Shell for SystemShell { + async fn run(&self, program: &str, args: &[String]) -> Result { + let out = tokio::process::Command::new(program) + .args(args) + .kill_on_drop(true) + .output() + .await + .map_err(|e| HvError::Backend(format!("spawn {program}: {e}")))?; + Ok(CmdOutput { + code: out.status.code(), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + }) + } +} + +/// Records every argv and answers success. Never touches the host. +#[derive(Default)] +pub struct RecordingShell { + calls: Mutex>>, +} + +impl RecordingShell { + /// Every command run so far, program first. + #[must_use] + pub fn calls(&self) -> Vec> { + self.calls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } +} + +#[async_trait] +impl Shell for RecordingShell { + async fn run(&self, program: &str, args: &[String]) -> Result { + let mut recorded = vec![program.to_owned()]; + recorded.extend(args.iter().cloned()); + self.calls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(recorded); + Ok(CmdOutput { + code: Some(0), + stdout: String::new(), + stderr: String::new(), + }) + } +} + +/// Convenience: run and require exit 0. +pub async fn sh(shell: &dyn Shell, program: &str, args: &[&str]) -> Result { + let owned: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + shell.run(program, &owned).await?.ok(program) +} diff --git a/crates/proof-fc-host/src/sister.rs b/crates/proof-fc-host/src/sister.rs new file mode 100644 index 000000000..a89af0d5b --- /dev/null +++ b/crates/proof-fc-host/src/sister.rs @@ -0,0 +1,284 @@ +//! The **sister** miner guest: a second Firecracker microVM the host boots +//! beside the RLM VM for one paid run, with no network interface, fed the +//! artefact bytes the RLM already inspected over vsock, held to the run's +//! deadline, then destroyed. The host — not the RLM — knows it happened, +//! which is what [`SisterAttestation`] records. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use proof_vm_agent::{BootedVm, HvError}; +use proof_vm_proto::guest::{ + check_version, HostToMiner, MinerToHost, SisterRequest, SisterResult, MINER_PORT, +}; +use proof_vm_proto::{SisterAttestation, API_VERSION}; +use sha2::{Digest, Sha256}; + +use crate::config::{HostConfig, MAX_ARTIFACT_TAR_BYTES}; +use crate::images::{image_path, ImageCache}; +use crate::jail::{self, VmBoot}; +use crate::shell::Shell; +use crate::vsock::GuestChannel; + +/// Longest jail id the jailer accepts. +const MAX_JAIL_ID: usize = 64; + +/// What a sister run needs from the host. +pub struct SisterCtx { + /// Host config (sister image pin, sizes, timeouts). + pub cfg: Arc, + /// Command runner. + pub shell: Arc, + /// Verified-image cache. + pub images: Arc, +} + +/// `-s`, trimmed so the jailer accepts it. +#[must_use] +pub fn sister_id(parent_vm_id: &str, seq: u64) -> String { + let suffix = format!("-s{seq}"); + let keep = MAX_JAIL_ID.saturating_sub(suffix.len()); + let mut prefix = parent_vm_id.to_owned(); + prefix.truncate(keep); + format!("{prefix}{suffix}") +} + +/// Refuse a request the host will not boot a sister for. +/// +/// # Errors +/// +/// [`HvError::Spec`]: wrong topic, oversized or mis-hashed artefact. +pub fn check_request(parent: &BootedVm, req: &SisterRequest) -> Result, HvError> { + if req.topic_id != parent.topic_id { + return Err(HvError::Spec(format!( + "sister request names topic {:?}, vm is bound to {:?}", + req.topic_id, parent.topic_id + ))); + } + let tar = req + .artifact_tar + .bytes() + .map_err(|e| HvError::Spec(format!("artifact_tar: {e}")))?; + if tar.is_empty() || tar.len() > MAX_ARTIFACT_TAR_BYTES { + return Err(HvError::Spec(format!( + "artifact_tar is {} bytes (1..={MAX_ARTIFACT_TAR_BYTES})", + tar.len() + ))); + } + let got = hex::encode(Sha256::digest(&tar)); + if !got.eq_ignore_ascii_case(req.artifact_digest.trim()) { + return Err(HvError::Spec(format!( + "artifact_tar hashes to {got}, request says {}", + req.artifact_digest + ))); + } + if req.entrypoint.is_empty() || req.deadline_s == 0 { + return Err(HvError::Spec( + "entrypoint and deadline_s are required".into(), + )); + } + Ok(tar) +} + +fn u64_ms(d: Duration) -> u64 { + u64::try_from(d.as_millis()).unwrap_or(u64::MAX) +} + +/// Boot a sister for `req`, run it, destroy it, and attest. +/// +/// # Errors +/// +/// [`HvError::Spec`] (refused before boot), [`HvError::Image`] (sister image +/// missing / mismatched), [`HvError::Backend`] / [`HvError::Guest`] (the +/// host could not run it at all). A run cut at the deadline is **not** an +/// error: it is a result with `timed_out: true`. +pub async fn run( + ctx: &SisterCtx, + parent: &BootedVm, + seq: u64, + req: &SisterRequest, +) -> Result<(SisterResult, SisterAttestation), HvError> { + check_request(parent, req)?; + let cfg = &ctx.cfg; + let image = image_path(&cfg.image_dir, &cfg.sister_image_digest)?; + ctx.images.verify(&image, &cfg.sister_image_digest).await?; + let id = sister_id(&parent.vm_id, seq); + let boot = VmBoot { + id: id.clone(), + vcpus: cfg.sister_vcpus, + mem_mib: cfg.sister_mem_mib, + rootfs: image, + scratch_mib: cfg.sister_scratch_mib, + net: None, + }; + let root = jail::prepare(cfg, ctx.shell.as_ref(), &boot).await?; + let mut child = jail::spawn(cfg, &id)?; + let started = Instant::now(); + tracing::info!(sister = %id, parent = %parent.vm_id, topic_id = %parent.topic_id, "sister guest booting (no network)"); + let budget = Duration::from_secs(req.deadline_s).saturating_add(cfg.deadline_grace); + let outcome = async { + let mut ch = GuestChannel::connect_within(&root, MINER_PORT, cfg.boot_timeout).await?; + match ch.recv_within::(cfg.boot_timeout).await? { + MinerToHost::Ready { api_version, .. } => { + check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))? + } + other => { + return Err(HvError::Guest(format!( + "sister spoke before ready: {other:?}" + ))); + } + } + ch.send(&HostToMiner::Run { + api_version: API_VERSION, + artifact_tar: req.artifact_tar.clone(), + entrypoint: req.entrypoint.clone(), + deadline_s: req.deadline_s, + declared_flops: req.declared_flops, + seed: req.seed, + params: req.params.clone(), + }) + .await?; + ch.recv_within::(budget).await + } + .await; + jail::kill(&mut child).await; + let wall_ms = u64_ms(started.elapsed()); + if let Err(e) = jail::destroy(cfg, ctx.shell.as_ref(), &id).await { + tracing::warn!(sister = %id, "sister jail cleanup: {e}"); + } + let (exit_code, timed_out, stdout_tail, flops_used, outputs) = match outcome { + Ok(MinerToHost::Done { + exit_code, + timed_out, + stdout_tail, + flops_used, + outputs, + }) => (exit_code, timed_out, stdout_tail, flops_used, outputs), + // Nothing ran: zero is a measurement here, so the RLM can write a + // reject the control plane persists instead of a 503. + Ok(MinerToHost::Failed { error }) => ( + None, + false, + format!("guest failed: {error}"), + Some(0), + vec![], + ), + Ok(MinerToHost::Ready { .. }) => { + return Err(HvError::Guest("sister answered ready twice".into())); + } + Err(HvError::Deadline(_)) => ( + None, + true, + format!( + "killed by the host {}s after the deadline", + cfg.deadline_grace.as_secs() + ), + None, + vec![], + ), + Err(e) => return Err(e), + }; + let result = SisterResult { + sister_vm_id: id.clone(), + image_digest: cfg.sister_image_digest.clone(), + exit_code, + timed_out, + stdout_tail, + flops_used, + wall_ms, + outputs, + }; + let attestation = SisterAttestation { + sister_vm_id: id, + image_digest: cfg.sister_image_digest.clone(), + sandboxed: true, + network: "none".into(), + flops_used, + wall_ms, + exit_code, + }; + Ok((result, attestation)) +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_vm_proto::guest::StagedFile; + + fn parent() -> BootedVm { + BootedVm { + vm_id: "topic-a-0001".into(), + topic_id: "topic-a".into(), + image_digest: format!("sha256:{}", "cc".repeat(32)), + } + } + + fn request(tar: &[u8]) -> SisterRequest { + SisterRequest { + topic_id: "topic-a".into(), + submission_digest: "d".into(), + artifact_digest: hex::encode(Sha256::digest(tar)), + artifact_tar: StagedFile::new("artifact.tar", tar), + entrypoint: vec!["./run.sh".into()], + deadline_s: 60, + declared_flops: 1, + seed: 7, + params: std::collections::BTreeMap::default(), + } + } + + #[test] + fn sister_ids_fit_the_jailer_and_requests_are_bound_and_hashed() { + assert_eq!(sister_id("topic-a-0001", 3), "topic-a-0001-s3"); + let long = "t".repeat(63); + let id = sister_id(&long, 12); + assert!(id.len() <= MAX_JAIL_ID, "{id}"); + assert!(id.ends_with("-s12")); + let tar = b"tar bytes"; + check_request(&parent(), &request(tar)).expect("bound + hashed"); + let mut other = request(tar); + other.topic_id = "topic-b".into(); + assert!(matches!( + check_request(&parent(), &other), + Err(HvError::Spec(_)) + )); + let mut wrong = request(tar); + wrong.artifact_digest = "00".repeat(32); + let err = check_request(&parent(), &wrong).expect_err("hash"); + assert!(err.to_string().contains("hashes to"), "{err}"); + let mut empty = request(b""); + empty.artifact_digest = hex::encode(Sha256::digest(b"")); + assert!(check_request(&parent(), &empty).is_err()); + let mut junk = request(tar); + junk.artifact_tar.bytes_b64 = "!!".into(); + assert!(check_request(&parent(), &junk).is_err()); + let mut no_entry = request(tar); + no_entry.entrypoint.clear(); + assert!(check_request(&parent(), &no_entry).is_err()); + } + + /// The host never boots a sister whose image is not on disk at the + /// pinned digest — and this test proves no process is spawned for it. + #[tokio::test] + async fn a_missing_sister_image_refuses_before_any_jail_or_process() { + let mut cfg = HostConfig::defaults(); + cfg.image_dir = + std::env::temp_dir().join(format!("proof-fc-sister-{}", std::process::id())); + std::fs::create_dir_all(&cfg.image_dir).expect("dir"); + cfg.sister_image_digest = format!("sha256:{}", "bb".repeat(32)); + let shell = Arc::new(crate::shell::RecordingShell::default()); + let ctx = SisterCtx { + cfg: Arc::new(cfg), + shell: shell.clone(), + images: Arc::new(ImageCache::default()), + }; + let err = run(&ctx, &parent(), 1, &request(b"tar")) + .await + .expect_err("no image"); + assert!(matches!(err, HvError::Image(_)), "{err}"); + assert!( + shell.calls().is_empty(), + "nothing prepared, nothing spawned" + ); + } +} diff --git a/crates/proof-fc-host/src/vsock.rs b/crates/proof-fc-host/src/vsock.rs new file mode 100644 index 000000000..fe49400f6 --- /dev/null +++ b/crates/proof-fc-host/src/vsock.rs @@ -0,0 +1,227 @@ +//! Host side of Firecracker vsock. +//! +//! Host → guest: connect to `/v.sock`, send `CONNECT \n`, +//! read `OK \n`, then speak framed JSON. Guest → host: Firecracker +//! connects to `/v.sock_`, so the host listens there. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use proof_vm_agent::HvError; +use proof_vm_proto::guest::{read_frame, write_frame}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; + +use crate::jail::VSOCK_IN_JAIL; + +/// `/v.sock`. +#[must_use] +pub fn uds_path(jail_root: &Path) -> PathBuf { + jail_root.join(VSOCK_IN_JAIL) +} + +/// `/v.sock_` — where Firecracker delivers guest-initiated +/// connections to host `port`. +#[must_use] +pub fn listener_path(jail_root: &Path, port: u32) -> PathBuf { + jail_root.join(format!("{VSOCK_IN_JAIL}_{port}")) +} + +/// One framed channel to a guest port. +#[derive(Debug)] +pub struct GuestChannel { + stream: BufReader, +} + +impl GuestChannel { + /// Connect to guest `port` through the VM's vsock UDS. + /// + /// # Errors + /// + /// [`HvError::Guest`] when the guest is not listening or the handshake + /// is not `OK`. + pub async fn connect(jail_root: &Path, port: u32) -> Result { + let path = uds_path(jail_root); + let mut stream = UnixStream::connect(&path) + .await + .map_err(|e| HvError::Guest(format!("vsock {}: {e}", path.display())))?; + stream + .write_all(format!("CONNECT {port}\n").as_bytes()) + .await + .map_err(|e| HvError::Guest(format!("vsock connect {port}: {e}")))?; + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream + .read_line(&mut line) + .await + .map_err(|e| HvError::Guest(format!("vsock handshake {port}: {e}")))?; + if !line.starts_with("OK ") { + return Err(HvError::Guest(format!( + "vsock port {port}: guest not listening ({})", + line.trim() + ))); + } + Ok(Self { stream }) + } + + /// Keep retrying `connect` until the guest answers or `budget` runs out. + /// + /// # Errors + /// + /// [`HvError::Guest`] with the last failure. + pub async fn connect_within( + jail_root: &Path, + port: u32, + budget: Duration, + ) -> Result { + let deadline = tokio::time::Instant::now() + budget; + let mut last = HvError::Guest(format!("vsock port {port}: never came up")); + while tokio::time::Instant::now() < deadline { + match Self::connect(jail_root, port).await { + Ok(c) => return Ok(c), + Err(e) => last = e, + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + Err(last) + } + + /// Wrap an accepted guest-initiated stream. + #[must_use] + pub fn from_stream(stream: UnixStream) -> Self { + Self { + stream: BufReader::new(stream), + } + } + + /// Send one frame. + /// + /// # Errors + /// + /// [`HvError::Guest`]. + pub async fn send(&mut self, value: &T) -> Result<(), HvError> { + write_frame(self.stream.get_mut(), value) + .await + .map_err(|e| HvError::Guest(format!("send: {e}"))) + } + + /// Receive one frame. + /// + /// # Errors + /// + /// [`HvError::Guest`]. + pub async fn recv(&mut self) -> Result { + read_frame(&mut self.stream) + .await + .map_err(|e| HvError::Guest(format!("recv: {e}"))) + } + + /// Receive one frame within `budget`. + /// + /// # Errors + /// + /// [`HvError::Deadline`] on timeout, else [`HvError::Guest`]. + pub async fn recv_within( + &mut self, + budget: Duration, + ) -> Result { + tokio::time::timeout(budget, self.recv()) + .await + .map_err(|_| HvError::Deadline(budget.as_secs()))? + } +} + +/// Listen for guest-initiated connections to host `port` (bind before boot). +/// +/// # Errors +/// +/// [`HvError::Backend`]. +pub fn listen(jail_root: &Path, port: u32) -> Result { + let path = listener_path(jail_root, port); + let _ = std::fs::remove_file(&path); + UnixListener::bind(&path) + .map_err(|e| HvError::Backend(format!("listen {}: {e}", path.display()))) +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_vm_proto::guest::{HostToRlm, RlmToHost}; + use proof_vm_proto::API_VERSION; + + fn root(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("proof-fc-vsock-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("dir"); + d + } + + /// A stand-in for Firecracker's UDS end: answers the CONNECT handshake + /// and then speaks the guest protocol. Bound before it is spawned so the + /// host side cannot race it on a current-thread runtime. + async fn fake_firecracker(listener: UnixListener, ok: bool) { + let (stream, _) = listener.accept().await.expect("accept"); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.expect("connect line"); + assert_eq!(line, "CONNECT 5000\n"); + if !ok { + return; + } + stream + .get_mut() + .write_all(b"OK 1073741824\n") + .await + .expect("ok"); + let hello: HostToRlm = read_frame(&mut stream).await.expect("hello"); + assert!(matches!(hello, HostToRlm::Hello { .. })); + write_frame( + stream.get_mut(), + &RlmToHost::Ready { + agent: "fake-guest".into(), + api_version: API_VERSION, + }, + ) + .await + .expect("ready"); + } + + #[tokio::test] + async fn the_handshake_then_frames_and_a_silent_guest_refuses() { + let r = root("ok"); + let bound = UnixListener::bind(uds_path(&r)).expect("bind"); + let server = tokio::spawn(fake_firecracker(bound, true)); + let mut ch = GuestChannel::connect_within(&r, 5000, Duration::from_secs(5)) + .await + .expect("connect"); + ch.send(&HostToRlm::Hello { + api_version: API_VERSION, + topic_id: "topic-a".into(), + vm_id: "topic-a-0001".into(), + }) + .await + .expect("send"); + let ready: RlmToHost = ch.recv_within(Duration::from_secs(5)).await.expect("ready"); + assert!(matches!(ready, RlmToHost::Ready { .. })); + server.await.expect("server"); + + let r2 = root("silent"); + let bound = UnixListener::bind(uds_path(&r2)).expect("bind"); + let server = tokio::spawn(fake_firecracker(bound, false)); + let err = GuestChannel::connect(&r2, 5000).await.expect_err("no OK"); + assert!(matches!(err, HvError::Guest(_)), "{err}"); + server.await.expect("server"); + + let r3 = root("absent"); + let err = GuestChannel::connect_within(&r3, 5000, Duration::from_millis(700)) + .await + .expect_err("no socket"); + assert!(matches!(err, HvError::Guest(_)), "{err}"); + assert_eq!(listener_path(&r3, 5001), r3.join("v.sock_5001")); + let l = listen(&r3, 5001).expect("listen"); + drop(l); + let _ = listen(&r3, 5001).expect("rebinding replaces a stale socket"); + } +} diff --git a/crates/proof-vm-proto/src/guest.rs b/crates/proof-vm-proto/src/guest.rs index b6fe890bb..60c53375a 100644 --- a/crates/proof-vm-proto/src/guest.rs +++ b/crates/proof-vm-proto/src/guest.rs @@ -7,7 +7,7 @@ //! | Port | Direction | Purpose | //! |------|-----------|---------| //! | [`RLM_JOB_PORT`] | host → RLM guest | [`HostToRlm`] / [`RlmToHost`]: hello, secret staging, jobs | -//! | [`SISTER_PORT`] | RLM guest → host | [`SisterRequest`] / [`SisterResult`]: "run this artefact in a sister guest" | +//! | [`SISTER_PORT`] | RLM guest → host | [`SisterRequest`] / [`SisterAnswer`]: "run this artefact in a sister guest" | //! | [`MINER_PORT`] | host → miner guest | [`HostToMiner`] / [`MinerToHost`]: the run itself | //! //! The RLM guest never talks to the miner guest; the host relays the @@ -146,6 +146,23 @@ pub struct SisterRequest { pub params: BTreeMap, } +/// Host → RLM guest on [`SISTER_PORT`]: the answer to a [`SisterRequest`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SisterAnswer { + /// The sister booted, ran, and was destroyed. + Result { + /// What happened. + result: SisterResult, + }, + /// The host refused to boot a sister (bad bind, digest mismatch, no + /// image, a second sister in the same job, host not ready). + Refused { + /// Why (never a secret). + error: String, + }, +} + /// Host → RLM guest: what happened in the sister. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SisterResult { From eca33cc106d4d9023606b8b9feec583d4de08ef4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 16:53:17 +0000 Subject: [PATCH 03/12] feat(proof-vm-orchestrator): kvm-host agent binary (https, bearer file) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thin main over proof-vm-agent + proof-fc-host: clap flags with PROOF_VM_AGENT_* env twins for the systemd EnvironmentFile, kernel and sister image pins required at boot (malformed = exit 1, never invented), egress allowlist entries CIDR[:port[/tcp|udp]], owner key dir staged over vsock only. TLS via axum-server (rustls) from operator cert + key; a non-loopback bind without them exits 1 so the bearer never crosses a network in clear; plain http only on loopback. A missing token file does not stop the process — every request is refused until it exists (re-read per request, rotation without restart). Graceful shutdown on ctrl-c. Co-authored-by: Mathis --- Cargo.lock | 56 +++++ bins/proof-vm-orchestrator/Cargo.toml | 27 +++ bins/proof-vm-orchestrator/src/main.rs | 324 +++++++++++++++++++++++++ crates/proof-fc-host/src/sister.rs | 2 +- 4 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 bins/proof-vm-orchestrator/Cargo.toml create mode 100644 bins/proof-vm-orchestrator/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index ff8b1bd7a..c600a713a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "ark-bls12-377" version = "0.4.0" @@ -463,6 +472,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-server" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" +dependencies = [ + "arc-swap", + "bytes", + "either", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -1737,6 +1768,16 @@ dependencies = [ "serde", ] +[[package]] +name = "fs-err" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3697,6 +3738,21 @@ dependencies = [ "url", ] +[[package]] +name = "proof-vm-orchestrator-bin" +version = "0.1.0" +dependencies = [ + "axum", + "axum-server", + "clap", + "proof-fc-host", + "proof-vm-agent", + "proof-vm-proto", + "telemetry", + "tokio", + "tracing", +] + [[package]] name = "proof-vm-proto" version = "0.1.0" diff --git a/bins/proof-vm-orchestrator/Cargo.toml b/bins/proof-vm-orchestrator/Cargo.toml new file mode 100644 index 000000000..8bb8918bd --- /dev/null +++ b/bins/proof-vm-orchestrator/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "proof-vm-orchestrator-bin" +description = "proof-vm-orchestrator: the Firecracker topic-VM agent for the dedicated KVM host (HTTPS :8200, bearer from a file). Never runs on the control-plane droplet." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "proof-vm-orchestrator" +path = "src/main.rs" + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +axum-server = { version = "0.8", features = ["tls-rustls"] } +clap = { version = "4", features = ["derive", "env"] } +proof-fc-host = { path = "../../crates/proof-fc-host" } +proof-vm-agent = { path = "../../crates/proof-vm-agent" } +proof-vm-proto = { path = "../../crates/proof-vm-proto" } +telemetry = { path = "../../crates/telemetry" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } +tracing = "0.1" + +[lints] +workspace = true diff --git a/bins/proof-vm-orchestrator/src/main.rs b/bins/proof-vm-orchestrator/src/main.rs new file mode 100644 index 000000000..6a36f6b86 --- /dev/null +++ b/bins/proof-vm-orchestrator/src/main.rs @@ -0,0 +1,324 @@ +//! `proof-vm-orchestrator` — Firecracker topic-VM agent for the **dedicated +//! KVM host** (HTTPS `:8200`). +//! +//! The Proof control plane (`proof-challenge`, `FirecrackerOrchestrator`) +//! is its only client. It boots one jailed RLM microVM per topic from the +//! digest the control plane pins, runs every miner artefact in a sister +//! microVM with no network, and stamps what it saw onto the report. It +//! never runs on the control-plane droplet, never on a Lium pod, and never +//! receives a key from the control plane — owner key material is read from +//! `--owner-key-dir` on this host and staged over vsock. +//! +//! Fail-closed at boot: malformed kernel / sister image pins exit 1, a +//! non-loopback bind without a TLS certificate + key exits 1. A missing bearer +//! file does not stop the process — every request is refused until it exists +//! (the file is re-read per request, so rotation needs no restart). + +#![forbid(unsafe_code)] + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; +use std::time::Duration; + +use axum_server::tls_rustls::RustlsConfig; +use clap::Parser; +use proof_fc_host::{EgressAllow, FirecrackerHypervisor, HostConfig}; +use proof_vm_agent::{agent_router, AgentState, BearerAuth, Hypervisor}; +use proof_vm_proto::DEFAULT_AGENT_PORT; + +/// KVM-host agent CLI. Every flag has a `PROOF_VM_AGENT_*` env twin for the +/// systemd `EnvironmentFile`. +#[derive(Debug, Parser)] +#[command( + name = "proof-vm-orchestrator", + about = "Firecracker topic-VM agent (KVM host, HTTPS :8200)" +)] +struct Cli { + /// Bind address. Non-loopback requires --tls-cert and --tls-key. + #[arg(long, env = "PROOF_VM_AGENT_BIND", default_value_t = SocketAddr::from(([0, 0, 0, 0], DEFAULT_AGENT_PORT)))] + bind: SocketAddr, + /// Bearer token file (re-read per request; never logged). + #[arg(long, env = "PROOF_VM_AGENT_TOKEN_FILE")] + token_file: PathBuf, + /// TLS certificate chain (PEM). + #[arg(long, env = "PROOF_VM_AGENT_TLS_CERT")] + tls_cert: Option, + /// TLS private key (PEM). + #[arg(long, env = "PROOF_VM_AGENT_TLS_KEY")] + tls_key: Option, + /// Statically linked firecracker binary. + #[arg( + long, + env = "PROOF_VM_AGENT_FIRECRACKER_BIN", + default_value = "/usr/local/bin/firecracker" + )] + firecracker_bin: PathBuf, + /// jailer binary (same release as firecracker). + #[arg( + long, + env = "PROOF_VM_AGENT_JAILER_BIN", + default_value = "/usr/local/bin/jailer" + )] + jailer_bin: PathBuf, + /// jailer --chroot-base-dir. + #[arg( + long, + env = "PROOF_VM_AGENT_CHROOT_BASE", + default_value = "/srv/jailer" + )] + chroot_base: PathBuf, + /// Root filesystem images named sha256-.ext4. + #[arg( + long, + env = "PROOF_VM_AGENT_IMAGE_DIR", + default_value = "/var/lib/proof-vm/images" + )] + image_dir: PathBuf, + /// Guest kernel (vmlinux). + #[arg( + long, + env = "PROOF_VM_AGENT_KERNEL", + default_value = "/var/lib/proof-vm/vmlinux" + )] + kernel: PathBuf, + /// sha256: pin of the kernel. Required; never invent one. + #[arg(long, env = "PROOF_VM_AGENT_KERNEL_DIGEST")] + kernel_digest: String, + /// sha256: pin of the miner (sister) guest rootfs in --image-dir. Required. + #[arg(long, env = "PROOF_VM_AGENT_SISTER_IMAGE_DIGEST")] + sister_image_digest: String, + /// uid the jailer drops Firecracker to. + #[arg(long, env = "PROOF_VM_AGENT_JAIL_UID", default_value_t = 65534)] + jail_uid: u32, + /// gid the jailer drops Firecracker to. + #[arg(long, env = "PROOF_VM_AGENT_JAIL_GID", default_value_t = 65534)] + jail_gid: u32, + /// RLM VM scratch drive (MiB). + #[arg(long, env = "PROOF_VM_AGENT_SCRATCH_MIB", default_value_t = proof_fc_host::config::DEFAULT_SCRATCH_MIB)] + scratch_mib: u32, + /// Sister guest vCPUs (host-sized; the RLM never picks). + #[arg(long, env = "PROOF_VM_AGENT_SISTER_VCPUS", default_value_t = proof_fc_host::config::DEFAULT_SISTER_VCPUS)] + sister_vcpus: u32, + /// Sister guest memory (MiB). + #[arg(long, env = "PROOF_VM_AGENT_SISTER_MEM_MIB", default_value_t = proof_fc_host::config::DEFAULT_SISTER_MEM_MIB)] + sister_mem_mib: u32, + /// Sister scratch drive (MiB). + #[arg(long, env = "PROOF_VM_AGENT_SISTER_SCRATCH_MIB", default_value_t = proof_fc_host::config::DEFAULT_SISTER_SCRATCH_MIB)] + sister_scratch_mib: u32, + /// Seconds a guest agent may take to come up. + #[arg(long, env = "PROOF_VM_AGENT_BOOT_TIMEOUT_SECS", default_value_t = 120)] + boot_timeout_secs: u64, + /// Seconds past a job's deadline before the host kills it. + #[arg(long, env = "PROOF_VM_AGENT_DEADLINE_GRACE_SECS", default_value_t = 30)] + deadline_grace_secs: u64, + /// Seconds for jobs with no deadline of their own. + #[arg( + long, + env = "PROOF_VM_AGENT_DEFAULT_JOB_TIMEOUT_SECS", + default_value_t = 3_600 + )] + default_job_timeout_secs: u64, + /// Directory of owner key files staged into the RLM VM over vsock. + #[arg(long, env = "PROOF_VM_AGENT_OWNER_KEY_DIR")] + owner_key_dir: Option, + /// Uplink interface RLM VMs are masqueraded through. + #[arg(long, env = "PROOF_VM_AGENT_UPLINK", default_value = "eth0")] + uplink: String, + /// First /30 of the host<->guest pool. + #[arg(long, env = "PROOF_VM_AGENT_NET_BASE", default_value = "172.16.0.0")] + net_base: std::net::Ipv4Addr, + /// Egress allowlist entries `CIDR[:port[/tcp|udp]]` (repeat or comma-separate). + /// Empty = RLM VMs get no egress. + #[arg(long, env = "PROOF_VM_AGENT_EGRESS_ALLOW", value_delimiter = ',')] + egress_allow: Vec, + /// Where retained jails are moved. + #[arg( + long, + env = "PROOF_VM_AGENT_RETAIN_DIR", + default_value = "/var/lib/proof-vm/retained" + )] + retain_dir: PathBuf, +} + +fn host_config(cli: &Cli) -> Result { + let mut allow = Vec::new(); + for raw in cli + .egress_allow + .iter() + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + { + allow.push(EgressAllow::parse(raw).map_err(|e| e.to_string())?); + } + let cfg = HostConfig { + firecracker_bin: cli.firecracker_bin.clone(), + jailer_bin: cli.jailer_bin.clone(), + chroot_base: cli.chroot_base.clone(), + image_dir: cli.image_dir.clone(), + kernel: cli.kernel.clone(), + kernel_digest: cli.kernel_digest.trim().to_owned(), + sister_image_digest: cli.sister_image_digest.trim().to_owned(), + jail_uid: cli.jail_uid, + jail_gid: cli.jail_gid, + scratch_mib: cli.scratch_mib, + sister_vcpus: cli.sister_vcpus, + sister_mem_mib: cli.sister_mem_mib, + sister_scratch_mib: cli.sister_scratch_mib, + boot_timeout: Duration::from_secs(cli.boot_timeout_secs), + deadline_grace: Duration::from_secs(cli.deadline_grace_secs), + default_job_timeout: Duration::from_secs(cli.default_job_timeout_secs), + owner_key_dir: cli.owner_key_dir.clone(), + uplink: cli.uplink.trim().to_owned(), + net_base: cli.net_base, + egress_allow: allow, + retain_dir: cli.retain_dir.clone(), + }; + cfg.validate().map_err(|e| e.to_string())?; + Ok(cfg) +} + +/// TLS is mandatory off loopback: the bearer must never cross a network in clear. +fn tls_required( + bind: SocketAddr, + cert: Option<&PathBuf>, + key: Option<&PathBuf>, +) -> Result, String> { + match (cert, key) { + (Some(c), Some(k)) => Ok(Some((c.clone(), k.clone()))), + (None, None) if bind.ip().is_loopback() => Ok(None), + (None, None) => Err(format!( + "bind {bind} is not loopback: --tls-cert and --tls-key are required (the bearer never travels in clear)" + )), + _ => Err("--tls-cert and --tls-key go together".into()), + } +} + +fn main() -> ExitCode { + let _ = telemetry::init_tracing(); + let cli = Cli::parse(); + let rt = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + tracing::error!("runtime: {e}"); + return ExitCode::from(1); + } + }; + match rt.block_on(run(&cli)) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + ExitCode::from(1) + } + } +} + +async fn run(cli: &Cli) -> Result<(), String> { + let cfg = host_config(cli)?; + let tls = tls_required(cli.bind, cli.tls_cert.as_ref(), cli.tls_key.as_ref())?; + let hypervisor = Arc::new(FirecrackerHypervisor::new(cfg).map_err(|e| e.to_string())?); + match hypervisor.ready() { + Ok(()) => tracing::info!("firecracker + jailer + /dev/kvm present; agent ready"), + Err(e) => tracing::warn!("{e}; every create will answer 503 until fixed"), + } + let auth = Arc::new(BearerAuth::from_file(&cli.token_file)); + if auth.configured() { + tracing::info!(token_file = %cli.token_file.display(), "bearer token file present (contents not logged)"); + } else { + tracing::warn!( + token_file = %cli.token_file.display(), + "bearer token file missing or empty; every request is refused until it exists" + ); + } + tracing::info!( + egress_allow = hypervisor.config().egress_allow.len(), + owner_key_dir = ?hypervisor.config().owner_key_dir, + sister_vcpus = hypervisor.config().sister_vcpus, + sister_mem_mib = hypervisor.config().sister_mem_mib, + "host config" + ); + let state = AgentState::new(hypervisor as Arc, auth); + let app = agent_router(state); + let handle = axum_server::Handle::new(); + let shutdown = handle.clone(); + tokio::spawn(async move { + let _ = tokio::signal::ctrl_c().await; + shutdown.graceful_shutdown(Some(Duration::from_secs(10))); + }); + if let Some((cert, key)) = tls { + let config = RustlsConfig::from_pem_file(&cert, &key) + .await + .map_err(|e| format!("tls {} / {}: {e}", cert.display(), key.display()))?; + tracing::info!(bind = %cli.bind, "proof-vm-orchestrator listening (https)"); + return axum_server::bind_rustls(cli.bind, config) + .handle(handle) + .serve(app.into_make_service()) + .await + .map_err(|e| e.to_string()); + } + tracing::warn!(bind = %cli.bind, "proof-vm-orchestrator listening in clear on loopback (tests / local TLS terminator only)"); + axum_server::bind(cli.bind) + .handle(handle) + .serve(app.into_make_service()) + .await + .map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cli(extra: &[&str]) -> Cli { + let mut argv = vec![ + "proof-vm-orchestrator", + "--token-file", + "/etc/proof-vm/token", + "--kernel-digest", + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--sister-image-digest", + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ]; + argv.extend_from_slice(extra); + Cli::try_parse_from(argv).expect("cli") + } + + #[test] + fn config_needs_real_pins_and_parses_the_allowlist() { + let c = cli(&["--egress-allow", "203.0.113.10/32:443,1.1.1.1:53/udp"]); + let cfg = host_config(&c).expect("config"); + assert_eq!(cfg.egress_allow.len(), 2); + assert_eq!(cfg.sister_vcpus, 2); + assert_eq!(cfg.sister_mem_mib, 4_096); + assert_eq!(cfg.boot_timeout, Duration::from_mins(2)); + let bad = cli(&["--egress-allow", "not-an-address"]); + assert!(host_config(&bad).is_err()); + let mut invented = cli(&[]); + invented.kernel_digest = "latest".into(); + assert!(host_config(&invented).is_err(), "no invented pins"); + assert!( + Cli::try_parse_from(["proof-vm-orchestrator"]).is_err(), + "pins and token file are required" + ); + } + + #[test] + fn tls_is_required_off_loopback() { + let public: SocketAddr = "0.0.0.0:8200".parse().expect("addr"); + let local: SocketAddr = "127.0.0.1:8200".parse().expect("addr"); + assert!(tls_required(public, None, None).is_err()); + assert!(tls_required(local, None, None) + .expect("loopback clear") + .is_none()); + let cert = PathBuf::from("/etc/proof-vm/tls.crt"); + let key = PathBuf::from("/etc/proof-vm/tls.key"); + assert!(tls_required(public, Some(&cert), Some(&key)) + .expect("tls") + .is_some()); + assert!(tls_required(public, Some(&cert), None).is_err()); + assert_eq!(cli(&[]).bind, public); + } +} diff --git a/crates/proof-fc-host/src/sister.rs b/crates/proof-fc-host/src/sister.rs index a89af0d5b..a768abeb4 100644 --- a/crates/proof-fc-host/src/sister.rs +++ b/crates/proof-fc-host/src/sister.rs @@ -120,7 +120,7 @@ pub async fn run( let mut ch = GuestChannel::connect_within(&root, MINER_PORT, cfg.boot_timeout).await?; match ch.recv_within::(cfg.boot_timeout).await? { MinerToHost::Ready { api_version, .. } => { - check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))? + check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; } other => { return Err(HvError::Guest(format!( From edd860d2d262cba2e7eccff8f28ed1b49952cc99 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 16:57:20 +0000 Subject: [PATCH 04/12] feat(proof-challenge): prefer the firecracker orchestrator when configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The custom-family runner registry is no longer hard-wired empty. The host resolves its topic-vm orchestrator once at boot: PROOF_VM_ORCHESTRATOR_URL + PROOF_VM_ORCHESTRATOR_TOKEN_FILE + PROOF_RLM_VM_IMAGE_DIGEST select the live FirecrackerOrchestrator (4 vCPU / 8192 MiB RLM VM by default); URL unset, or set but refused (plain http off loopback, no token-file env), keeps UnwiredVmOrchestrator with the reason logged. Token and digest are checked at ready(), so a missing bearer file or an unpinned image is a 503 naming the env var, fixable without a restart, never a boot error and never a host fallback. The generic VmBackedRunner is registered under exactly the custom ids the operator lists in PROOF_VM_RUNNER_CUSTOM_IDS (comma-separated; malformed ids skipped with a warning). No ids → empty registry → every custom topic 503 (registration stays an operator action, no runner is compiled in). Tests cover the unset / half-configured / fully configured paths and the registry. Co-authored-by: Mathis --- Cargo.lock | 1 + bins/proof-challenge/Cargo.toml | 1 + bins/proof-challenge/src/main.rs | 200 +++++++++++++++++++++++++++++-- 3 files changed, 193 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c600a713a..fa6402413 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3502,6 +3502,7 @@ dependencies = [ "proof-rlm-scorer", "proof-rlm-store", "proof-task", + "proof-vm-fc", "reqwest 0.12.28", "serde_json", "sha2 0.10.9", diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index ff00fbb71..dc5c45a18 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -26,6 +26,7 @@ proof-rlm = { path = "../../crates/proof-rlm" } proof-rlm-scorer = { path = "../../crates/proof-rlm-scorer" } proof-rlm-store = { path = "../../crates/proof-rlm-store" } proof-task = { path = "../../crates/proof-task" } +proof-vm-fc = { path = "../../crates/proof-vm-fc" } serde_json = "1" telemetry = { path = "../../crates/telemetry" } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 7e18d296e..e6141dfff 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -25,9 +25,13 @@ use proof_challenge::{ }; use proof_eval::{custom_ids_ref, registered_custom, FamilyMux}; use proof_harvest::{HarvestLimits, LiumProofHarvest}; -use proof_rlm::RunnerRegistry; +use proof_rlm::{ + RunnerRegistry, TopicVmOrchestrator, UnwiredVmOrchestrator, VmBackedRunner, VmTemplate, + RLM_VM_IMAGE_DIGEST_ENV, VM_ORCHESTRATOR_TOKEN_FILE_ENV, VM_ORCHESTRATOR_URL_ENV, +}; use proof_rlm_scorer::{ArtefactStore, RlmScorer}; use proof_rlm_store::{MemoryRlmStore, PgRlmStore, RlmStore}; +use proof_vm_fc::{parse_custom_ids, FirecrackerOrchestrator, VM_RUNNER_CUSTOM_IDS_ENV}; use tokio::net::TcpListener; /// Operator Proof challenge service CLI. @@ -274,20 +278,93 @@ fn build_live_scorer( /// Route the `custom` metric family to the RLM scorer over the default harvest. /// -/// The runner registry starts **empty**: no benchmark, model, or repository is -/// compiled in, so every custom topic answers 503 (`RunnerUnwired`) until an -/// operator or the topic's RLM registers a runner under its `custom_id`. It -/// never falls back to the digest-pinned harvest and never spends. +/// No benchmark, model, or repository is compiled in: the registry holds only +/// the generic `VmBackedRunner`, under the custom ids the operator lists in +/// `PROOF_VM_RUNNER_CUSTOM_IDS`, over the topic-VM orchestrator +/// [`topic_vm_orchestrator`] resolved. With no ids the registry is empty and +/// every custom topic answers 503 (`RunnerUnwired`); with ids but an unwired +/// or unpinned orchestrator, 503 naming the missing env var. It never falls +/// back to the digest-pinned harvest and never spends. fn with_custom_family( harvest: Arc, rlm_store: Arc, artefact_root: &Path, ) -> Arc { - let scorer = RlmScorer::new(Arc::new(RunnerRegistry::new()), rlm_store) + let scorer = RlmScorer::new(Arc::new(runner_registry()), rlm_store) .with_artefacts(Some(ArtefactStore::new(artefact_root))); Arc::new(FamilyMux::new(harvest).with_custom_family(Arc::new(scorer))) } +/// The topic-VM orchestrator this host talks to, plus the RLM VM template. +/// +/// `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` + +/// `PROOF_RLM_VM_IMAGE_DIGEST` select the live `FirecrackerOrchestrator` +/// (HTTPS to the agent on the dedicated KVM host, 4 vCPU / 8192 MiB by +/// default). URL unset → `UnwiredVmOrchestrator` (503, names the env vars). +/// URL set but malformed (not https, no token file env) → also unwired, with +/// the error logged: a half-configured orchestrator never becomes a host +/// fallback. Token / digest are checked at `ready()` so they can be fixed +/// without a restart. +fn topic_vm_orchestrator() -> (Arc, VmTemplate) { + match FirecrackerOrchestrator::from_env() { + Ok(Some(fc)) => { + let template = fc.template().clone(); + match fc.ready() { + Ok(()) => tracing::info!( + url = %fc.url(), vcpus = template.vcpus, mem_mib = template.mem_mib, + image = %template.image_digest, + "firecracker topic-vm orchestrator wired (bearer file present, contents not logged)" + ), + Err(e) => tracing::warn!( + url = %fc.url(), + "firecracker topic-vm orchestrator configured but not ready ({e}); custom \ + topics answer 503 until fixed" + ), + } + (Arc::new(fc), template) + } + Ok(None) => { + tracing::warn!( + "no topic-vm orchestrator ({VM_ORCHESTRATOR_URL_ENV} / \ + {VM_ORCHESTRATOR_TOKEN_FILE_ENV} / {RLM_VM_IMAGE_DIGEST_ENV} unset); every custom \ + topic answers 503 and nothing runs on this host" + ); + (Arc::new(UnwiredVmOrchestrator), VmTemplate::from_env()) + } + Err(e) => { + tracing::warn!( + "topic-vm orchestrator refused ({e}); staying unwired, custom topics 503" + ); + (Arc::new(UnwiredVmOrchestrator), VmTemplate::from_env()) + } + } +} + +/// `custom_id → VmBackedRunner` for every id in `PROOF_VM_RUNNER_CUSTOM_IDS`. +fn runner_registry() -> RunnerRegistry { + let (orchestrator, template) = topic_vm_orchestrator(); + let runner = Arc::new(VmBackedRunner::new(orchestrator, template)); + let raw = std::env::var(VM_RUNNER_CUSTOM_IDS_ENV).unwrap_or_default(); + registry_for(&raw, &runner) +} + +fn registry_for(raw_ids: &str, runner: &Arc) -> RunnerRegistry { + let mut registry = RunnerRegistry::new(); + for id in parse_custom_ids(raw_ids) { + match registry.register(&id, runner.clone()) { + Ok(()) => tracing::info!(custom_id = %id, "vm-backed runner registered"), + Err(e) => tracing::warn!("{VM_RUNNER_CUSTOM_IDS_ENV}: {e}; skipped"), + } + } + if registry.is_empty() { + tracing::warn!( + "{VM_RUNNER_CUSTOM_IDS_ENV} names no custom id; the runner registry is empty and \ + every custom topic answers 503 (registration is an operator action)" + ); + } + registry +} + fn database_url(cli: &Cli) -> Result, String> { if let Some(url) = cli.database_url.as_deref().map(str::trim) { if !url.is_empty() { @@ -586,14 +663,16 @@ mod tests { std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); } - /// No runner is compiled in: every custom id refuses through the mux - /// (`RunnerUnwired`, the 503 root cause), the harvest still owns the - /// nll / throughput route, and nothing is registered. + /// No runner is compiled in: with no orchestrator env and no listed ids, + /// every custom id refuses through the mux (`RunnerUnwired`, the 503 root + /// cause), the harvest still owns the nll / throughput route, and nothing + /// is registered. #[test] fn live_scorer_registers_no_custom_runner_and_refuses_every_custom_id() { let _guard = LIUM_ENV .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + clear_vm_env(); let pubkey = stub_ssh_pubkey("proof-families"); std::env::set_var("LIUM_API_KEY", "test-key-not-a-real-secret"); std::env::set_var("LIUM_SSH_PUBLIC_KEY_FILE", &pubkey); @@ -679,6 +758,109 @@ mod tests { path } + /// Every env var the topic-vm wiring reads. Cleared under `LIUM_ENV`. + fn clear_vm_env() { + for name in [ + VM_ORCHESTRATOR_URL_ENV, + VM_ORCHESTRATOR_TOKEN_FILE_ENV, + RLM_VM_IMAGE_DIGEST_ENV, + VM_RUNNER_CUSTOM_IDS_ENV, + proof_vm_fc::RLM_VM_VCPUS_ENV, + proof_vm_fc::RLM_VM_MEM_MIB_ENV, + proof_vm_fc::VM_ORCHESTRATOR_CA_FILE_ENV, + ] { + std::env::remove_var(name); + } + } + + /// Registration is an operator action: no ids → empty registry; listed + /// ids bind the one generic `VmBackedRunner`; a malformed id is skipped, + /// never a boot error. + #[test] + fn runner_registry_binds_the_vm_runner_only_to_listed_custom_ids() { + let runner = Arc::new(VmBackedRunner::unwired()); + assert!(registry_for("", &runner).is_empty()); + assert!(registry_for(" , ", &runner).is_empty()); + let reg = registry_for("metric_a, Bad Id ,metric-b,metric_a", &runner); + assert_eq!( + reg.ids(), + vec!["metric-b".to_owned(), "metric_a".to_owned()] + ); + let resolved = reg.resolve("metric_a").expect("registered"); + let err = resolved.ready().expect_err("unwired orchestrator"); + assert!(matches!(err, proof_rlm::RunnerError::NotWired(_)), "{err}"); + assert!(err.to_string().contains(VM_ORCHESTRATOR_URL_ENV), "{err}"); + assert!( + reg.resolve("metric_c").is_err(), + "unlisted ids stay unregistered" + ); + } + + /// URL + token file + digest select the live `FirecrackerOrchestrator` + /// with the locked 4 vCPU / 8192 MiB shape; anything less keeps the + /// unwired orchestrator (503), including a half-configured plain-http URL. + #[test] + fn firecracker_orchestrator_is_preferred_only_when_fully_configured() { + let _guard = LIUM_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + clear_vm_env(); + let (unwired, template) = topic_vm_orchestrator(); + assert!(matches!( + unwired.ready(), + Err(proof_rlm::VmError::NotWired(_)) + )); + assert!(template.image_digest.is_empty()); + + let dir = std::env::temp_dir().join(format!("proof-vm-wire-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let token = dir.join("vm_orchestrator_token"); + std::fs::write(&token, "vm-bearer-not-a-real-secret\n").expect("token"); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "https://kvm.example.invalid:8200"); + std::env::set_var(VM_ORCHESTRATOR_TOKEN_FILE_ENV, &token); + let (pinned_less, template) = topic_vm_orchestrator(); + let err = pinned_less.ready().expect_err("no image pin"); + assert!(matches!(err, proof_rlm::VmError::NotWired(_)), "{err}"); + assert!(err.to_string().contains(RLM_VM_IMAGE_DIGEST_ENV), "{err}"); + assert_eq!( + (template.vcpus, template.mem_mib), + (4, 8_192), + "locked shape" + ); + + std::env::set_var( + RLM_VM_IMAGE_DIGEST_ENV, + format!("sha256:{}", "ab".repeat(32)), + ); + let (live, template) = topic_vm_orchestrator(); + live.ready().expect("url + token + digest = wired"); + template.validate().expect("pinned"); + std::env::set_var(VM_RUNNER_CUSTOM_IDS_ENV, "metric_a"); + let reg = runner_registry(); + assert_eq!(reg.ids(), vec!["metric_a".to_owned()]); + reg.resolve("metric_a") + .expect("registered") + .ready() + .expect("runner over the live orchestrator is ready"); + + std::fs::write(&token, "\n").expect("empty token"); + let (live, _) = topic_vm_orchestrator(); + let err = live.ready().expect_err("empty bearer file"); + assert!( + err.to_string().contains(VM_ORCHESTRATOR_TOKEN_FILE_ENV), + "{err}" + ); + + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "http://10.0.0.7:8200"); + let (refused, _) = topic_vm_orchestrator(); + let err = refused + .ready() + .expect_err("plain http off loopback is never wired"); + assert!(err.to_string().contains(VM_ORCHESTRATOR_URL_ENV), "{err}"); + clear_vm_env(); + let _ = std::fs::remove_dir_all(&dir); + } + static LIUM_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Compose always points `PROOF_PIN_FILE` at the committed pin. Empty From f084aeebc693d2890d3586e83a32962b26cd1049 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 17:03:46 +0000 Subject: [PATCH 05/12] docs(proof-vm): kvm-host agent unit, env example, runbook, boundary docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy: systemd unit for proof-vm-orchestrator (dedicated KVM host only — ConditionPathExists=/dev/kvm, hardened, NET_ADMIN for TAP + nftables), env example with every PROOF_VM_AGENT_* knob (pins left empty: computed from staged files, never invented), proof-challenge env block for the client side (URL / token file / CA / image digest / locked 4 vCPU 8192 MiB / runner ids), secrets README rows for the token, CA, and presence-only owner key. docs: runbook (what runs where, locked rules table, host prerequisites and layout, install, CP wiring, mandatory end-to-end submission verification with the 503 probes, operate, security model, v1 limitations); PROOF.md isolation boundary now describes both orchestrators, the sister guest, and host stamping; COMPLETENESS row for the orchestrator (implemented / operator-gated, no digest pinned yet); ARCHITECTURE topology + binary row; AGENTS.md key-roles row for the bearer file and verification item 7 (topic VMs, zero live Firecracker in CI); deploy/AGENTS.md section; miner page: your code runs offline in a sister guest, the host stamps sandboxed and flops_used. Co-authored-by: Mathis --- AGENTS.md | 4 +- deploy/AGENTS.md | 19 ++ deploy/env/proof-challenge.env.example | 35 +++- deploy/env/proof-vm-orchestrator.env.example | 71 +++++++ deploy/secrets/README.md | 3 + deploy/systemd/proof-vm-orchestrator.service | 46 +++++ docs/AGENTS.md | 1 + docs/ARCHITECTURE.md | 6 +- docs/COMPLETENESS.md | 3 +- docs/PROOF.md | 56 ++++-- docs/external-miner/proof.md | 18 +- docs/runbooks/proof-vm-orchestrator.md | 184 +++++++++++++++++++ 12 files changed, 418 insertions(+), 28 deletions(-) create mode 100644 deploy/env/proof-vm-orchestrator.env.example create mode 100644 deploy/systemd/proof-vm-orchestrator.service create mode 100644 docs/runbooks/proof-vm-orchestrator.md diff --git a/AGENTS.md b/AGENTS.md index 926036d7e..7236cfa94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,7 @@ Working branch: **`main`**. Prod ships from annotated tags `v*.*.*` cut on `main | `gateway_admin_token` | Gateway + seal scripts | Bearer for **`/v1/admin/*`** (seal, backends, attest-grant). **Required** when `BASE_GATEWAY_REQUIRE_OWNER=1` | | `bounty_sk` | Bounty / smoke | Signed bounty leaves; pub must match trust root | | `proof_sk` | Proof / smoke | Signed `proof` leaves and topic documents; pub must match trust root | +| `vm_orchestrator_token` (`PROOF_VM_ORCHESTRATOR_TOKEN_FILE` ↔ `PROOF_VM_AGENT_TOKEN_FILE`) | Proof CP ↔ KVM-host agent | Bearer **file** for the topic-VM orchestrator (`proof-vm-orchestrator`, HTTPS). Re-read per request on both sides, never logged, never on `/v1/status`. Not a wallet | | Gateway owner wallet + `BASE_GATEWAY_REQUIRE_OWNER` | Gateway | Master-only **identity** check (live/prod). **Not** required to seal or serve `/v1/weights/latest` | | Validator wallet | Validator | On-chain weight **submit** only — validators *fetch* sealed weights; they do not need a gateway wallet | @@ -73,7 +74,8 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). +8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 71aa1ddbf..2800c9745 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -67,6 +67,25 @@ checks and an acknowledgement request, and the binary does not drive its leaf-signing helpers. Deployment configuration cannot fill these implementation gaps. See [`docs/WHITEPAPER.md`](../docs/WHITEPAPER.md). +## Proof topic VMs (Firecracker on a dedicated KVM host) + +Custom-family topics run their RLM in one Firecracker microVM per `topic_id` +and every miner run in a **sister** Firecracker guest with no network — on a +**dedicated KVM host**, not on any droplet (DO has no nested virt), not on +Lium. That host runs `proof-vm-orchestrator` as a systemd unit +([`systemd/proof-vm-orchestrator.service`](systemd/proof-vm-orchestrator.service), +env [`env/proof-vm-orchestrator.env.example`](env/proof-vm-orchestrator.env.example)), +**not** a compose service. The master's `proof-challenge` is only its HTTPS +client: set `PROOF_VM_ORCHESTRATOR_URL`, `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` +(bearer **file** under `deploy/secrets/proof/`, same bytes as the host's +`/etc/proof-vm/token`, mode 0400), `PROOF_RLM_VM_IMAGE_DIGEST`, and +`PROOF_VM_RUNNER_CUSTOM_IDS` in `deploy/env/proof-challenge.env`. Unset → +unwired (503); token missing, digest unpinned, or agent down → 503, never a +host-local fallback. Kernel / RLM / sister image digests are computed from the +files the operator stages (`sha256sum`) — never invented, never in git. +Procedure and the mandatory submission verification: +[`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). + ## Local testnet E2E Full procedure: [`docs/runbooks/local-testnet-e2e.md`](../docs/runbooks/local-testnet-e2e.md). diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 6cd11dd78..ff055f8f5 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -103,15 +103,34 @@ PROOF_SIM_STUB_WIN=false # persistent proof-artifacts volume. # PROOF_ARTEFACT_ROOT=/var/lib/proof/artefacts # -# Topic-VM orchestrator (the RLM runs inside a VM per topic; miner code in a -# Firecracker guest under it). Not implemented in this repo yet: with these -# unset every custom topic answers 503 (runner not wired / not registered) -# and nothing rents or spends. Names only — never a value in git, never -# logged, never on /v1/status. -# PROOF_VM_ORCHESTRATOR_URL= +# Topic-VM orchestrator: the RLM runs inside one Firecracker microVM per +# topic on a DEDICATED KVM HOST (never this droplet, never a Lium pod), and +# every miner run happens in a SISTER Firecracker guest with no network. This +# host is only the client (`FirecrackerOrchestrator`, crates/proof-vm-fc); +# the agent is `proof-vm-orchestrator` (deploy/systemd/, runbook +# docs/runbooks/proof-vm-orchestrator.md). Fail-closed: URL unset → unwired +# (503, names the vars); URL set but not https:// → refused, stays unwired; +# token file missing/empty or image digest unpinned → 503 naming the var +# (fixable without a restart). No host-local execution path exists. +# Names only — never a value in git, never logged, never on /v1/status. +# PROOF_VM_ORCHESTRATOR_URL=https://kvm-host.example.invalid:8200 +# Bearer FILE (mode 0400) matching the agent's PROOF_VM_AGENT_TOKEN_FILE. # PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token -# sha256: digest of the RLM VM image the orchestrator boots (unpinned = never boots). +# Optional PEM root when the agent's certificate chains to a private CA. +# PROOF_VM_ORCHESTRATOR_CA_FILE=/run/base/proof/vm_orchestrator_ca.pem +# sha256: digest of the RLM VM rootfs the agent boots (must be staged on the +# KVM host as images/sha256-.ext4). Unpinned = nothing ever boots. +# Take it from the image you built; DO NOT INVENT ONE. # PROOF_RLM_VM_IMAGE_DIGEST= +# RLM VM shape. Locked defaults 4 vCPU / 8192 MiB; override only on purpose. +# PROOF_RLM_VM_VCPUS=4 +# PROOF_RLM_VM_MEM_MIB=8192 +# Custom metric ids the generic VmBackedRunner serves over the orchestrator +# (comma-separated). Registration is an operator action: unset = empty +# registry = every custom topic 503. Ids are topic data, never a code list. +# PROOF_VM_RUNNER_CUSTOM_IDS= # Owner paid-inference key file probed (presence only) at awaiting_owner_keys -# before the baseline run; staged into the topic VM, never read by this host. +# before the baseline run. The MATERIAL is staged into the topic VM by the +# KVM-host agent from its own PROOF_VM_AGENT_OWNER_KEY_DIR; this host never +# reads or sends it. # PROOF_RLM_OWNER_INFERENCE_KEY_FILE=/run/base/proof/rlm_owner_inference_key diff --git a/deploy/env/proof-vm-orchestrator.env.example b/deploy/env/proof-vm-orchestrator.env.example new file mode 100644 index 000000000..59b5080d4 --- /dev/null +++ b/deploy/env/proof-vm-orchestrator.env.example @@ -0,0 +1,71 @@ +# operator-managed, never committed to git. +# Environment for /etc/proof-vm/orchestrator.env on the DEDICATED KVM HOST +# (deploy/systemd/proof-vm-orchestrator.service). Not a compose env file: +# the agent never runs on the control-plane droplet or on a Lium pod. +# +# Nothing in this file is a secret value. The bearer is a FILE the agent +# re-reads on every request (rotate by rewriting it, no restart). Owner key +# material is a DIRECTORY the agent stages into the RLM VM over vsock; the +# control plane only ever probes its own copy for presence. + +# HTTPS listener the control plane's PROOF_VM_ORCHESTRATOR_URL points at. +# A non-loopback bind without TLS cert + key refuses to start. +PROOF_VM_AGENT_BIND=0.0.0.0:8200 +PROOF_VM_AGENT_TLS_CERT=/etc/proof-vm/tls.crt +PROOF_VM_AGENT_TLS_KEY=/etc/proof-vm/tls.key + +# Bearer token file (mode 0400, root). The same token goes into the control +# plane's PROOF_VM_ORCHESTRATOR_TOKEN_FILE. Missing/empty = every request 401. +PROOF_VM_AGENT_TOKEN_FILE=/etc/proof-vm/token + +# Firecracker + jailer of the SAME release, statically linked (musl). +PROOF_VM_AGENT_FIRECRACKER_BIN=/usr/local/bin/firecracker +PROOF_VM_AGENT_JAILER_BIN=/usr/local/bin/jailer +PROOF_VM_AGENT_CHROOT_BASE=/srv/jailer + +# Root filesystem images, one file per pin: sha256-<64 hex>.ext4. The RLM VM +# image digest is chosen by the control plane (PROOF_RLM_VM_IMAGE_DIGEST) and +# must be present here; the agent re-hashes it before every first boot. +PROOF_VM_AGENT_IMAGE_DIR=/var/lib/proof-vm/images + +# Guest kernel and its pin. REQUIRED. Take the digest from the kernel you +# actually staged (sha256sum); never invent one. +PROOF_VM_AGENT_KERNEL=/var/lib/proof-vm/vmlinux +# PROOF_VM_AGENT_KERNEL_DIGEST=sha256: + +# Miner (sister) guest rootfs pin, present in PROOF_VM_AGENT_IMAGE_DIR. +# REQUIRED. The sister has no network; it speaks the vsock miner protocol +# (proof-vm-proto::guest) and reports flops_used. Never invent one. +# PROOF_VM_AGENT_SISTER_IMAGE_DIGEST=sha256: + +# uid/gid the jailer drops Firecracker to (nobody/nogroup by default). +PROOF_VM_AGENT_JAIL_UID=65534 +PROOF_VM_AGENT_JAIL_GID=65534 + +# Sizes. The RLM VM shape (4 vCPU / 8192 MiB) is set by the control plane; +# the sister is sized HERE, never by the RLM. +PROOF_VM_AGENT_SCRATCH_MIB=8192 +PROOF_VM_AGENT_SISTER_VCPUS=2 +PROOF_VM_AGENT_SISTER_MEM_MIB=4096 +PROOF_VM_AGENT_SISTER_SCRATCH_MIB=2048 + +# Timeouts (seconds). A run is held to its topic deadline inside the guest; +# the host kills it deadline + grace later as a backstop. +PROOF_VM_AGENT_BOOT_TIMEOUT_SECS=120 +PROOF_VM_AGENT_DEADLINE_GRACE_SECS=30 +PROOF_VM_AGENT_DEFAULT_JOB_TIMEOUT_SECS=3600 + +# Owner paid-inference key material staged into the RLM VM over vsock (files +# in this directory, by name). Optional. Mode 0400, root. Never in git. +# PROOF_VM_AGENT_OWNER_KEY_DIR=/etc/proof-vm/owner-keys + +# Egress the RLM VM may reach, and nothing else: CIDR[:port[/tcp|udp]], +# comma-separated. Typically the RLM judge InferenceOffer origin, the +# artefact locator hosts miners use, and a resolver. EMPTY = NO EGRESS. +# The sister miner guest never has a network interface regardless. +PROOF_VM_AGENT_UPLINK=eth0 +PROOF_VM_AGENT_NET_BASE=172.16.0.0 +# PROOF_VM_AGENT_EGRESS_ALLOW=203.0.113.10/32:443,1.1.1.1/32:53/udp + +# Where `retain` teardowns move a topic's jail (scratch, console, config). +PROOF_VM_AGENT_RETAIN_DIR=/var/lib/proof-vm/retained diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 4b115dfd3..bd2a54a4c 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -39,6 +39,9 @@ chmod 0400 deploy/secrets/gateway_admin_token | `proof/inference_api_key` | proof-challenge | Provider API key for the eval image. **Never commit, never log.** Mode **0400**, uid **65532** | | `proof/inference_base_url` | proof-challenge | Optional secret-backed origin (`PROOF_INFERENCE_BASE_URL_FILE`) when pin `[inference].base_url` and the topic omit one. **Never commit, never log.** Mode **0400**, uid **65532** | | `proof/eval_executor_offer.json` | proof-challenge | Live `1x` `EvalExecutorOffer` (`offer_id`, `lium_template_id`, `machine_shape`, `max_proof_deadline_s`, `eval_image_digest`, `config_commitment`, status). Sibling of the judge offer, no secret inside; still operator state, **never commit**. Build with `cargo run -p xtask -- proof-executor-offer …`; rotate live via `POST /v1/admin/proof/executor`. Missing/closed/shape ≠ pin `gpu_class` → `can_score=false` / 503. Mode **0400**, uid **65532** | +| `proof/vm_orchestrator_token` | proof-challenge | Bearer for the topic-VM orchestrator agent on the dedicated KVM host (`PROOF_VM_ORCHESTRATOR_TOKEN_FILE`); same bytes as the host's `/etc/proof-vm/token`. Re-read per request (rotate by rewriting both). **Never commit, never log.** Missing/empty → custom topics 503. Mode **0400**, uid **65532** | +| `proof/vm_orchestrator_ca.pem` | proof-challenge | Optional PEM root the agent's TLS certificate chains to (`PROOF_VM_ORCHESTRATOR_CA_FILE`). Public material, still operator state. Mode **0400**, uid **65532** | +| `proof/rlm_owner_inference_key` | proof-challenge | Probed for **presence only** at `awaiting_owner_keys` (`PROOF_RLM_OWNER_INFERENCE_KEY_FILE`). The material the RLM uses is staged by the KVM-host agent from its own `PROOF_VM_AGENT_OWNER_KEY_DIR`; this host never reads or sends it. **Never commit, never log.** Mode **0400**, uid **65532** | | `bounty/admin_tokens` | bounty-challenge | Operator bearer for `POST /v1/admin/adjudicate` | | `bounty/session_secret` | bounty-challenge | Pairing session HMAC secret. Empty/missing no longer crashes boot (`/health` stays up; pairing will not survive restart). `remote-deploy.sh` fills a 32-byte value from urandom when the file is missing or 0-length. | diff --git a/deploy/systemd/proof-vm-orchestrator.service b/deploy/systemd/proof-vm-orchestrator.service new file mode 100644 index 000000000..a8e435e51 --- /dev/null +++ b/deploy/systemd/proof-vm-orchestrator.service @@ -0,0 +1,46 @@ +[Unit] +Description=Proof topic-VM orchestrator (Firecracker + jailer agent; dedicated KVM host only) +Documentation=https://github.com/CortexLM/cortex/blob/main/docs/runbooks/proof-vm-orchestrator.md +# Never install this unit on a control-plane droplet or a Lium pod: it needs +# /dev/kvm, a pinned kernel + rootfs images, and CAP_NET_ADMIN for the per-VM +# TAP + nftables allowlist. The control plane reaches it over HTTPS only. +After=network-online.target +Wants=network-online.target +ConditionPathExists=/dev/kvm +ConditionPathExists=/etc/proof-vm/orchestrator.env + +[Service] +Type=simple +# Pins (kernel + sister image digests), TLS paths, egress allowlist, token +# file path. No secret value lives in the env file: the bearer is a file the +# agent re-reads per request, owner key material is a directory it stages +# over vsock. See deploy/env/proof-vm-orchestrator.env.example. +EnvironmentFile=/etc/proof-vm/orchestrator.env +ExecStart=/usr/local/bin/proof-vm-orchestrator +Restart=on-failure +RestartSec=5 +TimeoutStopSec=60 +KillMode=control-group +# The jailer needs root to chroot / pivot_root / mknod and to drop to the jail +# uid; the agent itself needs NET_ADMIN for TAP + nftables. Everything else is +# locked down. +User=root +NoNewPrivileges=no +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ProtectKernelTunables=no +ProtectControlGroups=no +# Writable: jails, images (verified, never written by the agent but reflinked +# from), retained jails, console logs. +ReadWritePaths=/srv/jailer /var/lib/proof-vm /run +# Read-only: binaries, pins, TLS, bearer, owner keys. +ReadOnlyPaths=/etc/proof-vm /usr/local/bin/firecracker /usr/local/bin/jailer +CapabilityBoundingSet=CAP_SYS_ADMIN CAP_SYS_CHROOT CAP_MKNOD CAP_NET_ADMIN CAP_SETUID CAP_SETGID CAP_CHOWN CAP_DAC_OVERRIDE CAP_FOWNER CAP_KILL CAP_SYS_RESOURCE +AmbientCapabilities= +LimitNOFILE=65536 +# JSON tracing; never logs the bearer or owner key material. +Environment=RUST_LOG=info + +[Install] +WantedBy=multi-user.target diff --git a/docs/AGENTS.md b/docs/AGENTS.md index d1a435a59..ff7469431 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -20,6 +20,7 @@ When a spike or evidence report conflicts with a frozen spec or runbook, the nor | [`runbooks/local-testnet-e2e.md`](runbooks/local-testnet-e2e.md) | Local laptop/VM full subnet stack on testnet 541 + ephemeral gateway tunnel | | [`runbooks/staging-testnet-e2e.md`](runbooks/staging-testnet-e2e.md) | Staging droplet testnet end-to-end validation | | [`runbooks/proof-submit-e2e.md`](runbooks/proof-submit-e2e.md) | Proof (and Bounty) submit → score: cargo tests, local `--force-sim`, staging curl/ctx | +| [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | Proof topic VMs: Firecracker + jailer agent on the dedicated KVM host, CP wiring, sister-guest verification, security model | | [`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md) | Trust-root key rotation | | [`runbooks/gateway-failover.md`](runbooks/gateway-failover.md) | Gateway kill/restart / failover checks | | [`runbooks/measurement-repin-socket-proxy.md`](runbooks/measurement-repin-socket-proxy.md) | Socket-proxy measurement re-pin | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c3563b80c..29ad4223f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -46,7 +46,10 @@ Master host (role-master overlay + master profile) gateway · postgres · socket-proxy bounty-challenge · proof-challenge │ │ - │ └─ Proof harvest → Lium evaluation pod + │ ├─ Proof harvest → Lium evaluation pod (nll / throughput) + │ └─ HTTPS + bearer file → dedicated KVM host (custom topics) + │ proof-vm-orchestrator: jailer/Firecracker RLM VM per topic, + │ sister Firecracker guest (no network) per miner run │ signed bundles ▼ Validator host (role-validator overlay) @@ -66,6 +69,7 @@ terminates in the host reverse proxy, not in the gateway process. | `validator` | Fetch/mirror bundle, verify, recompute, peer cross-check, CRV4 submit, dissent | | `bounty-challenge` | **Master-only:** internal pair/reports/adjudicate; **reads** CortexLM/backend public API for scoring and signs leaves from those rows. An unreadable feed pays nobody — `E` is covered with `ChallengeInternal`, share burns to uid 0 — rather than scoring offline | | `proof-challenge` | **Master-only:** signed topics, holdout loading, evaluation orchestration. Library payout is a sum of WTA/discovery topic masses; the binary has no automatic leaf-emission loop yet | +| `proof-vm-orchestrator` | **Dedicated KVM host only** (never a droplet, never Lium): Firecracker + jailer agent behind HTTPS + a bearer file. One RLM microVM per Proof topic from the digest the control plane pins, sister miner guest with no network per paid run, host-stamped `sandboxed` / `flops_used`. Client side is `proof-vm-fc::FirecrackerOrchestrator`; runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | | `updater` | Digest-pinned rollouts via `docker-socket-proxy` (master) | | `trustroot` | Offline keygen / sign / verify for owner-signed TOML | | `bundle` | SCALE types, seal, verify (`PROTOCOL_VERSION`) | diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 34285cf95..2f095bf89 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -85,7 +85,8 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | -| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No live VM orchestrator, no registered runner, no challenge content:** every custom topic answers **503** until an operator / RLM registers a runner. | +| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used`, destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index 699f15841..cddc423a6 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -145,12 +145,15 @@ baseline + an open topic are on the host. database. Every rule of the current version is ticked with evidence **before any paid inference**; one red, missing, duplicated, or evidence-less item is a persisted reject with no spend. -- The RLM runs **inside a VM attributed to its topic**, reached only through - the orchestrator boundary (`TopicVmOrchestrator`). Miner code runs in a - Firecracker guest under that VM when the topic says - `constraints.firecracker_required`. The control plane never runs RLM - logic and never hands the VM a host path or a secret; an unwired - orchestrator is a **503**, not a host-local fallback. +- The RLM runs **inside a Firecracker microVM attributed to its topic** on a + dedicated KVM host, reached only through the orchestrator boundary + (`TopicVmOrchestrator` → `FirecrackerOrchestrator` → `proof-vm-orchestrator` + agent). Miner code runs in a **sister** Firecracker guest with no network + beside that VM; the host, not the RLM, stamps `sandboxed` and the + guest-measured `flops_used` on the report. The control plane never runs + RLM logic and never hands the VM a host path or a secret; an unwired + orchestrator, a missing bearer file, or an unpinned RLM image is a + **503**, not a host-local fallback. - `PROOF_FORCE_SIM` is CI/local opt-in only. Never a fallback. Forbidden on droplet overlays. Under sim, a sealed topic scores with harness numbers relative to the seal (`sim_win_document`); skill-only `sim_document` @@ -401,12 +404,41 @@ resumes from the persisted state. `TopicVmOrchestrator` (create / attach / run / teardown-or-retain) is the only way RLM work happens. `VmJob`s carry public data (signed topic, digests, -rule set, request) — never a host path, a key, or a judge origin. The shipped -orchestrator is `UnwiredVmOrchestrator` (refuses, names -`PROOF_VM_ORCHESTRATOR_URL` / `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`; -`PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM VM image). The generic -`VmBackedRunner` turns inspect / evaluate into VM jobs; registering it under -a `custom_id` is an operator / RLM action. The Lium harvest for +rule set, request) — never a host path, a key, or a judge origin. Two +orchestrators exist: `UnwiredVmOrchestrator` (the default; refuses, names +`PROOF_VM_ORCHESTRATOR_URL` / `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`) and the +live `FirecrackerOrchestrator` (`crates/proof-vm-fc`), a thin HTTPS client of +the `proof-vm-orchestrator` agent on a **dedicated KVM host** (never the +control-plane droplet, never a Lium pod, never nested). The host prefers it +when `PROOF_VM_ORCHESTRATOR_URL` (https; plain http only on loopback) and +`PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; the bearer is a file re-read per +request and never logged; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM VM rootfs +(4 vCPU / 8192 MiB by default; unpinned = `ready()` fails, 503 naming the +var). On the KVM host, jailer boots **one Firecracker RLM microVM per +`topic_id`** from that digest (re-hashed before boot), hands it jobs over +vsock, stages the owner key material from the host's own directory (the +control plane only probes its copy for presence), and gives it an nftables +egress allowlist (empty = no egress). Every paid run (`Baseline`, +`Evaluate`) that the RLM asks for happens in a **sister** Firecracker guest +with **no network**: the RLM ships the artefact bytes it already inspected +over vsock, the host boots the sister from its own pinned image, holds it to +the topic deadline, destroys it, and writes the `SisterAttestation`. The +agent then **stamps** the report: `sandboxed` is `true` only when a sister +ran, `flops_used` is the sister guest's measurement — an RLM cannot claim a +sandbox the host did not boot, and a sister that measured nothing yields no +usage (503, never a substituted number). Hard binds on both sides: a job +must name the VM's topic (envelope **and** job) or the agent answers 409; the +client refuses a job for another topic before any request, checks every +echo, refuses a created VM on another digest, and refuses a +`firecracker_required` run that came back without the sister attestation. +Teardown honours the topic's `retain` policy (default **destroy**; `retain` +keeps the jail for audit). Deploy: `deploy/systemd/proof-vm-orchestrator.service`, +runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). +CI runs the fake hypervisor only; no GitHub runner ever boots Firecracker. +The generic `VmBackedRunner` turns inspect / evaluate into VM jobs; +registering it under a `custom_id` is an operator action +(`PROOF_VM_RUNNER_CUSTOM_IDS`, comma-separated; unset = empty registry). The +Lium harvest for `nll` / `throughput` and the live `1x` `EvalExecutorOffer` (`proof-executor`) govern the harvest rent; on the custom path each run request records the resolved executor plan's deadline (tighter of topic and plan) and diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 3257c681c..49370023a 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -341,11 +341,19 @@ Primary: `metric.primary` (`max` or `min`, as the topic says). Win: beat the sealed value by `metric.epsilon_rel` relative (`primary >= sealed * (1 + epsilon_rel)` for `max`). The metric is computed by the runner registered on the host under `metric.custom_id`; nothing -about it is compiled into the network. If the topic sets -`constraints.firecracker_required`, your code runs only inside a Firecracker -guest under the topic's own VM; if it sets `constraints.model_pin`, every -paid call must name exactly that model; `task_slice` / `params` are opaque -runner inputs the topic defines. +about it is compiled into the network. The topic's RLM runs in its own +Firecracker microVM on a dedicated KVM host, and **your code runs in a +separate ("sister") Firecracker guest beside it that has no network +interface**: the RLM fetches your artefact from `artifact_uri`, inspects it, +and ships the bytes into the sister over vsock. Plan for an offline run — +nothing your code does at run time can reach the internet, the RLM, or the +host. The host (not the RLM) stamps `sandboxed` on your report from the +guest it booted, and the `flops_used` your verdict carries is what that +guest measured. If the topic sets `constraints.firecracker_required`, a run +that did not happen in that sister guest is not evidence; if it sets +`constraints.model_pin`, every paid call must name exactly that model; +`task_slice` / `params` are opaque runner inputs the topic defines (they are +exported to your run's environment). **Anti-cheat checklist — every rule in the topic's `checklist` (current version) must pass before a single paid inference call is made.** Read the diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md new file mode 100644 index 000000000..4fe2b4f80 --- /dev/null +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -0,0 +1,184 @@ +# Runbook — Proof topic-VM orchestrator (Firecracker on a dedicated KVM host) + +Operator procedure for the `proof-vm-orchestrator` agent that boots one +Firecracker RLM microVM per Proof topic and a **sister** Firecracker guest +for every miner run. Product spec: [`../PROOF.md`](../PROOF.md) § Isolation +boundary. Code: `crates/proof-vm-proto` (wire), `crates/proof-vm-fc` +(control-plane client), `crates/proof-vm-agent` (agent API), +`crates/proof-fc-host` (Firecracker backend), `bins/proof-vm-orchestrator`. + +## What runs where + +```text +master droplet (DO) dedicated KVM host (bare metal, /dev/kvm) +┌─────────────────────────────────┐ ┌────────────────────────────────────────────┐ +│ proof-challenge │ HTTPS │ proof-vm-orchestrator (systemd, :8200) │ +│ RlmScorer → RunnerRegistry │ bearer │ bearer file · one running VM per topic_id │ +│ [custom_id] → VmBackedRunner │ ───────▶ │ jailer ─ firecracker RLM VM (4 vCPU/8 GiB)│ +│ FirecrackerOrchestrator │ │ ▲ vsock: jobs, owner-key staging │ +│ PROOF_VM_ORCHESTRATOR_URL │ │ │ sister request (paid jobs only) │ +│ PROOF_VM_ORCHESTRATOR_TOKEN_ │ │ jailer ─ firecracker sister guest │ +│ FILE (never logged) │ │ no NIC · artefact over vsock · destroyed │ +│ PROOF_RLM_VM_IMAGE_DIGEST │ │ host stamps sandboxed + flops_used │ +│ VmJob = public data only │ │ nftables per-VM egress allowlist │ +└─────────────────────────────────┘ └────────────────────────────────────────────┘ +``` + +Locked by design (do not move any of it): + +| Rule | Where it is enforced | +|------|----------------------| +| Firecracker microVMs, **sisters** (RLM VM + miner guest) on one dedicated KVM host — not the control-plane droplet, not Lium, not nested | agent runs only where `/dev/kvm` exists (`ConditionPathExists`); nothing in `proof-challenge` can exec | +| The RLM never sees the host filesystem or secrets — only `VmJob` payloads | `proof-vm-proto` types; jobs are the signed topic, digests, rule versions; tests assert no path / key / origin in any body | +| RLM image pin `PROOF_RLM_VM_IMAGE_DIGEST=sha256:…`, empty = fail-closed | `FirecrackerOrchestrator::ready()` → `NotWired` naming the var; agent re-hashes `images/sha256-.ext4` before boot | +| Auth: `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`, bearer file first, never logged | client reads the file per request; agent compares SHA-256 digests in constant time, re-reads its file per request | +| RLM VM 4 vCPU / 8192 MiB | `proof_vm_fc::DEFAULT_RLM_VCPUS` / `DEFAULT_RLM_MEM_MIB` | +| Sister sized by the host / topic deadline only, never by the RLM | `PROOF_VM_AGENT_SISTER_VCPUS` / `_MEM_MIB`; the RLM's request carries no size | +| `retain` default **destroy** on topic close | `TopicVmSpec::for_topic` → `RetainPolicy::Destroy` | +| Hard `topic_id ↔ VM` bind | agent: request topic **and** job topic must equal the VM's (409 `topic_mismatch`); client refuses a job for another topic before any request and checks every echo | +| Zero live Firecracker in CI | every test uses `FakeHypervisor` / `RecordingShell`; `FirecrackerHypervisor::ready()` refuses without `firecracker`, `jailer`, `/dev/kvm` and the test asserts nothing was spawned | + +## Host prerequisites + +- Bare-metal (or nested-virt-capable) Linux with `/dev/kvm`, `CONFIG_VHOST_VSOCK`, nftables, `iproute2`, `e2fsprogs` (`mkfs.ext4`), `coreutils` (`cp --reflink`, `truncate`). +- `firecracker` and `jailer` of the **same** release, statically linked (musl), at `/usr/local/bin/`. Record the release you installed in your change log. +- A filesystem that supports reflinks under `/srv/jailer` and `/var/lib/proof-vm` (XFS with `reflink=1` or btrfs) so per-VM rootfs copies are instant. ext4 works but copies whole images per boot. +- Layout: + +```text +/etc/proof-vm/orchestrator.env # from deploy/env/proof-vm-orchestrator.env.example (0600 root) +/etc/proof-vm/token # bearer, 0400 root; same bytes as the CP's PROOF_VM_ORCHESTRATOR_TOKEN_FILE +/etc/proof-vm/tls.crt + tls.key # agent certificate; CP pins the CA via PROOF_VM_ORCHESTRATOR_CA_FILE if private +/etc/proof-vm/owner-keys/ # optional; files staged into the RLM VM over vsock (0400 root) +/var/lib/proof-vm/vmlinux # guest kernel; pin = sha256sum → PROOF_VM_AGENT_KERNEL_DIGEST +/var/lib/proof-vm/images/sha256-.ext4 # RLM rootfs (CP pin) and sister rootfs (agent pin) +/var/lib/proof-vm/retained/ # `retain` teardowns land here +/srv/jailer/firecracker// # live jails (root/, console.log, net.nft) +``` + +Digests: `sha256sum vmlinux`, `sha256sum .ext4`, then name the rootfs +file `sha256-.ext4`. **Never write a digest you did not compute from the +file you staged.** An unpinned or mis-pinned image never boots; that is the +intended failure. + +Guest images are **not** built by this repository. The RLM image must ship a +guest agent listening on vsock port `5000` and speaking `HostToRlm` / +`RlmToHost`; the sister image must ship one listening on port `5002` speaking +`HostToMiner` / `MinerToHost` (both in `crates/proof-vm-proto/src/guest.rs`; +frames are 4-byte big-endian length + JSON, `api_version: 1`). The RLM guest +asks for a sister by connecting to host port `5001` with a `SisterRequest` +carrying the artefact tarball it already fetched and inspected. + +## Install + +```bash +install -m 0755 target/release/proof-vm-orchestrator /usr/local/bin/proof-vm-orchestrator +install -m 0644 deploy/systemd/proof-vm-orchestrator.service /etc/systemd/system/ +install -d -m 0750 /etc/proof-vm /var/lib/proof-vm/images /var/lib/proof-vm/retained /srv/jailer +install -m 0600 deploy/env/proof-vm-orchestrator.env.example /etc/proof-vm/orchestrator.env +# edit: PROOF_VM_AGENT_KERNEL_DIGEST, PROOF_VM_AGENT_SISTER_IMAGE_DIGEST, +# PROOF_VM_AGENT_EGRESS_ALLOW, PROOF_VM_AGENT_UPLINK, TLS paths +head -c 32 /dev/urandom | base64 -w0 > /etc/proof-vm/token && chmod 0400 /etc/proof-vm/token +systemctl daemon-reload && systemctl enable --now proof-vm-orchestrator +journalctl -u proof-vm-orchestrator -n 50 +``` + +Boot log must show `firecracker + jailer + /dev/kvm present; agent ready` and +`bearer token file present (contents not logged)`. A malformed pin exits 1; a +non-loopback bind without TLS exits 1. + +Verify from the host (the bearer is required even for health): + +```bash +curl -fsS --cacert /etc/proof-vm/tls.crt -H "Authorization: Bearer $(cat /etc/proof-vm/token)" \ + https://127.0.0.1:8200/v1/health +# {"api_version":1,"ready":true,"reason":"","hypervisor":"firecracker","vms":0} +``` + +## Wire the control plane (master droplet) + +In `deploy/env/proof-challenge.env` (age-materialized, never git): + +```bash +PROOF_VM_ORCHESTRATOR_URL=https://:8200 +PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token # same bytes as /etc/proof-vm/token +PROOF_VM_ORCHESTRATOR_CA_FILE=/run/base/proof/vm_orchestrator_ca.pem # only for a private CA +PROOF_RLM_VM_IMAGE_DIGEST=sha256: +PROOF_VM_RUNNER_CUSTOM_IDS= +``` + +Put the token under `deploy/secrets/proof/` (mounted at `/run/base/proof`, +mode 0400, uid 65532). Restart `proof-challenge`; its boot log must show +`firecracker topic-vm orchestrator wired` and one `vm-backed runner +registered` line per id. `GET /v1/status` → `registered_custom` lists the +ids; an open custom topic with a listed id appears in `scorable_topics`. + +The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / +`PROOF_RLM_VM_MEM_MIB` exist for a deliberate change only. + +## Verify a submission end to end (mandatory, see root `AGENTS.md`) + +1. `POST /v1/submissions` on a custom topic with a listed id → the first job + creates the topic's VM (`journalctl -u proof-vm-orchestrator`: `topic vm + booted`, `rlm guest ready`, `owner key material staged` when a key dir is + set), inspection runs (`Inspect` job), then `Evaluate` → `sister guest + booting (no network)` → `sister guest run attested` with `sandboxed=true` + and the guest's `flops_used`. +2. The persisted row's verdict carries that `flops_used`; the artefact zip's + `report.json` has `sandboxed: true`. +3. Failure probes, each **503 with no row and no rent**: + - stop the agent → `orchestrator unreachable`; + - empty `/etc/proof-vm/token` → `orchestrator refused the bearer`; + - remove `PROOF_RLM_VM_IMAGE_DIGEST` → `PROOF_RLM_VM_IMAGE_DIGEST … missing or out of range`; + - delete the RLM image file → agent `503 not_ready: image … no … on this host`; + - `firecracker_required` topic whose RLM never asked for a sister → + `firecracker_required run came back without the host's sister-guest attestation`. +4. `GET /v1/proof/topics` still leaks no holdout; `GET /v1/status` shows no + URL, token, or path. + +## Operate + +| Task | How | +|------|-----| +| Rotate the bearer | write the new token to `/etc/proof-vm/token` and to the CP's token file; no restart on either side (both re-read per request) | +| Rotate the RLM image | stage `images/sha256-.ext4`, set `PROOF_RLM_VM_IMAGE_DIGEST` on the CP, restart `proof-challenge`; running VMs keep the old image until torn down | +| Close a topic | the CP tears the VM down with the topic's `retain` policy (default destroy). `retain` moves `/srv/jailer/firecracker/` to `/var/lib/proof-vm/retained/` (scratch, console log, config) | +| Agent restart | live VMs die with the agent (no `--daemonize`); `attach` then answers 404 and the CP's next job creates a fresh VM. Rules, checklists, and promotions live in the CP's RLM store, not in the VM | +| Egress change | edit `PROOF_VM_AGENT_EGRESS_ALLOW`, restart the agent; existing VMs keep their table until torn down | +| Inspect a VM | `nft list table inet proof_vm_pfc`, `cat /srv/jailer/firecracker//console.log`, `ls /srv/jailer/firecracker//root/` | + +## Security model + +- **CP never mounts host paths into the RLM VM.** The only bytes that cross + the wire are `VmJob`s (signed topic, digests, rule versions, run request) + and their outputs; every test on both sides asserts no path, key, or origin + in a body. +- **Keys: presence-probe on the CP, material staged by the agent only.** + `PROOF_RLM_OWNER_INFERENCE_KEY_FILE` is probed for presence at + `awaiting_owner_keys`; the bytes the RLM uses come from the KVM host's + `PROOF_VM_AGENT_OWNER_KEY_DIR` over vsock and are never logged. +- **Egress allowlist.** Each RLM VM gets its own nftables table: forward from + its TAP only to `PROOF_VM_AGENT_EGRESS_ALLOW`, established replies back, + masquerade out the uplink, drop the rest. Empty list = no egress. The + sister guest has **no network interface**; the artefact arrives over vsock. +- **Hard `topic_id ↔ VM` bind.** Agent: every job / teardown names the topic + twice (envelope + job) and both must match the VM's. Client: a job for + another topic never leaves the process; every echo is checked; a created VM + must report the pinned digest. +- **Host-stamped facts.** `sandboxed` and `flops_used` on paid outputs are + overwritten by the agent from the sister it booted. An RLM claiming a + sandbox without a sister is corrected to `false` and the CP refuses the + report for a `firecracker_required` topic; a sister that measured nothing + yields `flops_used: null` → 503, never a substituted number. +- **Jailer.** Firecracker runs chrooted under `/srv/jailer/firecracker//root` + as `PROOF_VM_AGENT_JAIL_UID`, with a read-only rootfs copy, a fresh scratch + drive, and `/dev/kvm` + `/dev/net/tun` mknod'ed by the jailer. No + `--daemonize` / `--new-pid-ns`, so the agent's child handle is the VM. + +## Limitations (v1) + +- Agent restarts drop live VMs (state is in the CP's RLM store; the next job re-creates). +- One sister per paid job; a second `SisterRequest` in the same job is refused. +- Allowlist entries are IPv4 CIDRs; hostnames must be resolved by the operator (allow the resolver's `:53/udp` if the RLM needs DNS). +- mTLS between CP and agent is a follow-up; today the bearer file over TLS is the auth. +- Guest images (RLM, sister) and their vsock agents are built outside this repository against `proof-vm-proto::guest`. From 9a9b020cc37536b9036b41a1a7ac17d0ae205145 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 17:13:58 +0000 Subject: [PATCH 06/12] fix(proof-fc-host): open the sister listener socket to the jail uid Firecracker connects to /v.sock_5001 as the jail uid while the agent binds it as root; a 0755 socket inode would refuse every guest-initiated sister request. chmod 0666 after bind (the jail root directory, owned by the jail uid and not world-searchable, is what confines it); test asserts the mode. Co-authored-by: Mathis --- crates/proof-fc-host/src/vsock.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/proof-fc-host/src/vsock.rs b/crates/proof-fc-host/src/vsock.rs index fe49400f6..85520a58f 100644 --- a/crates/proof-fc-host/src/vsock.rs +++ b/crates/proof-fc-host/src/vsock.rs @@ -133,20 +133,31 @@ impl GuestChannel { } } -/// Listen for guest-initiated connections to host `port` (bind before boot). +/// Listen for guest-initiated connections to host `port`. +/// +/// Firecracker connects to this socket as the **jail uid**, while the agent +/// binds it as root, so the socket inode is opened to `0666`; the jail root +/// directory (owned by the jail uid, not world-searchable) is what confines +/// it. /// /// # Errors /// /// [`HvError::Backend`]. pub fn listen(jail_root: &Path, port: u32) -> Result { + use std::os::unix::fs::PermissionsExt; let path = listener_path(jail_root, port); let _ = std::fs::remove_file(&path); - UnixListener::bind(&path) - .map_err(|e| HvError::Backend(format!("listen {}: {e}", path.display()))) + let listener = UnixListener::bind(&path) + .map_err(|e| HvError::Backend(format!("listen {}: {e}", path.display())))?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666)) + .map_err(|e| HvError::Backend(format!("chmod {}: {e}", path.display())))?; + Ok(listener) } #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; use proof_vm_proto::guest::{HostToRlm, RlmToHost}; use proof_vm_proto::API_VERSION; @@ -221,6 +232,12 @@ mod tests { assert!(matches!(err, HvError::Guest(_)), "{err}"); assert_eq!(listener_path(&r3, 5001), r3.join("v.sock_5001")); let l = listen(&r3, 5001).expect("listen"); + let mode = std::fs::metadata(listener_path(&r3, 5001)) + .expect("socket") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o666, "the jail uid must be able to connect"); drop(l); let _ = listen(&r3, 5001).expect("rebinding replaces a stale socket"); } From 060f63d91013b08f0c85c3bd66f6a09696b7de8d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:19:46 +0000 Subject: [PATCH 07/12] fix(proof-vm-proto): bind sister attestation to its paid job Greptile P1 "Bind sister evidence" (crates/proof-vm-proto/src/lib.rs): SisterAttestation now carries the topic_id, submission_digest and artifact_digest the host verified before it booted the sister, and bind_evidence() is the one fail-closed check both the agent (before stamping) and the control plane (before accepting) run: the report and the attestation must name exactly the paid job's identities, a sister on a job that runs no miner code is refused, and ErrorCode::EvidenceMismatch (502) names the refusal on the wire. Also adds VmState::Crashed for the agent's dead-VM reaping (P1 "Recover dead VMs"): a VM whose process exited outside a teardown is never advertised as running. Co-authored-by: Mathis --- crates/proof-vm-proto/src/lib.rs | 281 ++++++++++++++++++++++++++++++- 1 file changed, 276 insertions(+), 5 deletions(-) diff --git a/crates/proof-vm-proto/src/lib.rs b/crates/proof-vm-proto/src/lib.rs index 13aa45e78..4ddc81468 100644 --- a/crates/proof-vm-proto/src/lib.rs +++ b/crates/proof-vm-proto/src/lib.rs @@ -13,14 +13,21 @@ //! material the control plane never reads, and boots a **sister** miner //! guest (no network, no host filesystem) when the RLM asks for a run. //! The host — not the RLM — stamps [`SisterAttestation`] and the report's -//! `sandboxed` / `flops_used`. +//! `sandboxed` / `flops_used`. The attestation names the topic, submission, +//! and artefact it was produced for, and [`bind_evidence`] is the one +//! fail-closed check both the agent (before stamping) and the control +//! plane (before accepting) run: evidence for one artefact is never +//! evidence for another. //! //! Nothing here names a benchmark, a model, or a repository. #![forbid(unsafe_code)] #![allow(clippy::module_name_repetitions)] -use proof_rlm::{RetainPolicy, SandboxPolicy, TopicVmSpec, VmHandle, VmJob, VmJobOutput}; +use proof_rlm::{ + CustomRunReport, CustomRunRequest, RetainPolicy, SandboxPolicy, TopicVmSpec, VmHandle, VmJob, + VmJobOutput, +}; use serde::{Deserialize, Serialize}; pub mod guest; @@ -89,6 +96,10 @@ pub enum VmState { Retained, /// Torn down under [`RetainPolicy::Destroy`]: nothing left on the host. Destroyed, + /// The VM process exited outside a teardown. The agent noticed, released + /// the jail per the record's `retain` policy, and stopped advertising the + /// VM; the topic may get a fresh one. Never attachable, never takes a job. + Crashed, } /// One topic VM as the agent knows it. @@ -122,13 +133,23 @@ pub struct RunJobRequest { /// What the host attests about the sister miner guest a job used. /// /// Written by the agent from what it booted and observed, never copied from -/// the RLM guest. The control plane cross-checks it against the report. +/// the RLM guest. It is evidence for **one** paid job: the host copies +/// `topic_id` / `submission_digest` / `artifact_digest` from the sister +/// request it verified against that job before booting, and both the agent +/// and the control plane run [`bind_evidence`] so a sister that ran artefact +/// A can never stamp a report for artefact B. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SisterAttestation { /// Host id of the sister VM (destroyed after the run). pub sister_vm_id: String, /// `sha256:` digest of the miner-guest image the host verified and booted. pub image_digest: String, + /// Topic the paid job (and the RLM VM) is bound to. + pub topic_id: String, + /// Frozen submission digest of the paid job the sister ran for. + pub submission_digest: String, + /// sha256 hex of the artefact tarball the host re-hashed and booted. + pub artifact_digest: String, /// The host booted the sister and the run happened inside it. pub sandboxed: bool, /// Network the sister had. Always `none`: the artefact travels over vsock. @@ -141,6 +162,157 @@ pub struct SisterAttestation { pub exit_code: Option, } +impl SisterAttestation { + /// The identities this evidence is bound to. + #[must_use] + pub fn binding(&self) -> EvidenceBinding { + EvidenceBinding::new( + &self.topic_id, + &self.submission_digest, + &self.artifact_digest, + ) + } +} + +/// The identities a paid job — and any sister evidence for it — are bound to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EvidenceBinding { + /// Topic id. + pub topic_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Artefact digest, lower-case hex. + pub artifact_digest: String, +} + +impl EvidenceBinding { + /// Normalised (trimmed; artefact digest lower-cased). + #[must_use] + pub fn new(topic_id: &str, submission_digest: &str, artifact_digest: &str) -> Self { + Self { + topic_id: topic_id.trim().to_owned(), + submission_digest: submission_digest.trim().to_owned(), + artifact_digest: artifact_digest.trim().to_ascii_lowercase(), + } + } + + fn of_request(req: &CustomRunRequest) -> Self { + Self::new(&req.topic_id, &req.submission_digest, &req.artifact_digest) + } + + fn of_report(report: &CustomRunReport) -> Self { + Self::new( + &report.topic_id, + &report.submission_digest, + &report.artifact_digest, + ) + } + + /// The identities a paid job runs for; `None` for jobs that run no miner code. + #[must_use] + pub fn of_job(job: &VmJob) -> Option { + match job { + VmJob::Baseline { request } | VmJob::Evaluate { request, .. } => { + Some(Self::of_request(request)) + } + VmJob::ProposeRules { .. } | VmJob::Inspect { .. } | VmJob::Archive { .. } => None, + } + } + + /// The identities the RLM's own report names; `None` for non-paid outputs. + #[must_use] + pub fn of_output(output: &VmJobOutput) -> Option { + match output { + VmJobOutput::Baseline(report) => Some(Self::of_report(report)), + VmJobOutput::Evaluated(run) => Some(Self::of_report(&run.report)), + VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => None, + } + } + + /// The first field on which `other` differs from `self`. + #[must_use] + pub fn first_mismatch(&self, other: &Self) -> Option<(&'static str, String, String)> { + [ + ("topic_id", &self.topic_id, &other.topic_id), + ( + "submission_digest", + &self.submission_digest, + &other.submission_digest, + ), + ( + "artifact_digest", + &self.artifact_digest, + &other.artifact_digest, + ), + ] + .into_iter() + .find(|(_, want, got)| want != got) + .map(|(field, want, got)| (field, got.clone(), want.clone())) + } +} + +/// Why sister evidence is not evidence for a job. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum EvidenceError { + /// A sister was attested for a job that runs no miner code. + #[error("a sister guest was attested for a job that runs no miner code")] + Unpaid, + /// The attestation names another topic / submission / artefact than the job. + #[error("sister evidence names {field} {got:?}, the paid job names {want:?}")] + SisterMismatch { + /// Which identity differs. + field: &'static str, + /// What the attestation says. + got: String, + /// What the job says. + want: String, + }, + /// The RLM's report names another topic / submission / artefact than the job. + #[error("report names {field} {got:?}, the paid job names {want:?}")] + ReportMismatch { + /// Which identity differs. + field: &'static str, + /// What the report says. + got: String, + /// What the job says. + want: String, + }, +} + +/// Fail-closed binding of a job's output and sister evidence to the job. +/// +/// `Ok` only when the report (for paid outputs) and the attestation (when +/// present) name exactly the job's topic, submission, and artefact. Run by +/// the agent before it stamps `sandboxed` / `flops_used` and by the control +/// plane before it accepts them. +/// +/// # Errors +/// +/// [`EvidenceError`] naming the first identity that differs. +pub fn bind_evidence( + job: &VmJob, + output: &VmJobOutput, + sister: Option<&SisterAttestation>, +) -> Result<(), EvidenceError> { + let Some(want) = EvidenceBinding::of_job(job) else { + return match sister { + Some(_) => Err(EvidenceError::Unpaid), + None => Ok(()), + }; + }; + if let Some(report) = EvidenceBinding::of_output(output) { + if let Some((field, got, want)) = want.first_mismatch(&report) { + return Err(EvidenceError::ReportMismatch { field, got, want }); + } + } + if let Some(s) = sister { + if let Some((field, got, want)) = want.first_mismatch(&s.binding()) { + return Err(EvidenceError::SisterMismatch { field, got, want }); + } + } + Ok(()) +} + /// `POST /v1/vms/{vm_id}/jobs` response. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunJobResponse { @@ -198,6 +370,9 @@ pub enum ErrorCode { Backend, /// The guest answered with the wrong output shape. WrongOutput, + /// The report or the sister attestation names another topic, submission, + /// or artefact than the paid job ([`bind_evidence`]). Nothing was stamped. + EvidenceMismatch, } impl ErrorCode { @@ -210,7 +385,7 @@ impl ErrorCode { Self::BadSpec => 400, Self::TopicMismatch | Self::AlreadyExists | Self::Busy => 409, Self::NotFound => 404, - Self::Backend | Self::WrongOutput => 502, + Self::Backend | Self::WrongOutput | Self::EvidenceMismatch => 502, } } } @@ -247,7 +422,8 @@ pub enum ProtoError { #[cfg(test)] mod tests { use super::*; - use proof_rlm::fixtures::{pinned_template, request}; + use proof_rlm::fixtures::{pinned_template, report_for, request}; + use proof_rlm::RunOutcome; #[test] fn paths_are_stable_and_codes_map_to_statuses() { @@ -262,8 +438,103 @@ mod tests { assert_eq!(ErrorCode::TopicMismatch.status(), 409); assert_eq!(ErrorCode::NotFound.status(), 404); assert_eq!(ErrorCode::Backend.status(), 502); + assert_eq!(ErrorCode::EvidenceMismatch.status(), 502); assert_eq!(DEFAULT_AGENT_PORT, 8200); assert_eq!(API_VERSION, 1); + assert!(serde_json::to_string(&VmState::Crashed) + .expect("json") + .contains("crashed")); + } + + fn sister_for(req: &proof_rlm::CustomRunRequest) -> SisterAttestation { + SisterAttestation { + sister_vm_id: "topic-a-0001-s1".into(), + image_digest: format!("sha256:{}", "dd".repeat(32)), + topic_id: req.topic_id.clone(), + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.to_ascii_uppercase(), + sandboxed: true, + network: "none".into(), + flops_used: Some(7), + wall_ms: 10, + exit_code: Some(0), + } + } + + /// Sister evidence is bound to the paid job it was produced for: the + /// attestation for artefact A is refused on a job for artefact B, and so + /// is a report that names another submission than the job. + #[test] + fn sister_evidence_binds_to_the_paid_job_it_ran_for() { + let a = request(); + let mut b = request(); + b.submission_digest = "submission-b".into(); + b.artifact_digest = "ba".repeat(32); + let job_b = VmJob::Evaluate { + request: b.clone(), + checklist_digest: "c".into(), + rules_version: 1, + }; + let out_b = VmJobOutput::Evaluated(RunOutcome { + report: report_for(&b, 0.9), + logs: vec![], + }); + bind_evidence(&job_b, &out_b, Some(&sister_for(&b))).expect("same identities"); + bind_evidence(&job_b, &out_b, None).expect("no sister is not a mismatch"); + assert_eq!( + bind_evidence(&job_b, &out_b, Some(&sister_for(&a))), + Err(EvidenceError::SisterMismatch { + field: "submission_digest", + got: a.submission_digest.clone(), + want: b.submission_digest.clone(), + }) + ); + let mut other_artifact = sister_for(&b); + other_artifact.artifact_digest = a.artifact_digest.clone(); + let err = bind_evidence(&job_b, &out_b, Some(&other_artifact)).expect_err("artefact"); + assert!(matches!( + err, + EvidenceError::SisterMismatch { + field: "artifact_digest", + .. + } + )); + assert!(err.to_string().contains("sister evidence names"), "{err}"); + let out_a = VmJobOutput::Baseline(report_for(&a, 0.9)); + assert!(matches!( + bind_evidence( + &VmJob::Baseline { request: b.clone() }, + &out_a, + Some(&sister_for(&b)) + ), + Err(EvidenceError::ReportMismatch { + field: "submission_digest", + .. + }) + )); + let inspect = VmJob::Inspect { + request: a.clone(), + rules: proof_rlm::fixtures::rules(), + }; + let inspected = VmJobOutput::Inspected(proof_rlm::InspectOutcome { + checklist: proof_rlm::fixtures::green( + &proof_rlm::fixtures::rules(), + &a.submission_digest, + ), + artifact: vec![], + }); + bind_evidence(&inspect, &inspected, None).expect("no miner code, no sister"); + assert_eq!( + bind_evidence(&inspect, &inspected, Some(&sister_for(&a))), + Err(EvidenceError::Unpaid) + ); + let binding = sister_for(&a).binding(); + assert_eq!( + binding.artifact_digest, + a.artifact_digest.to_ascii_lowercase() + ); + assert_eq!(EvidenceBinding::of_job(&inspect), None); + assert_eq!(EvidenceBinding::of_output(&VmJobOutput::Archived), None); } #[test] From f488fbc44a38cbf0028c49518efc9af30b89e2c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:19:46 +0000 Subject: [PATCH 08/12] fix(proof-vm-agent): refuse unbound evidence; reap dead vms Greptile P1 "Bind sister evidence": run_job calls bind_evidence before stamp_output, so a hypervisor (or a compromised guest behind it) that presents artefact A's sister attestation for a paid job on artefact B, or a report naming another submission than the job, gets 502 evidence_mismatch and nothing is stamped. Greptile P1 "Recover dead VMs" (crates/proof-vm-agent/src/router.rs): Hypervisor::alive() probes the VM process; attach, create, run and health probe every Running record and reap a dead one per its retain policy (teardown), record it Crashed, and stop advertising it, so the topic gets a fresh VM instead of a 409 forever. A VM that dies under a job is reaped by that job on its way out (it holds the lock). Fake hypervisor gains set_sister_replay, set_rlm_report_submission, kill and set_dies_under_job; tests cover replayed evidence (502), the dead-VM recreate path, death under a job, and the health sweep. Co-authored-by: Mathis --- crates/proof-vm-agent/src/fixtures_tests.rs | 76 ++++++- crates/proof-vm-agent/src/hypervisor.rs | 17 +- crates/proof-vm-agent/src/lib.rs | 220 +++++++++++++++++++- crates/proof-vm-agent/src/router.rs | 183 ++++++++++++---- crates/proof-vm-agent/src/stamp.rs | 4 + 5 files changed, 460 insertions(+), 40 deletions(-) diff --git a/crates/proof-vm-agent/src/fixtures_tests.rs b/crates/proof-vm-agent/src/fixtures_tests.rs index a3f62a596..52bb74d84 100644 --- a/crates/proof-vm-agent/src/fixtures_tests.rs +++ b/crates/proof-vm-agent/src/fixtures_tests.rs @@ -10,6 +10,7 @@ clippy::unwrap_used )] +use std::collections::BTreeSet; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -23,7 +24,7 @@ use proof_rlm::{ ArtifactFile, Checklist, CustomRunRequest, InspectOutcome, LogFile, RetainPolicy, RunOutcome, TopicVmSpec, VmJob, VmJobOutput, }; -use proof_vm_proto::SisterAttestation; +use proof_vm_proto::{EvidenceBinding, SisterAttestation}; use crate::auth::BearerAuth; use crate::hypervisor::{BootedVm, HvError, Hypervisor, JobOutcome}; @@ -44,11 +45,20 @@ pub struct FakeHypervisor { sister: AtomicBool, /// What that sister measures. sister_flops: Mutex>, + /// Identities the sister attests instead of the job's (a compromised + /// guest replaying evidence from another run). `None` = honest. + sister_replay: Mutex>, /// What the RLM writes into its own report before the host stamps it. rlm_claims_sandboxed: AtomicBool, rlm_flops: Mutex>, + /// Submission digest the RLM writes into the report instead of the job's. + rlm_report_submission: Mutex>, proposed: Mutex>, job_delay: Mutex>, + /// VMs whose process "exited" outside a teardown. + dead: Mutex>, + /// The VM process exits while the next job runs. + dies_under_job: AtomicBool, boots: Mutex>, jobs: Mutex>, teardowns: Mutex>, @@ -63,13 +73,17 @@ impl FakeHypervisor { red: Mutex::new(None), sister: AtomicBool::new(true), sister_flops: Mutex::new(Some(1)), + sister_replay: Mutex::new(None), rlm_claims_sandboxed: AtomicBool::new(false), rlm_flops: Mutex::new(Some(999_999)), + rlm_report_submission: Mutex::new(None), proposed: Mutex::new(vec![ChecklistRule { id: "rlm_rule".into(), text: "a rule the fake rlm wrote".into(), }]), job_delay: Mutex::new(None), + dead: Mutex::new(BTreeSet::new()), + dies_under_job: AtomicBool::new(false), boots: Mutex::new(Vec::new()), jobs: Mutex::new(Vec::new()), teardowns: Mutex::new(Vec::new()), @@ -102,6 +116,12 @@ impl FakeHypervisor { *self.sister_flops.lock().unwrap() = v; } + /// Attest the sister for `binding`'s identities instead of the job's — + /// what a compromised guest replaying another run's evidence looks like. + pub fn set_sister_replay(&self, binding: Option) { + *self.sister_replay.lock().unwrap() = binding; + } + /// What the RLM claims before the host corrects it. pub fn set_rlm_claims_sandboxed(&self, v: bool) { self.rlm_claims_sandboxed.store(v, Ordering::SeqCst); @@ -111,6 +131,11 @@ impl FakeHypervisor { *self.rlm_flops.lock().unwrap() = v; } + /// Make the RLM's report name this submission instead of the job's. + pub fn set_rlm_report_submission(&self, digest: Option<&str>) { + *self.rlm_report_submission.lock().unwrap() = digest.map(str::to_owned); + } + pub fn set_proposed(&self, rules: Vec) { *self.proposed.lock().unwrap() = rules; } @@ -120,6 +145,16 @@ impl FakeHypervisor { *self.job_delay.lock().unwrap() = d; } + /// Simulate the VM process exiting outside a teardown. + pub fn kill(&self, vm_id: &str) { + self.dead.lock().unwrap().insert(vm_id.to_owned()); + } + + /// Make the VM process exit while the next job runs (the job fails). + pub fn set_dies_under_job(&self, v: bool) { + self.dies_under_job.store(v, Ordering::SeqCst); + } + pub fn boots(&self) -> Vec { self.boots.lock().unwrap().clone() } @@ -136,6 +171,9 @@ impl FakeHypervisor { let mut r = report_for(req, *self.primary.lock().unwrap()); r.sandboxed = self.rlm_claims_sandboxed.load(Ordering::SeqCst); r.flops_used = *self.rlm_flops.lock().unwrap(); + if let Some(other) = self.rlm_report_submission.lock().unwrap().clone() { + r.submission_digest = other; + } r } @@ -143,9 +181,20 @@ impl FakeHypervisor { if !self.sister.load(Ordering::SeqCst) { return None; } + let binding = self + .sister_replay + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| { + EvidenceBinding::new(&req.topic_id, &req.submission_digest, &req.artifact_digest) + }); Some(SisterAttestation { sister_vm_id: format!("{}-s{}", vm.vm_id, req.submission_digest.len()), image_digest: miner_image_digest(), + topic_id: binding.topic_id, + submission_digest: binding.submission_digest, + artifact_digest: binding.artifact_digest, sandboxed: true, network: "none".into(), flops_used: *self.sister_flops.lock().unwrap(), @@ -182,6 +231,22 @@ impl Hypervisor for FakeHypervisor { Ok(vm) } + async fn alive(&self, vm: &BootedVm) -> bool { + let booted = self + .boots + .lock() + .unwrap() + .iter() + .any(|b| b.vm_id == vm.vm_id); + let torn = self + .teardowns + .lock() + .unwrap() + .iter() + .any(|(id, _)| *id == vm.vm_id); + booted && !torn && !self.dead.lock().unwrap().contains(&vm.vm_id) + } + async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result { self.jobs .lock() @@ -191,6 +256,15 @@ impl Hypervisor for FakeHypervisor { if let Some(d) = delay { tokio::time::sleep(d).await; } + if self.dies_under_job.swap(false, Ordering::SeqCst) { + self.dead.lock().unwrap().insert(vm.vm_id.clone()); + } + if self.dead.lock().unwrap().contains(&vm.vm_id) { + return Err(HvError::Guest(format!( + "vm {} process exited under the job", + vm.vm_id + ))); + } Ok(match job { VmJob::ProposeRules { .. } => JobOutcome { output: VmJobOutput::Rules(self.proposed.lock().unwrap().clone()), diff --git a/crates/proof-vm-agent/src/hypervisor.rs b/crates/proof-vm-agent/src/hypervisor.rs index 9440c10ef..fde29f962 100644 --- a/crates/proof-vm-agent/src/hypervisor.rs +++ b/crates/proof-vm-agent/src/hypervisor.rs @@ -31,6 +31,10 @@ pub enum HvError { /// The job's deadline passed before the guest answered. #[error("job deadline of {0}s passed")] Deadline(u64), + /// The work was cancelled by the host (the job it served ended first). + /// Whatever it had booted is already torn down. + #[error("cancelled: {0}")] + Cancelled(String), } /// One booted topic VM as the backend tracks it. @@ -67,13 +71,22 @@ pub trait Hypervisor: Send + Sync { fn ready(&self) -> Result<(), HvError>; /// Boot the RLM VM for `spec` under `vm_id`, verify the image digest - /// first, wait for the guest agent, stage owner key material. + /// first, wait for the guest agent, stage owner key material. A boot that + /// fails at any step leaves nothing behind on the host. async fn boot(&self, vm_id: &str, spec: &TopicVmSpec) -> Result; + /// Whether the VM's process is still running. The agent asks before it + /// advertises a VM as running or hands it a job; a VM whose process is + /// gone is reaped (per its retain policy) and its topic may get a fresh + /// one. A VM this backend never booted, or already tore down, is not alive. + async fn alive(&self, vm: &BootedVm) -> bool; + /// Run one job inside the VM, booting a sister guest if the RLM asks. + /// Any sister the job did not finish with is destroyed before this + /// returns; the attestation names the job's identities. async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result; /// Stop the VM; keep its scratch under `Retain`. `Ok(true)` only when the - /// requested end state was reached. + /// requested end state was reached. Also how a dead VM is reaped. async fn teardown(&self, vm: &BootedVm, policy: RetainPolicy) -> Result; } diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs index 3f7549d56..5e4b6cd4e 100644 --- a/crates/proof-vm-agent/src/lib.rs +++ b/crates/proof-vm-agent/src/lib.rs @@ -12,9 +12,15 @@ //! | run | `POST /v1/vms/{vm_id}/jobs` | request `topic_id` **and** the job's own topic must equal the VM's | //! | teardown | `DELETE /v1/vms/{vm_id}` | request `topic_id` must equal the VM's; destroy or retain | //! +//! "Running" means the hypervisor confirms the process is alive: a VM that +//! died outside a teardown is reaped per its retain policy, recorded as +//! `crashed`, and its topic may create a fresh one. +//! //! The agent never mounts a host path into a guest, never receives a key //! from the control plane, and stamps `sandboxed` / `flops_used` on paid -//! outputs from the sister guest **it** booted ([`stamp_output`]). The +//! outputs from the sister guest **it** booted ([`stamp_output`]) — and only +//! after `proof_vm_proto::bind_evidence` confirmed the attestation and the +//! report name that job's topic, submission, and artefact. The //! [`Hypervisor`] behind it is Firecracker + jailer in production //! (`proof-fc-host`) and [`fixtures::FakeHypervisor`] in every test — no //! test here or in CI boots a VM. @@ -396,6 +402,218 @@ mod tests { )); } + /// Sister evidence is bound to the job it was produced for. A hypervisor + /// (or a compromised guest behind it) that presents the attestation of + /// artefact A for a paid job on artefact B gets a 502 and nothing is + /// stamped; so does a report that names another submission than the job. + #[tokio::test] + async fn replayed_sister_evidence_for_another_artifact_is_refused() { + let hv = FakeHypervisor::new(0.8); + let (app, _) = app(hv.clone(), "replay"); + let rec = create(&app).await; + let a = request(); + let mut b = request(); + b.submission_digest = "submission-b".into(); + b.artifact_digest = "ba".repeat(32); + let evaluate = |req: &proof_rlm::CustomRunRequest| { + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Evaluate { + request: req.clone(), + checklist_digest: token_for(req).checklist_digest().to_owned(), + rules_version: 1, + }, + }) + .expect("json") + }; + hv.set_sister_replay(Some(proof_vm_proto::EvidenceBinding::new( + &a.topic_id, + &a.submission_digest, + &a.artifact_digest, + ))); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(evaluate(&b)), + ) + .await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(err.code, ErrorCode::EvidenceMismatch); + assert!(err.error.contains("submission_digest"), "{}", err.error); + assert!(err.error.contains("submission-b"), "{}", err.error); + + hv.set_sister_replay(None); + hv.set_rlm_report_submission(Some("submission-c")); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(evaluate(&b)), + ) + .await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(err.code, ErrorCode::EvidenceMismatch); + assert!(err.error.starts_with("report names"), "{}", err.error); + + hv.set_rlm_report_submission(None); + let (status, out): (StatusCode, RunJobResponse) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some(evaluate(&b)), + ) + .await; + assert_eq!(status, StatusCode::OK); + let sister = out.sister.expect("honest sister"); + assert_eq!(sister.submission_digest, "submission-b"); + assert_eq!(sister.artifact_digest, "ba".repeat(32)); + assert_eq!(sister.topic_id, b.topic_id); + let VmJobOutput::Evaluated(run) = out.output else { + panic!("shape"); + }; + assert!(run.report.sandboxed); + run.report.verify(&b).expect("bound to b"); + } + + /// A VM whose process exits outside a teardown is not advertised as + /// running: attach is 404, a job on it is 404 (no dispatch), the host is + /// asked to release it per the retain policy, and the topic may create a + /// fresh VM instead of being blocked by the dead one. + #[tokio::test] + async fn a_dead_vm_is_reaped_and_its_topic_can_recreate() { + let hv = FakeHypervisor::new(0.8); + let (app, state) = app(hv.clone(), "dead"); + let rec = create(&app).await; + assert_eq!(state.running().await.len(), 1); + hv.kill(&rec.handle.vm_id); + + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "a dead vm is not attachable"); + assert_eq!(err.code, ErrorCode::NotFound); + assert_eq!( + hv.teardowns(), + vec![(rec.handle.vm_id.clone(), RetainPolicy::Destroy)], + "reaped once, per the record's retain policy" + ); + assert!(state.running().await.is_empty()); + + let req = request(); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(err.error.contains("Crashed"), "{}", err.error); + assert!(hv.jobs().is_empty(), "no job reaches a dead vm"); + + let fresh = create(&app).await; + assert_ne!(fresh.handle.vm_id, rec.handle.vm_id); + assert_eq!(fresh.state, VmState::Running); + assert_eq!(hv.boots().len(), 2); + let (status, attached): (StatusCode, VmRecord) = call( + &app, + "GET", + &paths::vm_by_topic("topic-a"), + Some(TOKEN), + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(attached, fresh); + assert_eq!(hv.teardowns().len(), 1, "the fresh vm is not reaped"); + + // Teardown of the crashed record is idempotent and confirmed. + let (status, down): (StatusCode, TeardownResponse) = call( + &app, + "DELETE", + &paths::vm(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(TeardownRequest { + topic_id: "topic-a".into(), + policy: RetainPolicy::Destroy, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(down.state, VmState::Crashed); + assert!(down.confirmed); + assert_eq!(hv.teardowns().len(), 1, "not released twice"); + } + + /// The process dies while a job is in flight: the job fails, the VM is + /// reaped on the way out (the job holds the lock), and the next create + /// for the topic boots a fresh VM without a 409. Health sweeps too. + #[tokio::test] + async fn a_vm_that_dies_under_a_job_is_reaped_by_that_job() { + let hv = FakeHypervisor::new(0.8); + let (app, state) = app(hv.clone(), "dies-under-job"); + let rec = create(&app).await; + hv.set_dies_under_job(true); + let req = request(); + let (status, err): (StatusCode, ErrorBody) = call( + &app, + "POST", + &paths::vm_jobs(&rec.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(RunJobRequest { + topic_id: req.topic_id.clone(), + job: VmJob::Archive { + topic_id: req.topic_id.clone(), + }, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(err.code, ErrorCode::Backend); + assert_eq!(hv.jobs().len(), 1, "the job was dispatched and failed"); + assert_eq!( + hv.teardowns(), + vec![(rec.handle.vm_id.clone(), RetainPolicy::Destroy)], + "reaped by the failing job" + ); + assert!(state.running().await.is_empty()); + let fresh = create(&app).await; + assert_eq!(fresh.handle.topic_id, "topic-a"); + let (_, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!(health.vms, 2, "the crashed record stays for audit"); + hv.kill(&fresh.handle.vm_id); + let (_, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!(health.vms, 2); + assert_eq!(hv.teardowns().len(), 2, "health sweeps the dead too"); + assert!(state.running().await.is_empty()); + } + #[tokio::test] async fn a_busy_vm_refuses_a_second_job_and_retain_keeps_the_record() { let hv = FakeHypervisor::new(0.8); diff --git a/crates/proof-vm-agent/src/router.rs b/crates/proof-vm-agent/src/router.rs index be53e875c..d97fcd715 100644 --- a/crates/proof-vm-agent/src/router.rs +++ b/crates/proof-vm-agent/src/router.rs @@ -1,5 +1,11 @@ //! The agent's HTTP surface: create / attach / run / teardown, bearer-gated, //! with the topic ↔ VM bind enforced on every call that names a VM. +//! +//! A `Running` record is only advertised while the hypervisor confirms the +//! VM's process is alive: attach, create, run, and health probe it first, and +//! a VM whose process exited outside a teardown is **reaped** — released per +//! its retain policy and recorded as [`VmState::Crashed`] — so its topic can +//! get a fresh VM instead of being blocked by a dead one. use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; @@ -13,10 +19,10 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use proof_rlm::{RetainPolicy, VmHandle}; use proof_vm_proto::{ - paths, AgentHealth, CreateVmRequest, ErrorBody, ErrorCode, RunJobRequest, RunJobResponse, - TeardownRequest, TeardownResponse, VmRecord, VmState, API_VERSION, + bind_evidence, paths, AgentHealth, CreateVmRequest, ErrorBody, ErrorCode, RunJobRequest, + RunJobResponse, TeardownRequest, TeardownResponse, VmRecord, VmState, API_VERSION, }; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use crate::auth::BearerAuth; use crate::hypervisor::{BootedVm, HvError, Hypervisor}; @@ -48,7 +54,10 @@ impl From for AgentError { let code = match e { HvError::NotReady(_) | HvError::Image(_) => ErrorCode::NotReady, HvError::Spec(_) => ErrorCode::BadSpec, - HvError::Guest(_) | HvError::Backend(_) | HvError::Deadline(_) => ErrorCode::Backend, + HvError::Guest(_) + | HvError::Backend(_) + | HvError::Deadline(_) + | HvError::Cancelled(_) => ErrorCode::Backend, }; Self::new(code, e.to_string()) } @@ -105,16 +114,32 @@ impl AgentState { } } - /// Running VMs, by id. + /// VMs that are running **and** whose process is alive right now. Dead + /// ones found on the way are reaped. pub async fn running(&self) -> Vec { - self.inner + self.sweep().await + } + + /// Probe every `Running` record; reap the dead. Returns the live ones. + pub async fn sweep(&self) -> Vec { + let ids: Vec = self + .inner .vms .read() .await .values() .filter(|e| e.record.state == VmState::Running) - .map(|e| e.record.clone()) - .collect() + .map(|e| e.record.handle.vm_id.clone()) + .collect(); + let mut live = Vec::new(); + for id in ids { + if let Some(rec) = self.probe(&id).await { + if rec.state == VmState::Running { + live.push(rec); + } + } + } + live } fn mint_vm_id(&self, topic_id: &str) -> String { @@ -133,6 +158,79 @@ impl AgentState { .ok_or_else(|| AgentError::new(ErrorCode::NotFound, format!("no vm {vm_id}")))?; Ok((e.record.clone(), e.booted.clone(), e.lock.clone())) } + + /// The record for `vm_id` as it truly stands: a `Running` record whose + /// process is gone is reaped first (when no job holds the VM) and is + /// never reported as running. `None` when there is no such VM. + async fn probe(&self, vm_id: &str) -> Option { + let (record, booted, lock) = self.entry(vm_id).await.ok()?; + if record.state != VmState::Running || self.inner.hypervisor.alive(&booted).await { + return Some(record); + } + match lock.try_lock_owned() { + Ok(guard) => Some(self.reap(&record, &booted, guard).await), + // A job is in flight on the dead VM: it fails on its own and reaps + // on its way out. Meanwhile the VM is not running for anyone. + Err(_) => Some(VmRecord { + state: VmState::Crashed, + ..record + }), + } + } + + /// Release a dead VM's host resources per its retain policy and record + /// it as crashed. The caller holds the VM's job lock so no job races the + /// teardown. + async fn reap( + &self, + record: &VmRecord, + booted: &BootedVm, + _job: OwnedMutexGuard<()>, + ) -> VmRecord { + let vm_id = &record.handle.vm_id; + tracing::warn!( + %vm_id, topic_id = %record.handle.topic_id, retain = ?record.retain, + "topic vm process exited outside teardown; reaping" + ); + match self.inner.hypervisor.teardown(booted, record.retain).await { + Ok(true) => {} + Ok(false) => tracing::error!(%vm_id, "reap: host did not confirm the release"), + Err(e) => tracing::error!(%vm_id, "reap: {e}"), + } + let mut vms = self.inner.vms.write().await; + match vms.get_mut(vm_id.as_str()) { + Some(e) if e.record.state == VmState::Running => { + e.record.state = VmState::Crashed; + e.record.clone() + } + Some(e) => e.record.clone(), + None => VmRecord { + state: VmState::Crashed, + ..record.clone() + }, + } + } + + /// The running, alive VM bound to `topic_id`, if any. + async fn live_vm_for(&self, topic_id: &str) -> Option { + let candidates: Vec = self + .inner + .vms + .read() + .await + .values() + .filter(|e| e.record.handle.topic_id == topic_id && e.record.state == VmState::Running) + .map(|e| e.record.handle.vm_id.clone()) + .collect(); + for id in candidates { + if let Some(rec) = self.probe(&id).await { + if rec.state == VmState::Running { + return Some(rec); + } + } + } + None + } } fn bind(record: &VmRecord, topic_id: &str, what: &str) -> Result<(), AgentError> { @@ -165,6 +263,8 @@ async fn health(State(state): State) -> Json { Ok(()) => (true, String::new()), Err(e) => (false, e.to_string()), }; + // Health is also where an idle host notices a VM that died in the meantime. + state.sweep().await; Json(AgentHealth { api_version: API_VERSION, ready, @@ -184,16 +284,13 @@ async fn create_vm( state.inner.hypervisor.ready()?; // Serialise creates: the topic ↔ VM check and the insert must be one step. let _create = state.inner.create_lock.lock().await; - if let Some(existing) = - state.inner.vms.read().await.values().find(|e| { - e.record.handle.topic_id == spec.topic_id && e.record.state == VmState::Running - }) - { + // A dead VM is reaped here and does not count: the topic gets a fresh one. + if let Some(existing) = state.live_vm_for(&spec.topic_id).await { return Err(AgentError::new( ErrorCode::AlreadyExists, format!( "topic {:?} already has vm {}", - spec.topic_id, existing.record.handle.vm_id + spec.topic_id, existing.handle.vm_id ), )); } @@ -233,20 +330,19 @@ async fn attach( State(state): State, Path(topic_id): Path, ) -> Result, AgentError> { - state - .inner - .vms - .read() - .await - .values() - .find(|e| e.record.handle.topic_id == topic_id && e.record.state == VmState::Running) - .map(|e| Json(e.record.clone())) - .ok_or_else(|| { - AgentError::new( - ErrorCode::NotFound, - format!("no running vm for topic {topic_id:?}"), - ) - }) + state.live_vm_for(&topic_id).await.map(Json).ok_or_else(|| { + AgentError::new( + ErrorCode::NotFound, + format!("no running vm for topic {topic_id:?}"), + ) + }) +} + +fn not_running(vm_id: &str, state: VmState) -> AgentError { + AgentError::new( + ErrorCode::NotFound, + format!("vm {vm_id} is {state:?}, not running"), + ) } async fn run_job( @@ -256,26 +352,41 @@ async fn run_job( ) -> Result, AgentError> { let (record, booted, lock) = state.entry(&vm_id).await?; if record.state != VmState::Running { - return Err(AgentError::new( - ErrorCode::NotFound, - format!("vm {vm_id} is {:?}, not running", record.state), - )); + return Err(not_running(&vm_id, record.state)); } bind(&record, &body.topic_id, "the request")?; bind(&record, body.job.topic_id(), "the job")?; - let _guard = lock + let guard = lock .try_lock_owned() .map_err(|_| AgentError::new(ErrorCode::Busy, format!("vm {vm_id} is running a job")))?; - let outcome = state.inner.hypervisor.run_job(&booted, &body.job).await?; + if !state.inner.hypervisor.alive(&booted).await { + let reaped = state.reap(&record, &booted, guard).await; + return Err(not_running(&vm_id, reaped.state)); + } + let outcome = match state.inner.hypervisor.run_job(&booted, &body.job).await { + Ok(outcome) => outcome, + Err(e) => { + // A guest that died under the job is reaped now, while we hold it. + if !state.inner.hypervisor.alive(&booted).await { + state.reap(&record, &booted, guard).await; + } + return Err(e.into()); + } + }; if !output_matches(&body.job, &outcome.output) { return Err(AgentError::new( ErrorCode::WrongOutput, "guest answered with another output shape", )); } + // Evidence for one job never stamps another: the attestation and the + // report must name this job's topic, submission, and artefact. + bind_evidence(&body.job, &outcome.output, outcome.sister.as_ref()) + .map_err(|e| AgentError::new(ErrorCode::EvidenceMismatch, e.to_string()))?; if let Some(s) = &outcome.sister { tracing::info!( - %vm_id, sister = %s.sister_vm_id, sandboxed = s.sandboxed, + %vm_id, sister = %s.sister_vm_id, submission = %s.submission_digest, + artifact = %s.artifact_digest, sandboxed = s.sandboxed, flops_used = ?s.flops_used, wall_ms = s.wall_ms, "sister guest run attested" ); } @@ -321,7 +432,7 @@ async fn teardown( VmState::Destroyed => { vms.remove(&vm_id); } - VmState::Retained | VmState::Running => { + VmState::Retained | VmState::Running | VmState::Crashed => { if let Some(e) = vms.get_mut(&vm_id) { e.record.state = end; } diff --git a/crates/proof-vm-agent/src/stamp.rs b/crates/proof-vm-agent/src/stamp.rs index 681f45871..9075b0432 100644 --- a/crates/proof-vm-agent/src/stamp.rs +++ b/crates/proof-vm-agent/src/stamp.rs @@ -59,9 +59,13 @@ mod tests { use proof_rlm::RunOutcome; fn sister(sandboxed: bool, flops: Option) -> SisterAttestation { + let req = request(); SisterAttestation { sister_vm_id: "topic-a-0001-s1".into(), image_digest: format!("sha256:{}", "dd".repeat(32)), + topic_id: req.topic_id, + submission_digest: req.submission_digest, + artifact_digest: req.artifact_digest, sandboxed, network: "none".into(), flops_used: flops, From c99678485978d8e882421b2a065edb1ed7bb5005 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:19:46 +0000 Subject: [PATCH 09/12] fix(proof-fc-host): release jails on failed boots and cut sisters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 "Clean failed boots" (lib.rs ~259): from jail::prepare on, the jail is owned by a JailGuard until the VM is registered. A failure at TAP setup, rules load, spawn, guest hello or staging destroys the process, the TAP, the nftables table and the jail directory before the error returns (boot_verified); a dropped request releases them through Drop. prepare() itself removes a half-built jail and still refuses to touch one that already exists. Greptile P1 "Clean cancelled sisters" (lib.rs ~343): run_job no longer aborts the sister task. It fires a CancellationToken (also via a drop guard) and waits for the task, and sister::run races the guest against that token and always runs jail.destroy() — kill + rm — before it returns, so a timed-out or finished job never leaves a sister jail or scratch on the host. HvError::Cancelled names the cut run. Greptile P1 "Bind sister evidence": serve_sisters carries the paid job's EvidenceBinding; check_request refuses a SisterRequest naming any other topic / submission / artefact before a jail is built, and the attestation copies those verified identities. Greptile P1 "Recover dead VMs": alive() = child.try_wait() is None. Tests (no Firecracker, no KVM): injected ip-tuntap and nft-f failures and a never-answering stand-in process all end in rm -rf of the jail; a dropped guard releases on the runtime; a cancelled sister is destroyed before run() returns; mismatched sister requests never prepare a jail. Co-authored-by: Mathis --- Cargo.lock | 1 + crates/proof-fc-host/Cargo.toml | 1 + crates/proof-fc-host/src/jail.rs | 274 +++++++++++++++++++++++- crates/proof-fc-host/src/lib.rs | 330 ++++++++++++++++++++++------- crates/proof-fc-host/src/shell.rs | 53 +++++ crates/proof-fc-host/src/sister.rs | 287 ++++++++++++++++++++----- 6 files changed, 814 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa6402413..eee09722b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3555,6 +3555,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "tokio-util", "tracing", ] diff --git a/crates/proof-fc-host/Cargo.toml b/crates/proof-fc-host/Cargo.toml index ad7dac5b1..7a4d90e5f 100644 --- a/crates/proof-fc-host/Cargo.toml +++ b/crates/proof-fc-host/Cargo.toml @@ -19,6 +19,7 @@ serde_json = "1" sha2 = "0.10" thiserror = "2" tokio = { version = "1", features = ["fs", "io-util", "net", "process", "rt", "sync", "time", "macros"] } +tokio-util = { version = "0.7", default-features = false } tracing = "0.1" [dev-dependencies] diff --git a/crates/proof-fc-host/src/jail.rs b/crates/proof-fc-host/src/jail.rs index 03d34097a..015ec9a43 100644 --- a/crates/proof-fc-host/src/jail.rs +++ b/crates/proof-fc-host/src/jail.rs @@ -5,9 +5,15 @@ //! rootfs copy, fresh scratch drive, `vm-config.json`) and referenced by //! jail-relative paths. Without `--daemonize` / `--new-pid-ns` the jailer //! `exec`s into Firecracker, so the child handle we hold **is** the VM. +//! +//! From [`prepare`] until the VM is registered (or, for a sister, until its +//! run ends) the jail is owned by a [`JailGuard`]: every failure path, a +//! cancelled task, or a dropped request destroys the jail — process, TAP, +//! nftables table, directory — so nothing a boot started is left behind. use std::path::{Path, PathBuf}; use std::process::Stdio; +use std::sync::Arc; use proof_vm_agent::HvError; use proof_vm_proto::guest::GUEST_CID; @@ -120,7 +126,9 @@ pub fn jailer_args(cfg: &HostConfig, id: &str) -> Vec { /// Build the jail root for `boot`: copies (reflink when the filesystem can), /// a fresh ext4 scratch drive, ownership for the jail uid, the config file, -/// and the per-VM nftables ruleset beside the root (never inside it). +/// and the per-VM nftables ruleset beside the root (never inside it). A step +/// that fails removes whatever was already built; a jail that already exists +/// is refused and left alone. /// /// # Errors /// @@ -131,10 +139,30 @@ pub async fn prepare( boot: &VmBoot, ) -> Result { let root = cfg.jail_root(&boot.id); - let root_s = root.display().to_string(); if root.exists() { - return Err(HvError::Backend(format!("jail {root_s} already exists"))); + return Err(HvError::Backend(format!( + "jail {} already exists", + root.display() + ))); + } + match build(cfg, shell, boot, &root).await { + Ok(()) => Ok(root), + Err(e) => { + if let Err(rm) = destroy(cfg, shell, &boot.id).await { + tracing::warn!(jail = %boot.id, "half-built jail not removed: {rm}"); + } + Err(e) + } } +} + +async fn build( + cfg: &HostConfig, + shell: &dyn Shell, + boot: &VmBoot, + root: &Path, +) -> Result<(), HvError> { + let root_s = root.display().to_string(); sh(shell, "mkdir", &["-p", &format!("{root_s}/run")]).await?; let kernel_src = cfg.kernel.display().to_string(); sh( @@ -177,7 +205,7 @@ pub async fn prepare( } let owner = format!("{}:{}", cfg.jail_uid, cfg.jail_gid); sh(shell, "chown", &["-R", &owner, &root_s]).await?; - Ok(root) + Ok(()) } fn write(path: &Path, bytes: &[u8]) -> Result<(), HvError> { @@ -247,6 +275,141 @@ pub async fn retain(cfg: &HostConfig, shell: &dyn Shell, id: &str) -> Result, + shell: Arc, + id: String, + net: Option, + child: Option, +) { + if let Some(mut child) = child { + kill(&mut child).await; + } + if let Some(net) = net { + for e in net.down(shell.as_ref()).await { + tracing::debug!(jail = %id, "network teardown on release: {e}"); + } + } + if let Err(e) = destroy(&cfg, shell.as_ref(), &id).await { + tracing::warn!(jail = %id, "jail not removed on release: {e}"); + } else { + tracing::info!(jail = %id, "jail released"); + } +} + +/// Owns a prepared jail — and, once spawned, its VM process — until it is +/// either handed over ([`keep`](Self::keep)) or torn down +/// ([`destroy`](Self::destroy)). Dropping an armed guard (an error before +/// the guest handshake, a cancelled sister task, a request the client gave +/// up on) releases everything on the runtime instead, so a boot that did not +/// finish never leaves a jail directory, a scratch drive, a TAP, or an +/// nftables table behind. +pub struct JailGuard { + cfg: Arc, + shell: Arc, + id: String, + root: PathBuf, + net: Option, + child: Option, + armed: bool, +} + +impl JailGuard { + /// [`prepare`] the jail for `boot` and take ownership of it. + /// + /// # Errors + /// + /// [`HvError::Backend`]; nothing is left behind. + pub async fn prepare( + cfg: Arc, + shell: Arc, + boot: &VmBoot, + ) -> Result { + let root = prepare(&cfg, shell.as_ref(), boot).await?; + Ok(Self { + cfg, + shell, + id: boot.id.clone(), + root, + net: boot.net.clone(), + child: None, + armed: true, + }) + } + + /// Jail id. + #[must_use] + pub fn id(&self) -> &str { + &self.id + } + + /// Jail root (`/firecracker//root`). + #[must_use] + pub fn root(&self) -> &Path { + &self.root + } + + /// [`spawn`] the VM process into this jail. A guard holds at most one. + /// + /// # Errors + /// + /// [`HvError::Backend`]; the guard stays armed, so the jail is still released. + pub fn spawn(&mut self) -> Result<(), HvError> { + if self.child.is_some() { + return Err(HvError::Backend(format!( + "jail {} already has a process", + self.id + ))); + } + self.child = Some(spawn(&self.cfg, &self.id)?); + Ok(()) + } + + /// Hand the jail and its process over: the caller now owns both and the + /// guard does nothing more. + #[must_use] + pub fn keep(mut self) -> Option { + self.armed = false; + self.child.take() + } + + /// Kill the process, tear the network down, remove the jail — now, and + /// to completion. + pub async fn destroy(mut self) { + self.armed = false; + release( + self.cfg.clone(), + self.shell.clone(), + std::mem::take(&mut self.id), + self.net.take(), + self.child.take(), + ) + .await; + } +} + +impl Drop for JailGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + let id = std::mem::take(&mut self.id); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + tracing::warn!(jail = %id, "jail dropped before hand-over; releasing"); + handle.spawn(release( + self.cfg.clone(), + self.shell.clone(), + id, + self.net.take(), + self.child.take(), + )); + } else { + tracing::error!(jail = %id, "jail dropped outside a runtime; not released"); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -372,4 +535,107 @@ mod tests { assert!(flat.iter().any(|l| l.starts_with("mv "))); let _ = std::fs::remove_dir_all(&c.chroot_base); } + + /// A step of `prepare` that fails removes what was already built; a jail + /// that already exists is refused without touching it. + #[tokio::test] + async fn a_half_built_jail_is_removed_and_an_existing_one_is_left_alone() { + let c = cfg("half"); + let shell = crate::shell::FailingShell::failing_on("mkfs.ext4"); + let err = prepare(&c, &shell, &boot(None)) + .await + .expect_err("mkfs failed"); + assert!(err.to_string().contains("mkfs.ext4"), "{err}"); + let lines = shell.lines(); + assert_eq!( + lines.last().map(String::as_str), + Some(format!("rm -rf {}", c.jail_dir("topic-a-0001").display()).as_str()), + "{lines:?}" + ); + assert!( + !lines.iter().any(|l| l.starts_with("chown")), + "stopped at mkfs" + ); + + std::fs::create_dir_all(c.jail_root("topic-a-0001")).expect("pre-existing jail"); + let fresh = RecordingShell::default(); + let err = prepare(&c, &fresh, &boot(None)).await.expect_err("exists"); + assert!(err.to_string().contains("already exists"), "{err}"); + assert!(fresh.calls().is_empty(), "not ours to remove"); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } + + /// The guard is the no-leak contract: dropped armed (an aborted task, a + /// request the client gave up on) it releases the jail on the runtime; + /// handed over with `keep` it does nothing; `destroy` releases inline. + #[tokio::test] + async fn a_dropped_guard_releases_the_jail_and_a_kept_one_does_not() { + let c = Arc::new(cfg("guard")); + let shell = Arc::new(RecordingShell::default()); + let plan = NetPlan::for_index(&c, 5); + let guard = JailGuard::prepare(c.clone(), shell.clone(), &boot(Some(plan))) + .await + .expect("prepare"); + assert_eq!(guard.id(), "topic-a-0001"); + assert_eq!(guard.root(), c.jail_root("topic-a-0001")); + let before = shell.calls().len(); + let aborted = tokio::spawn(async move { + let _held = guard; + std::future::pending::<()>().await; + }); + tokio::task::yield_now().await; + aborted.abort(); + let _ = aborted.await; + for _ in 0..50 { + if shell.calls().len() > before { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let after: Vec = shell.calls()[before..] + .iter() + .map(|c| c.join(" ")) + .collect(); + assert!( + after.contains(&"nft delete table inet proof_vm_pfc5".to_owned()), + "{after:?}" + ); + assert!(after.contains(&"ip link del pfc5".to_owned()), "{after:?}"); + assert_eq!( + after.last().map(String::as_str), + Some(format!("rm -rf {}", c.jail_dir("topic-a-0001").display()).as_str()), + "{after:?}" + ); + let _ = std::fs::remove_dir_all(&c.chroot_base); + + let c2 = Arc::new(cfg("kept")); + let shell2 = Arc::new(RecordingShell::default()); + let guard = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)) + .await + .expect("prepare"); + let before = shell2.calls().len(); + assert!(guard.keep().is_none(), "nothing spawned"); + tokio::task::yield_now().await; + assert_eq!(shell2.calls().len(), before, "a kept jail is not removed"); + + let Err(err) = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)).await else { + panic!("root still exists on disk"); + }; + assert!(err.to_string().contains("already exists"), "{err}"); + let _ = std::fs::remove_dir_all(c2.jail_root("topic-a-0001")); + let guard = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)) + .await + .expect("prepare again"); + let before = shell2.calls().len(); + guard.destroy().await; + let lines: Vec = shell2.calls()[before..] + .iter() + .map(|c| c.join(" ")) + .collect(); + assert_eq!( + lines, + vec![format!("rm -rf {}", c2.jail_dir("topic-a-0001").display())] + ); + let _ = std::fs::remove_dir_all(&c2.chroot_base); + } } diff --git a/crates/proof-fc-host/src/lib.rs b/crates/proof-fc-host/src/lib.rs index 7e0a8c815..61460d99d 100644 --- a/crates/proof-fc-host/src/lib.rs +++ b/crates/proof-fc-host/src/lib.rs @@ -13,10 +13,18 @@ //! host's own directory — the control plane never sees it ([`vsock`]); //! 4. runs jobs over vsock; while a **paid** job runs it listens for the //! RLM's sister request and boots a second microVM with **no network** -//! for the miner artefact, then attests that run ([`sister`]); +//! for the miner artefact — for that job's topic, submission, and +//! artefact only — then attests that run ([`sister`]); //! 5. tears the VM down: `Destroy` removes the jail, `Retain` moves it under //! `retain_dir` for audit. //! +//! Nothing a boot started outlives its failure: from the moment a jail is +//! prepared it is owned by a [`jail::JailGuard`] until the VM is registered +//! (or, for a sister, until its run ends), and every error, cancellation, or +//! dropped request releases the process, the TAP, the nftables table, and +//! the directory. A sister whose job ends first is cancelled cooperatively — +//! killed and destroyed before the job answers — never abandoned mid-flight. +//! //! Every host command goes through [`Shell`], so the tests in this crate //! assert the exact argv without spawning anything. Nothing in CI boots a //! VM: [`FirecrackerHypervisor::ready`] refuses on a host without @@ -46,16 +54,23 @@ use proof_vm_proto::guest::{ check_version, HostToRlm, RlmToHost, SisterAnswer, SisterRequest, StagedFile, RLM_JOB_PORT, SISTER_PORT, }; -use proof_vm_proto::{SisterAttestation, API_VERSION}; +use proof_vm_proto::{EvidenceBinding, SisterAttestation, API_VERSION}; use tokio::net::UnixListener; use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; pub use config::{EgressAllow, HostConfig, Proto}; pub use images::ImageCache; +pub use jail::JailGuard; pub use net::NetPlan; pub use shell::{RecordingShell, Shell, SystemShell}; pub use sister::SisterCtx; +/// How long a job waits for its sister task to finish killing and destroying +/// a sister after the job ended. Past this the task keeps cleaning up on its +/// own; it is never aborted mid-cleanup. +const SISTER_STOP_BUDGET: Duration = Duration::from_mins(1); + struct LiveVm { child: tokio::process::Child, root: PathBuf, @@ -154,31 +169,41 @@ impl FirecrackerHypervisor { } /// Accept sister requests from the RLM guest for one job; at most one - /// sister per job, none for jobs that run no miner code. + /// sister per job, none for jobs that run no miner code (`paid` is + /// `None`), and only for the paid job's own topic / submission / + /// artefact. Returns once `cancel` fires — after any sister in flight has + /// been killed and its jail destroyed — so the job never outruns its + /// sister's cleanup. async fn serve_sisters( ctx: Arc, listener: UnixListener, vm: BootedVm, seq: Arc, slot: Arc>>, - paid: bool, + paid: Option, + cancel: CancellationToken, ) { loop { - let Ok((stream, _)) = listener.accept().await else { + let accepted = tokio::select! { + () = cancel.cancelled() => return, + accepted = listener.accept() => accepted, + }; + let Ok((stream, _)) = accepted else { return; }; let mut ch = vsock::GuestChannel::from_stream(stream); - let answer = match ch - .recv_within::(Duration::from_mins(1)) - .await - { - Err(e) => SisterAnswer::Refused { + let received = tokio::select! { + () = cancel.cancelled() => return, + r = ch.recv_within::(Duration::from_mins(1)) => r, + }; + let answer = match (received, &paid) { + (Err(e), _) => SisterAnswer::Refused { error: e.to_string(), }, - Ok(_) if !paid => SisterAnswer::Refused { + (Ok(_), None) => SisterAnswer::Refused { error: "this job runs no miner code; no sister".into(), }, - Ok(req) => { + (Ok(req), Some(job)) => { let mut taken = slot.lock().await; if taken.is_some() { SisterAnswer::Refused { @@ -186,7 +211,7 @@ impl FirecrackerHypervisor { } } else { let n = seq.fetch_add(1, Ordering::SeqCst); - match sister::run(&ctx, &vm, n, &req).await { + match sister::run(&ctx, &vm, job, n, &req, &cancel).await { Ok((result, attestation)) => { *taken = Some(attestation); SisterAnswer::Result { result } @@ -201,8 +226,115 @@ impl FirecrackerHypervisor { if let Err(e) = ch.send(&answer).await { tracing::warn!(vm_id = %vm.vm_id, "sister answer not delivered: {e}"); } + if cancel.is_cancelled() { + return; + } } } + + /// Boot once the host is ready and the kernel + `image` verified: jail, + /// network, process, guest handshake, staging, registration. Any failure + /// after the jail exists releases the process, the TAP + nftables table, + /// and the jail directory before the error is returned; a dropped future + /// releases them through the guard. + async fn boot_verified( + &self, + vm_id: &str, + spec: &TopicVmSpec, + image: PathBuf, + ) -> Result { + let cfg = self.ctx.cfg.clone(); + let owner_files = self.owner_files()?; + let net = NetPlan::for_index(&cfg, self.net_index.fetch_add(1, Ordering::SeqCst)); + let boot = jail::VmBoot { + id: vm_id.to_owned(), + vcpus: spec.template.vcpus, + mem_mib: spec.template.mem_mib, + rootfs: image, + scratch_mib: cfg.scratch_mib, + net: Some(net.clone()), + }; + let mut jail = JailGuard::prepare(cfg.clone(), self.ctx.shell.clone(), &boot).await?; + let up = Self::bring_up( + &cfg, + self.ctx.shell.as_ref(), + &mut jail, + &net, + spec, + owner_files, + ) + .await; + if let Err(e) = up { + tracing::warn!(%vm_id, "topic vm boot failed; releasing its jail: {e}"); + jail.destroy().await; + return Err(e); + } + let root = jail.root().to_path_buf(); + let child = jail + .keep() + .ok_or_else(|| HvError::Backend(format!("vm {vm_id} has no process after boot")))?; + self.vms + .lock() + .await + .insert(vm_id.to_owned(), LiveVm { child, root, net }); + Ok(BootedVm { + vm_id: vm_id.to_owned(), + topic_id: spec.topic_id.clone(), + image_digest: spec.template.image_digest.clone(), + }) + } + + /// Everything after the jail exists, up to a guest that answered hello + /// and took its owner key material. On any error the guard the caller + /// holds still owns the jail, so the caller releases it. + async fn bring_up( + cfg: &HostConfig, + shell: &dyn Shell, + jail: &mut JailGuard, + net: &NetPlan, + spec: &TopicVmSpec, + owner_files: Vec, + ) -> Result<(), HvError> { + let vm_id = jail.id().to_owned(); + net.up(shell, cfg.jail_uid).await?; + let rules = cfg.jail_dir(&vm_id).join("net.nft").display().to_string(); + net.load_rules(shell, &rules).await?; + jail.spawn()?; + let root = jail.root(); + let mut ch = + vsock::GuestChannel::connect_within(root, RLM_JOB_PORT, cfg.boot_timeout).await?; + ch.send(&HostToRlm::Hello { + api_version: API_VERSION, + topic_id: spec.topic_id.clone(), + vm_id: vm_id.clone(), + }) + .await?; + match ch.recv_within::(cfg.boot_timeout).await? { + RlmToHost::Ready { api_version, agent } => { + check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; + tracing::info!(%vm_id, %agent, "rlm guest ready"); + } + other => { + return Err(HvError::Guest(format!( + "rlm guest answered {other:?} to hello" + ))) + } + } + if !owner_files.is_empty() { + let count = owner_files.len(); + ch.send(&HostToRlm::StageSecrets { files: owner_files }) + .await?; + match ch.recv_within::(cfg.boot_timeout).await? { + RlmToHost::Staged { count: got } if got == count => { + tracing::info!(%vm_id, count, "owner key material staged (contents not logged)"); + } + other => { + return Err(HvError::Guest(format!("staging answered {other:?}"))); + } + } + } + Ok(()) + } } #[async_trait] @@ -241,68 +373,13 @@ impl Hypervisor for FirecrackerHypervisor { .images .verify(&image, &spec.template.image_digest) .await?; - let owner_files = self.owner_files()?; - let net = NetPlan::for_index(&cfg, self.net_index.fetch_add(1, Ordering::SeqCst)); - let boot = jail::VmBoot { - id: vm_id.to_owned(), - vcpus: spec.template.vcpus, - mem_mib: spec.template.mem_mib, - rootfs: image, - scratch_mib: cfg.scratch_mib, - net: Some(net.clone()), - }; - let shell = self.ctx.shell.as_ref(); - let root = jail::prepare(&cfg, shell, &boot).await?; - net.up(shell, cfg.jail_uid).await?; - let rules = cfg.jail_dir(vm_id).join("net.nft").display().to_string(); - net.load_rules(shell, &rules).await?; - let mut child = jail::spawn(&cfg, vm_id)?; - let hello = async { - let mut ch = - vsock::GuestChannel::connect_within(&root, RLM_JOB_PORT, cfg.boot_timeout).await?; - ch.send(&HostToRlm::Hello { - api_version: API_VERSION, - topic_id: spec.topic_id.clone(), - vm_id: vm_id.to_owned(), - }) - .await?; - match ch.recv_within::(cfg.boot_timeout).await? { - RlmToHost::Ready { api_version, agent } => { - check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; - tracing::info!(%vm_id, %agent, "rlm guest ready"); - } - other => return Err(HvError::Guest(format!("rlm guest answered {other:?} to hello"))), - } - if !owner_files.is_empty() { - let count = owner_files.len(); - ch.send(&HostToRlm::StageSecrets { files: owner_files }).await?; - match ch.recv_within::(cfg.boot_timeout).await? { - RlmToHost::Staged { count: got } if got == count => { - tracing::info!(%vm_id, count, "owner key material staged (contents not logged)"); - } - other => { - return Err(HvError::Guest(format!("staging answered {other:?}"))); - } - } - } - Ok::<(), HvError>(()) - } - .await; - if let Err(e) = hello { - jail::kill(&mut child).await; - net.down(shell).await; - let _ = jail::destroy(&cfg, shell, vm_id).await; - return Err(e); - } - self.vms - .lock() - .await - .insert(vm_id.to_owned(), LiveVm { child, root, net }); - Ok(BootedVm { - vm_id: vm_id.to_owned(), - topic_id: spec.topic_id.clone(), - image_digest: spec.template.image_digest.clone(), - }) + self.boot_verified(vm_id, spec, image).await + } + + async fn alive(&self, vm: &BootedVm) -> bool { + let mut vms = self.vms.lock().await; + vms.get_mut(&vm.vm_id) + .is_some_and(|live| matches!(live.child.try_wait(), Ok(None))) } async fn run_job(&self, vm: &BootedVm, job: &VmJob) -> Result { @@ -313,9 +390,14 @@ impl Hypervisor for FirecrackerHypervisor { .ok_or_else(|| HvError::Backend(format!("vm {} is not running here", vm.vm_id)))?; live.root.clone() }; - let paid = matches!(job, VmJob::Baseline { .. } | VmJob::Evaluate { .. }); + // Only a paid job may ask for a sister, and only for its own identities. + let paid = EvidenceBinding::of_job(job); let listener = vsock::listen(&root, SISTER_PORT)?; let slot = Arc::new(Mutex::new(None)); + let cancel = CancellationToken::new(); + // Should this job be dropped mid-flight (the client gave up), the + // guard still fires the token and the sister task cleans up after itself. + let _stop_sisters = cancel.clone().drop_guard(); let sisters = tokio::spawn(Self::serve_sisters( Arc::new(SisterCtx { cfg: self.ctx.cfg.clone(), @@ -329,6 +411,7 @@ impl Hypervisor for FirecrackerHypervisor { )), slot.clone(), paid, + cancel.clone(), )); let budget = self.job_budget(job); let answer = async { @@ -340,7 +423,16 @@ impl Hypervisor for FirecrackerHypervisor { ch.recv_within::(budget).await } .await; - sisters.abort(); + // The job is over: stop serving sisters, and wait for a sister still + // in flight to be killed and its jail destroyed. Never abort the task + // — an aborted sister would leave its jail and scratch on disk. + cancel.cancel(); + if tokio::time::timeout(SISTER_STOP_BUDGET, sisters) + .await + .is_err() + { + tracing::warn!(vm_id = %vm.vm_id, "sister cleanup still running after the job; it finishes on its own"); + } let _ = std::fs::remove_file(vsock::listener_path(&root, SISTER_PORT)); let sister = slot.lock().await.take(); match answer? { @@ -468,6 +560,90 @@ mod tests { assert!(owner.is_empty()); } + /// A boot that fails after the jail is prepared — TAP setup, rules load, + /// or a guest that never says hello — releases everything it built: the + /// nftables table, the TAP, and the jail directory (with the rules file + /// and scratch inside it). Nothing is registered, nothing is alive. + #[tokio::test] + async fn a_boot_that_fails_before_the_handshake_releases_its_jail_and_network() { + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + for (tag, fail_on) in [("tap", "ip tuntap"), ("rules", "nft -f")] { + let c = cfg(tag); + let image = c.image_dir.join("rlm.ext4"); + std::fs::write(&image, b"rlm rootfs stand-in").expect("image"); + let shell = Arc::new(shell::FailingShell::failing_on(fail_on)); + let hv = FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config"); + let err = hv + .boot_verified("topic-a-0001", &spec, image) + .await + .expect_err("injected host failure"); + assert!(err.to_string().contains("injected failure"), "{tag}: {err}"); + let lines = shell.lines(); + let jail_dir = c.jail_dir("topic-a-0001").display().to_string(); + assert!( + lines.iter().any(|l| l.starts_with(fail_on)), + "{tag}: the failing step ran: {lines:?}" + ); + let failed_at = lines + .iter() + .position(|l| l.starts_with(fail_on)) + .expect("position"); + let after = &lines[failed_at + 1..]; + assert!( + after.contains(&"nft delete table inet proof_vm_pfc0".to_owned()), + "{tag}: table released: {after:?}" + ); + assert!( + after.contains(&"ip link del pfc0".to_owned()), + "{tag}: tap released: {after:?}" + ); + assert_eq!( + after.last().map(String::as_str), + Some(format!("rm -rf {jail_dir}").as_str()), + "{tag}: jail removed last: {after:?}" + ); + assert!(hv.vms.lock().await.is_empty(), "{tag}: nothing registered"); + let vm = BootedVm { + vm_id: "topic-a-0001".into(), + topic_id: req.topic_id.clone(), + image_digest: pinned_template().image_digest, + }; + assert!(!hv.alive(&vm).await, "{tag}: never alive"); + let _ = std::fs::remove_dir_all(c.chroot_base.parent().unwrap_or(&c.chroot_base)); + } + + // The process side: a stand-in "jailer" (a sleeping shell script, no + // Firecracker) that never brings a guest up. The handshake budget + // runs out, the process is killed, and the jail is released. + let c = { + let mut c = cfg("hello"); + c.boot_timeout = Duration::from_millis(400); + std::fs::write(&c.jailer_bin, b"#!/bin/sh\nexec sleep 30\n").expect("stand-in"); + std::fs::set_permissions(&c.jailer_bin, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + c + }; + let image = c.image_dir.join("rlm.ext4"); + std::fs::write(&image, b"rlm rootfs stand-in").expect("image"); + let shell = Arc::new(RecordingShell::default()); + let hv = FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config"); + let err = hv + .boot_verified("topic-a-0002", &spec, image) + .await + .expect_err("no guest ever answered"); + assert!(matches!(err, HvError::Guest(_)), "{err}"); + let lines: Vec = shell.calls().iter().map(|l| l.join(" ")).collect(); + assert!(lines.contains(&"ip link del pfc0".to_owned()), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!("rm -rf {}", c.jail_dir("topic-a-0002").display()).as_str()), + "{lines:?}" + ); + assert!(hv.vms.lock().await.is_empty()); + let _ = std::fs::remove_dir_all(c.chroot_base.parent().unwrap_or(&c.chroot_base)); + } + #[tokio::test] async fn owner_key_material_is_read_from_the_host_dir_only() { let mut c = cfg("owner"); diff --git a/crates/proof-fc-host/src/shell.rs b/crates/proof-fc-host/src/shell.rs index b7d347222..60dd9c393 100644 --- a/crates/proof-fc-host/src/shell.rs +++ b/crates/proof-fc-host/src/shell.rs @@ -111,3 +111,56 @@ pub async fn sh(shell: &dyn Shell, program: &str, args: &[&str]) -> Result = args.iter().map(|s| (*s).to_owned()).collect(); shell.run(program, &owned).await?.ok(program) } + +#[cfg(test)] +pub(crate) use failing::FailingShell; + +#[cfg(test)] +mod failing { + use super::{async_trait, CmdOutput, HvError, RecordingShell, Shell}; + + /// Records like [`RecordingShell`] but fails every command whose rendered + /// argv starts with `fail_prefix` (e.g. `"ip tuntap"`), to inject one + /// host failure and assert what the cleanup path does next. + pub struct FailingShell { + inner: RecordingShell, + fail_prefix: String, + } + + impl FailingShell { + /// Fail commands whose `program args…` string starts with `fail_prefix`. + #[must_use] + pub fn failing_on(fail_prefix: &str) -> Self { + Self { + inner: RecordingShell::default(), + fail_prefix: fail_prefix.to_owned(), + } + } + + /// Every command attempted so far, program first (failed ones included). + pub fn calls(&self) -> Vec> { + self.inner.calls() + } + + /// `calls()` joined as `program args…` lines. + pub fn lines(&self) -> Vec { + self.calls().iter().map(|c| c.join(" ")).collect() + } + } + + #[async_trait] + impl Shell for FailingShell { + async fn run(&self, program: &str, args: &[String]) -> Result { + let out = self.inner.run(program, args).await?; + let rendered = format!("{program} {}", args.join(" ")); + if rendered.starts_with(&self.fail_prefix) { + return Ok(CmdOutput { + code: Some(1), + stdout: String::new(), + stderr: format!("injected failure: {rendered}"), + }); + } + Ok(out) + } + } +} diff --git a/crates/proof-fc-host/src/sister.rs b/crates/proof-fc-host/src/sister.rs index a768abeb4..46080b0a1 100644 --- a/crates/proof-fc-host/src/sister.rs +++ b/crates/proof-fc-host/src/sister.rs @@ -2,7 +2,11 @@ //! beside the RLM VM for one paid run, with no network interface, fed the //! artefact bytes the RLM already inspected over vsock, held to the run's //! deadline, then destroyed. The host — not the RLM — knows it happened, -//! which is what [`SisterAttestation`] records. +//! which is what [`SisterAttestation`] records — for the job's own topic, +//! submission, and artefact only: a request naming any other identity is +//! refused before a jail is built. Whether the run ends, times out, or is +//! cancelled because the job it served finished first, the sister jail and +//! its scratch are destroyed before the host moves on. use std::sync::Arc; use std::time::{Duration, Instant}; @@ -11,12 +15,13 @@ use proof_vm_agent::{BootedVm, HvError}; use proof_vm_proto::guest::{ check_version, HostToMiner, MinerToHost, SisterRequest, SisterResult, MINER_PORT, }; -use proof_vm_proto::{SisterAttestation, API_VERSION}; +use proof_vm_proto::{EvidenceBinding, SisterAttestation, API_VERSION}; use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; use crate::config::{HostConfig, MAX_ARTIFACT_TAR_BYTES}; use crate::images::{image_path, ImageCache}; -use crate::jail::{self, VmBoot}; +use crate::jail::{JailGuard, VmBoot}; use crate::shell::Shell; use crate::vsock::GuestChannel; @@ -45,16 +50,31 @@ pub fn sister_id(parent_vm_id: &str, seq: u64) -> String { /// Refuse a request the host will not boot a sister for. /// +/// `job` is what the control plane asked the RLM to run; the sister must be +/// for exactly that topic, submission, and artefact, or its evidence would be +/// evidence for something else. +/// /// # Errors /// -/// [`HvError::Spec`]: wrong topic, oversized or mis-hashed artefact. -pub fn check_request(parent: &BootedVm, req: &SisterRequest) -> Result, HvError> { +/// [`HvError::Spec`]: wrong topic / submission / artefact, oversized or +/// mis-hashed artefact. +pub fn check_request( + parent: &BootedVm, + job: &EvidenceBinding, + req: &SisterRequest, +) -> Result, HvError> { if req.topic_id != parent.topic_id { return Err(HvError::Spec(format!( "sister request names topic {:?}, vm is bound to {:?}", req.topic_id, parent.topic_id ))); } + let asked = EvidenceBinding::new(&req.topic_id, &req.submission_digest, &req.artifact_digest); + if let Some((field, got, want)) = job.first_mismatch(&asked) { + return Err(HvError::Spec(format!( + "sister request names {field} {got:?}, the paid job names {want:?}" + ))); + } let tar = req .artifact_tar .bytes() @@ -84,21 +104,66 @@ fn u64_ms(d: Duration) -> u64 { u64::try_from(d.as_millis()).unwrap_or(u64::MAX) } +/// Wait for the sister's agent, hand it the run, wait for the answer. +async fn drive_guest( + cfg: &HostConfig, + root: &std::path::Path, + req: &SisterRequest, +) -> Result { + let mut ch = GuestChannel::connect_within(root, MINER_PORT, cfg.boot_timeout).await?; + match ch.recv_within::(cfg.boot_timeout).await? { + MinerToHost::Ready { api_version, .. } => { + check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; + } + other => { + return Err(HvError::Guest(format!( + "sister spoke before ready: {other:?}" + ))); + } + } + ch.send(&HostToMiner::Run { + api_version: API_VERSION, + artifact_tar: req.artifact_tar.clone(), + entrypoint: req.entrypoint.clone(), + deadline_s: req.deadline_s, + declared_flops: req.declared_flops, + seed: req.seed, + params: req.params.clone(), + }) + .await?; + let budget = Duration::from_secs(req.deadline_s).saturating_add(cfg.deadline_grace); + ch.recv_within::(budget).await +} + /// Boot a sister for `req`, run it, destroy it, and attest. /// +/// `job` binds the sister to the paid job being served; `cancel` is the +/// job's own lifetime — when it fires (the RLM answered, or the job's +/// deadline passed) the run is cut, the guest killed, and the jail destroyed +/// before this returns. The jail is destroyed on **every** exit, including a +/// dropped future ([`JailGuard`]). +/// /// # Errors /// /// [`HvError::Spec`] (refused before boot), [`HvError::Image`] (sister image /// missing / mismatched), [`HvError::Backend`] / [`HvError::Guest`] (the -/// host could not run it at all). A run cut at the deadline is **not** an -/// error: it is a result with `timed_out: true`. +/// host could not run it at all), [`HvError::Cancelled`] (the job ended +/// first). A run cut at the deadline is **not** an error: it is a result +/// with `timed_out: true`. pub async fn run( ctx: &SisterCtx, parent: &BootedVm, + job: &EvidenceBinding, seq: u64, req: &SisterRequest, + cancel: &CancellationToken, ) -> Result<(SisterResult, SisterAttestation), HvError> { - check_request(parent, req)?; + check_request(parent, job, req)?; + if cancel.is_cancelled() { + return Err(HvError::Cancelled( + "job ended before the sister booted".into(), + )); + } let cfg = &ctx.cfg; let image = image_path(&cfg.image_dir, &cfg.sister_image_digest)?; ctx.images.verify(&image, &cfg.sister_image_digest).await?; @@ -111,41 +176,23 @@ pub async fn run( scratch_mib: cfg.sister_scratch_mib, net: None, }; - let root = jail::prepare(cfg, ctx.shell.as_ref(), &boot).await?; - let mut child = jail::spawn(cfg, &id)?; + let mut jail = JailGuard::prepare(cfg.clone(), ctx.shell.clone(), &boot).await?; + if let Err(e) = jail.spawn() { + jail.destroy().await; + return Err(e); + } let started = Instant::now(); tracing::info!(sister = %id, parent = %parent.vm_id, topic_id = %parent.topic_id, "sister guest booting (no network)"); - let budget = Duration::from_secs(req.deadline_s).saturating_add(cfg.deadline_grace); - let outcome = async { - let mut ch = GuestChannel::connect_within(&root, MINER_PORT, cfg.boot_timeout).await?; - match ch.recv_within::(cfg.boot_timeout).await? { - MinerToHost::Ready { api_version, .. } => { - check_version(api_version).map_err(|e| HvError::Guest(e.to_string()))?; - } - other => { - return Err(HvError::Guest(format!( - "sister spoke before ready: {other:?}" - ))); - } - } - ch.send(&HostToMiner::Run { - api_version: API_VERSION, - artifact_tar: req.artifact_tar.clone(), - entrypoint: req.entrypoint.clone(), - deadline_s: req.deadline_s, - declared_flops: req.declared_flops, - seed: req.seed, - params: req.params.clone(), - }) - .await?; - ch.recv_within::(budget).await - } - .await; - jail::kill(&mut child).await; + let root = jail.root().to_path_buf(); + let outcome = tokio::select! { + out = drive_guest(cfg, &root, req) => out, + () = cancel.cancelled() => Err(HvError::Cancelled( + "job ended while the sister was running".into(), + )), + }; let wall_ms = u64_ms(started.elapsed()); - if let Err(e) = jail::destroy(cfg, ctx.shell.as_ref(), &id).await { - tracing::warn!(sister = %id, "sister jail cleanup: {e}"); - } + // Whatever happened, the sister is over: kill it and remove its jail now. + jail.destroy().await; let (exit_code, timed_out, stdout_tail, flops_used, outputs) = match outcome { Ok(MinerToHost::Done { exit_code, @@ -191,6 +238,9 @@ pub async fn run( let attestation = SisterAttestation { sister_vm_id: id, image_digest: cfg.sister_image_digest.clone(), + topic_id: job.topic_id.clone(), + submission_digest: job.submission_digest.clone(), + artifact_digest: job.artifact_digest.clone(), sandboxed: true, network: "none".into(), flops_used, @@ -202,7 +252,10 @@ pub async fn run( #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; + use crate::shell::RecordingShell; use proof_vm_proto::guest::StagedFile; fn parent() -> BootedVm { @@ -227,6 +280,11 @@ mod tests { } } + /// The paid job the sister is served for (same identities as `request`). + fn job(tar: &[u8]) -> EvidenceBinding { + EvidenceBinding::new("topic-a", "d", &hex::encode(Sha256::digest(tar))) + } + #[test] fn sister_ids_fit_the_jailer_and_requests_are_bound_and_hashed() { assert_eq!(sister_id("topic-a-0001", 3), "topic-a-0001-s3"); @@ -235,26 +293,72 @@ mod tests { assert!(id.len() <= MAX_JAIL_ID, "{id}"); assert!(id.ends_with("-s12")); let tar = b"tar bytes"; - check_request(&parent(), &request(tar)).expect("bound + hashed"); + check_request(&parent(), &job(tar), &request(tar)).expect("bound + hashed"); let mut other = request(tar); other.topic_id = "topic-b".into(); assert!(matches!( - check_request(&parent(), &other), + check_request(&parent(), &job(tar), &other), Err(HvError::Spec(_)) )); let mut wrong = request(tar); wrong.artifact_digest = "00".repeat(32); - let err = check_request(&parent(), &wrong).expect_err("hash"); - assert!(err.to_string().contains("hashes to"), "{err}"); + let err = check_request(&parent(), &job(tar), &wrong).expect_err("hash"); + assert!(err.to_string().contains("the paid job names"), "{err}"); let mut empty = request(b""); empty.artifact_digest = hex::encode(Sha256::digest(b"")); - assert!(check_request(&parent(), &empty).is_err()); + assert!(check_request(&parent(), &job(b""), &empty).is_err()); let mut junk = request(tar); junk.artifact_tar.bytes_b64 = "!!".into(); - assert!(check_request(&parent(), &junk).is_err()); + assert!(check_request(&parent(), &job(tar), &junk).is_err()); let mut no_entry = request(tar); no_entry.entrypoint.clear(); - assert!(check_request(&parent(), &no_entry).is_err()); + assert!(check_request(&parent(), &job(tar), &no_entry).is_err()); + // Consistent with its job on paper, but the bytes do not hash to the + // digest: the re-hash refuses it. + let mut mislabeled = request(tar); + mislabeled.artifact_digest = "00".repeat(32); + let err = check_request( + &parent(), + &EvidenceBinding::new("topic-a", "d", &"00".repeat(32)), + &mislabeled, + ) + .expect_err("hash"); + assert!(err.to_string().contains("hashes to"), "{err}"); + } + + /// A sister request for another submission or artefact than the paid job + /// is refused before any jail is built: replayed evidence cannot exist. + #[tokio::test] + async fn a_request_for_another_submission_or_artifact_never_boots() { + let tar = b"artifact b"; + let for_a = EvidenceBinding::new("topic-a", "submission-a", &"aa".repeat(32)); + let err = check_request(&parent(), &for_a, &request(tar)).expect_err("other job"); + assert!(matches!(err, HvError::Spec(_)), "{err}"); + assert!(err.to_string().contains("submission_digest"), "{err}"); + let same_submission = EvidenceBinding::new("topic-a", "d", &"aa".repeat(32)); + let err = + check_request(&parent(), &same_submission, &request(tar)).expect_err("other artefact"); + assert!(err.to_string().contains("artifact_digest"), "{err}"); + let shell = Arc::new(RecordingShell::default()); + let mut cfg = HostConfig::defaults(); + cfg.sister_image_digest = format!("sha256:{}", "bb".repeat(32)); + let ctx = SisterCtx { + cfg: Arc::new(cfg), + shell: shell.clone(), + images: Arc::new(ImageCache::default()), + }; + let err = run( + &ctx, + &parent(), + &for_a, + 1, + &request(tar), + &CancellationToken::new(), + ) + .await + .expect_err("refused"); + assert!(matches!(err, HvError::Spec(_)), "{err}"); + assert!(shell.calls().is_empty(), "no jail for a mismatched request"); } /// The host never boots a sister whose image is not on disk at the @@ -266,19 +370,100 @@ mod tests { std::env::temp_dir().join(format!("proof-fc-sister-{}", std::process::id())); std::fs::create_dir_all(&cfg.image_dir).expect("dir"); cfg.sister_image_digest = format!("sha256:{}", "bb".repeat(32)); - let shell = Arc::new(crate::shell::RecordingShell::default()); + let shell = Arc::new(RecordingShell::default()); let ctx = SisterCtx { cfg: Arc::new(cfg), shell: shell.clone(), images: Arc::new(ImageCache::default()), }; - let err = run(&ctx, &parent(), 1, &request(b"tar")) - .await - .expect_err("no image"); + let err = run( + &ctx, + &parent(), + &job(b"tar"), + 1, + &request(b"tar"), + &CancellationToken::new(), + ) + .await + .expect_err("no image"); assert!(matches!(err, HvError::Image(_)), "{err}"); assert!( shell.calls().is_empty(), "nothing prepared, nothing spawned" ); } + + /// A sister whose job ends first is cut and its jail destroyed before + /// `run` returns — the storage a cancelled evaluation used is gone. The + /// "jailer" here is a plain shell script that sleeps; no Firecracker, no + /// KVM, no VM. + #[tokio::test] + async fn a_cancelled_sister_is_killed_and_its_jail_destroyed() { + let base = + std::env::temp_dir().join(format!("proof-fc-sister-cancel-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + std::fs::create_dir_all(base.join("images")).expect("dir"); + let mut cfg = HostConfig::defaults(); + cfg.image_dir = base.join("images"); + cfg.chroot_base = base.join("jailer-root"); + cfg.kernel = base.join("vmlinux"); + cfg.kernel_digest = format!("sha256:{}", "aa".repeat(32)); + cfg.jailer_bin = base.join("jailer"); + std::fs::write(&cfg.jailer_bin, b"#!/bin/sh\nexec sleep 30\n").expect("stand-in"); + std::fs::set_permissions(&cfg.jailer_bin, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + let image_bytes = b"sister rootfs stand-in"; + let hex = hex::encode(Sha256::digest(image_bytes)); + std::fs::write( + cfg.image_dir.join(format!("sha256-{hex}.ext4")), + image_bytes, + ) + .expect("image"); + cfg.sister_image_digest = format!("sha256:{hex}"); + cfg.boot_timeout = Duration::from_secs(20); + let shell = Arc::new(RecordingShell::default()); + let ctx = SisterCtx { + cfg: Arc::new(cfg.clone()), + shell: shell.clone(), + images: Arc::new(ImageCache::default()), + }; + let cancel = CancellationToken::new(); + let tar = b"artifact"; + let started = Instant::now(); + let sister = { + let cancel = cancel.clone(); + async move { run(&ctx, &parent(), &job(tar), 7, &request(tar), &cancel).await } + }; + let canceller = async { + tokio::time::sleep(Duration::from_millis(300)).await; + cancel.cancel(); + }; + let (outcome, ()) = tokio::join!(sister, canceller); + let err = outcome.expect_err("cut by the job ending"); + assert!(matches!(err, HvError::Cancelled(_)), "{err}"); + assert!( + started.elapsed() < cfg.boot_timeout, + "did not wait for the guest handshake budget" + ); + let lines: Vec = shell.calls().iter().map(|c| c.join(" ")).collect(); + let jail_dir = cfg.jail_dir("topic-a-0001-s7").display().to_string(); + assert!( + lines + .iter() + .any(|l| l.starts_with("mkdir -p ") && l.contains("topic-a-0001-s7")), + "the sister jail was prepared: {lines:?}" + ); + assert_eq!( + lines.last().map(String::as_str), + Some(format!("rm -rf {jail_dir}").as_str()), + "destroyed before run returned: {lines:?}" + ); + assert!( + !cfg.jail_root("topic-a-0001-s7") + .join(crate::jail::VSOCK_IN_JAIL) + .exists(), + "nothing but the stand-in ever ran" + ); + let _ = std::fs::remove_dir_all(&base); + } } From 37b8d92168fcf1b17394f88afc3aa5899eb54f79 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:19:46 +0000 Subject: [PATCH 10/12] fix(proof-vm-fc): refuse orchestrator evidence that is not this job's Greptile P1 "Bind sister evidence": the client runs bind_evidence on every RunJobResponse before it accepts the stamps, so an attestation or report naming another topic / submission / artefact than the job is a VmError::Backend (503, no row) even if an agent ever emitted it. The live-agent test drives a replayed attestation through the fake agent (502 evidence_mismatch surfaces as Backend) and checks the client-side refusal on the same body. Co-authored-by: Mathis --- crates/proof-vm-fc/src/lib.rs | 23 +++++++---- crates/proof-vm-fc/tests/live_agent.rs | 56 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/proof-vm-fc/src/lib.rs b/crates/proof-vm-fc/src/lib.rs index d22c3704f..9b34c0dc3 100644 --- a/crates/proof-vm-fc/src/lib.rs +++ b/crates/proof-vm-fc/src/lib.rs @@ -24,8 +24,11 @@ //! Hard binds the client enforces on top of the agent's: a job must name //! the handle's topic before any request leaves; the agent must echo the //! same topic and VM; a created VM must report the digest that was pinned; -//! and a `firecracker_required` run must come back with the host's sister -//! attestation (`sandboxed: true`) or the output is not evidence. +//! a `firecracker_required` run must come back with the host's sister +//! attestation (`sandboxed: true`) or the output is not evidence; and that +//! attestation — like the report — must name the job's own topic, +//! submission, and artefact (`proof_vm_proto::bind_evidence`), so sister +//! evidence for one artefact never scores another. #![forbid(unsafe_code)] #![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] @@ -39,8 +42,8 @@ use proof_rlm::{ VmTemplate, RLM_VM_IMAGE_DIGEST_ENV, VM_ORCHESTRATOR_TOKEN_FILE_ENV, VM_ORCHESTRATOR_URL_ENV, }; use proof_vm_proto::{ - paths, AgentHealth, CreateVmRequest, ErrorBody, RunJobRequest, RunJobResponse, TeardownRequest, - TeardownResponse, VmRecord, API_VERSION, + bind_evidence, paths, AgentHealth, CreateVmRequest, ErrorBody, RunJobRequest, RunJobResponse, + TeardownRequest, TeardownResponse, VmRecord, API_VERSION, }; use reqwest::{Method, StatusCode}; use serde::de::DeserializeOwned; @@ -416,19 +419,23 @@ impl TopicVmOrchestrator for FirecrackerOrchestrator { } let needs_sister = job.requires_firecracker(); let timeout = self.job_timeout(&job); + let request = RunJobRequest { + topic_id: handle.topic_id.clone(), + job, + }; let resp: RunJobResponse = self .call( Method::POST, &paths::vm_jobs(&handle.vm_id), - Some(&RunJobRequest { - topic_id: handle.topic_id.clone(), - job, - }), + Some(&request), timeout, ) .await? .ok_or_else(|| backend(format!("orchestrator knows no vm {}", handle.vm_id)))?; check_echo(handle, &resp.topic_id, &resp.vm_id)?; + // The stamps are only evidence for the job they were produced for. + bind_evidence(&request.job, &resp.output, resp.sister.as_ref()) + .map_err(|e| backend(format!("orchestrator evidence is not this job's: {e}")))?; let attested = resp.sister.as_ref().is_some_and(|s| s.sandboxed); if needs_sister && !attested { return Err(backend( diff --git a/crates/proof-vm-fc/tests/live_agent.rs b/crates/proof-vm-fc/tests/live_agent.rs index bd2c8d2d1..5ba6fddc4 100644 --- a/crates/proof-vm-fc/tests/live_agent.rs +++ b/crates/proof-vm-fc/tests/live_agent.rs @@ -274,6 +274,62 @@ async fn a_firecracker_required_run_without_the_sister_attestation_is_not_eviden assert!(report.verify(&req).is_err()); } +/// Sister evidence is bound to the artefact it ran. A paid run for artefact +/// B that comes back with the attestation of artefact A is refused on both +/// sides: the agent answers 502 `evidence_mismatch` and never stamps, and a +/// client that received such a body would refuse it too. No row, no score. +#[tokio::test] +async fn replayed_sister_evidence_for_another_artifact_never_scores() { + let (agent, token) = live("replay").await; + let orch = Arc::new(client(&agent, &token, &pinned_template().image_digest)); + let runner = VmBackedRunner::new(orch.clone(), pinned_template()); + let a = request(); + let mut b = request(); + b.submission_digest = "submission-b".into(); + b.artifact_digest = "ba".repeat(32); + agent + .hypervisor + .set_sister_replay(Some(proof_vm_proto::EvidenceBinding::new( + &a.topic_id, + &a.submission_digest, + &a.artifact_digest, + ))); + let err = runner + .evaluate(&b, &token_for(&b)) + .await + .expect_err("evidence for a is not evidence for b"); + assert!(matches!(err, RunnerError::Backend(_)), "{err}"); + let text = err.to_string(); + assert!(text.contains("EvidenceMismatch"), "{text}"); + assert!(text.contains("submission_digest"), "{text}"); + + // The client's own check refuses the same body should an agent ever emit it. + let handle = orch.attach(&b.topic_id).await.expect("attach").expect("vm"); + let sister = proof_vm_proto::SisterAttestation { + sister_vm_id: format!("{}-s1", handle.vm_id), + image_digest: format!("sha256:{}", "dd".repeat(32)), + topic_id: a.topic_id.clone(), + submission_digest: a.submission_digest.clone(), + artifact_digest: a.artifact_digest.clone(), + sandboxed: true, + network: "none".into(), + flops_used: Some(1), + wall_ms: 1, + exit_code: Some(0), + }; + let job_b = VmJob::Baseline { request: b.clone() }; + let out_b = VmJobOutput::Baseline(proof_rlm::fixtures::report_for(&b, 0.5)); + assert!(proof_vm_proto::bind_evidence(&job_b, &out_b, Some(&sister)).is_err()); + + agent.hypervisor.set_sister_replay(None); + let run = runner + .evaluate(&b, &token_for(&b)) + .await + .expect("honest evidence for b scores b"); + assert!(run.report.sandboxed); + assert_eq!(run.report.submission_digest, "submission-b"); +} + #[tokio::test] async fn the_created_vm_must_run_the_pinned_image_and_a_fake_answer_is_refused() { let (agent, token) = live("pin").await; From d0835a1f18ac25801d8bb81980594f52002093e3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:19:46 +0000 Subject: [PATCH 11/12] docs(proof-vm): evidence binding, no-orphan jails, dead-vm reaping PROOF.md isolation boundary, the KVM-host runbook (verification probes, operate table, security model, limitations) and COMPLETENESS.md describe the bound SisterAttestation, the jail guard, cooperative sister cancellation and crashed-VM reaping. Co-authored-by: Mathis --- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 18 ++++++++++-- docs/runbooks/proof-vm-orchestrator.md | 40 ++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 2f095bf89..97f8b705f 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -86,7 +86,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. | -| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used`, destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index cddc423a6..a92683a83 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -426,13 +426,27 @@ the topic deadline, destroys it, and writes the `SisterAttestation`. The agent then **stamps** the report: `sandboxed` is `true` only when a sister ran, `flops_used` is the sister guest's measurement — an RLM cannot claim a sandbox the host did not boot, and a sister that measured nothing yields no -usage (503, never a substituted number). Hard binds on both sides: a job +usage (503, never a substituted number). The attestation is evidence for +**one job**: it names the topic, submission, and artefact the host verified +before booting the sister (a `SisterRequest` for any other identity is +refused before a jail exists), the agent refuses to stamp — 502 +`evidence_mismatch` — when the attestation or the report names another +identity than the job, and the client runs the same `bind_evidence` check +before accepting the stamps, so a sister that ran artefact A can never score +artefact B. Hard binds on both sides: a job must name the VM's topic (envelope **and** job) or the agent answers 409; the client refuses a job for another topic before any request, checks every echo, refuses a created VM on another digest, and refuses a `firecracker_required` run that came back without the sister attestation. Teardown honours the topic's `retain` policy (default **destroy**; `retain` -keeps the jail for audit). Deploy: `deploy/systemd/proof-vm-orchestrator.service`, +keeps the jail for audit). Nothing a boot started outlives its failure: a +boot that fails after the jail is prepared (TAP, rules, spawn, guest +handshake) releases the process, the TAP, the nftables table, and the jail; +a sister whose job ends first is cancelled cooperatively and destroyed +before the job answers; and a VM whose process exits outside a teardown is +reaped per its retain policy, recorded as `crashed`, and never advertised +as running — its topic gets a fresh VM on the next job. +Deploy: `deploy/systemd/proof-vm-orchestrator.service`, runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). CI runs the fake hypervisor only; no GitHub runner ever boots Firecracker. The generic `VmBackedRunner` turns inspect / evaluate into VM jobs; diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 4fe2b4f80..c48edc8c9 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -132,9 +132,27 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / - remove `PROOF_RLM_VM_IMAGE_DIGEST` → `PROOF_RLM_VM_IMAGE_DIGEST … missing or out of range`; - delete the RLM image file → agent `503 not_ready: image … no … on this host`; - `firecracker_required` topic whose RLM never asked for a sister → - `firecracker_required run came back without the host's sister-guest attestation`. + `firecracker_required run came back without the host's sister-guest attestation`; + - an RLM whose `SisterRequest` names another submission or artefact than + the job → agent log `sister request names submission_digest … the paid + job names …`, no sister jail is built, and the run comes back without + an attestation (same 503 as above). A hypervisor that ever presented + evidence for another identity would be a `502 evidence_mismatch` from + the agent and `orchestrator evidence is not this job's` on the CP. 4. `GET /v1/proof/topics` still leaks no holdout; `GET /v1/status` shows no URL, token, or path. +5. Cleanup probes on the KVM host (nothing a run started may outlive it): + - make `ip tuntap add` fail once (e.g. a stale `pfc` device) → the + agent logs `topic vm boot failed; releasing its jail` and + `/srv/jailer/firecracker/` is gone, along with the + `proof_vm_pfc` table; + - a paid run whose deadline passes while its sister is still up → the + sister is killed and `/srv/jailer/firecracker/-s` removed + before the job answers (log `jail released`); + - `kill -9` a topic VM's Firecracker → the next `attach` / `create` / + job / health logs `topic vm process exited outside teardown; reaping`, + the record becomes `crashed`, the jail is destroyed or retained per the + topic's policy, and the CP's next job creates a fresh VM (no 409). ## Operate @@ -144,6 +162,7 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / | Rotate the RLM image | stage `images/sha256-.ext4`, set `PROOF_RLM_VM_IMAGE_DIGEST` on the CP, restart `proof-challenge`; running VMs keep the old image until torn down | | Close a topic | the CP tears the VM down with the topic's `retain` policy (default destroy). `retain` moves `/srv/jailer/firecracker/` to `/var/lib/proof-vm/retained/` (scratch, console log, config) | | Agent restart | live VMs die with the agent (no `--daemonize`); `attach` then answers 404 and the CP's next job creates a fresh VM. Rules, checklists, and promotions live in the CP's RLM store, not in the VM | +| A topic VM crashed | nothing to do: the agent probes the process on every attach / create / job / health, reaps a dead VM per the topic's `retain` policy (destroy removes the jail; retain moves it for audit — read `console.log` there), records it `crashed`, and the CP's next job creates a fresh VM. A crashed record answers `DELETE` with `state: crashed, confirmed: true` | | Egress change | edit `PROOF_VM_AGENT_EGRESS_ALLOW`, restart the agent; existing VMs keep their table until torn down | | Inspect a VM | `nft list table inet proof_vm_pfc`, `cat /srv/jailer/firecracker//console.log`, `ls /srv/jailer/firecracker//root/` | @@ -170,6 +189,22 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / sandbox without a sister is corrected to `false` and the CP refuses the report for a `firecracker_required` topic; a sister that measured nothing yields `flops_used: null` → 503, never a substituted number. +- **Evidence is bound to its job.** The `SisterAttestation` names the + `topic_id`, `submission_digest`, and `artifact_digest` the host verified + against the paid job before it built the sister jail (a `SisterRequest` + for anything else is refused with no jail). Before stamping, the agent + checks the attestation **and** the RLM's report against the job + (`proof_vm_proto::bind_evidence`; mismatch → `502 evidence_mismatch`, + nothing stamped); the CP runs the same check before accepting. Sister + evidence for artefact A is never evidence for artefact B. +- **No orphaned host state.** A jail is owned by a guard from `prepare` + until the VM is registered (or the sister run ends): a failed TAP / rules / + spawn / handshake step, a cancelled or timed-out sister, or a request the + CP gave up on releases the process, the TAP, the nftables table, and the + directory. Sisters are cancelled cooperatively (killed + destroyed before + the job answers), never aborted mid-flight. A VM whose process died is + reaped per its retain policy and recorded `crashed`; its topic is free to + create a fresh one. - **Jailer.** Firecracker runs chrooted under `/srv/jailer/firecracker//root` as `PROOF_VM_AGENT_JAIL_UID`, with a read-only rootfs copy, a fresh scratch drive, and `/dev/kvm` + `/dev/net/tun` mknod'ed by the jailer. No @@ -177,7 +212,8 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / ## Limitations (v1) -- Agent restarts drop live VMs (state is in the CP's RLM store; the next job re-creates). +- Agent restarts drop live VMs (state is in the CP's RLM store; the next job re-creates). Jails of VMs that died with the agent are not swept at the next start; remove `/srv/jailer/firecracker/*` by hand before restarting. +- Dead VMs are detected on the next attach / create / job / health call, not by a background reaper; an idle host with a crashed VM reaps it when something asks. - One sister per paid job; a second `SisterRequest` in the same job is refused. - Allowlist entries are IPv4 CIDRs; hostnames must be resolved by the operator (allow the resolver's `:53/udp` if the RLM needs DNS). - mTLS between CP and agent is a follow-up; today the bearer file over TLS is the auth. From 68984c02e6edcf1efdc533f07c08f0300550e245 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 19:39:56 +0000 Subject: [PATCH 12/12] fix(proof-fc-host): keep the jail guard armed until the vm is registered Greptile P1 "Keep Cleanup Ownership" (lib.rs ~279): boot_verified took the registry lock after jail.keep() had disarmed the guard, so a request cancelled while that lock was contended killed the child (kill_on_drop) but left the jail directory, the TAP and the nftables table behind. The registry lock is now taken while the guard still owns the jail and nothing awaits between the hand-over and the insert; keep() returns the still-armed guard when there is no process to hand over, so no path disarms without registering. New test boots against a fake RLM guest over the jail's vsock UDS with a sleeping stand-in process, cancels the boot while the registry lock is held, and asserts the table, TAP and jail are released with nothing registered; the same boot then completes, is alive, and tears down when the lock is free. Co-authored-by: Mathis --- crates/proof-fc-host/src/jail.rs | 63 ++++++++++--- crates/proof-fc-host/src/lib.rs | 156 +++++++++++++++++++++++++++++-- 2 files changed, 198 insertions(+), 21 deletions(-) diff --git a/crates/proof-fc-host/src/jail.rs b/crates/proof-fc-host/src/jail.rs index 015ec9a43..aa4dbe647 100644 --- a/crates/proof-fc-host/src/jail.rs +++ b/crates/proof-fc-host/src/jail.rs @@ -367,11 +367,20 @@ impl JailGuard { } /// Hand the jail and its process over: the caller now owns both and the - /// guard does nothing more. - #[must_use] - pub fn keep(mut self) -> Option { - self.armed = false; - self.child.take() + /// guard does nothing more. Without a spawned process there is nothing + /// to hand over, so the guard comes back still owning the jail. + /// + /// # Errors + /// + /// The guard itself, still armed, when no process was spawned. + pub fn keep(mut self) -> Result> { + match self.child.take() { + Some(child) => { + self.armed = false; + Ok(child) + } + None => Err(Box::new(self)), + } } /// Kill the process, tear the network down, remove the jail — now, and @@ -412,6 +421,8 @@ impl Drop for JailGuard { #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; use crate::shell::RecordingShell; @@ -608,24 +619,20 @@ mod tests { ); let _ = std::fs::remove_dir_all(&c.chroot_base); + // Without a process there is nothing to hand over: `keep` gives the + // guard back, still owning the jail, and `destroy` releases inline. let c2 = Arc::new(cfg("kept")); let shell2 = Arc::new(RecordingShell::default()); let guard = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)) .await .expect("prepare"); - let before = shell2.calls().len(); - assert!(guard.keep().is_none(), "nothing spawned"); - tokio::task::yield_now().await; - assert_eq!(shell2.calls().len(), before, "a kept jail is not removed"); - + let Err(guard) = guard.keep() else { + panic!("nothing was spawned, nothing to keep"); + }; let Err(err) = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)).await else { panic!("root still exists on disk"); }; assert!(err.to_string().contains("already exists"), "{err}"); - let _ = std::fs::remove_dir_all(c2.jail_root("topic-a-0001")); - let guard = JailGuard::prepare(c2.clone(), shell2.clone(), &boot(None)) - .await - .expect("prepare again"); let before = shell2.calls().len(); guard.destroy().await; let lines: Vec = shell2.calls()[before..] @@ -636,6 +643,34 @@ mod tests { lines, vec![format!("rm -rf {}", c2.jail_dir("topic-a-0001").display())] ); + + // With a process (a sleeping stand-in, not Firecracker) `keep` hands + // it over and the guard does nothing more. + let _ = std::fs::remove_dir_all(c2.jail_root("topic-a-0001")); + let mut c3 = cfg("kept-process"); + c3.jailer_bin = c3.chroot_base.join("jailer"); + std::fs::create_dir_all(&c3.chroot_base).expect("base"); + std::fs::write(&c3.jailer_bin, b"#!/bin/sh\nexec sleep 30\n").expect("stand-in"); + std::fs::set_permissions(&c3.jailer_bin, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + let c3 = Arc::new(c3); + let shell3 = Arc::new(RecordingShell::default()); + let mut guard = JailGuard::prepare(c3.clone(), shell3.clone(), &boot(None)) + .await + .expect("prepare"); + guard.spawn().expect("stand-in spawned"); + let before = shell3.calls().len(); + let Ok(mut child) = guard.keep() else { + panic!("a process to keep"); + }; + tokio::task::yield_now().await; + assert_eq!(shell3.calls().len(), before, "a kept jail is not removed"); + assert!( + matches!(child.try_wait(), Ok(None)), + "the process is ours now" + ); + kill(&mut child).await; + let _ = std::fs::remove_dir_all(&c3.chroot_base); let _ = std::fs::remove_dir_all(&c2.chroot_base); } } diff --git a/crates/proof-fc-host/src/lib.rs b/crates/proof-fc-host/src/lib.rs index 61460d99d..6335e8f4c 100644 --- a/crates/proof-fc-host/src/lib.rs +++ b/crates/proof-fc-host/src/lib.rs @@ -270,13 +270,22 @@ impl FirecrackerHypervisor { return Err(e); } let root = jail.root().to_path_buf(); - let child = jail - .keep() - .ok_or_else(|| HvError::Backend(format!("vm {vm_id} has no process after boot")))?; - self.vms - .lock() - .await - .insert(vm_id.to_owned(), LiveVm { child, root, net }); + // Take the registry lock while the guard still owns the jail: a request + // cancelled here releases everything. Nothing awaits between the + // hand-over and the insert. + let mut vms = self.vms.lock().await; + let child = match jail.keep() { + Ok(child) => child, + Err(jail) => { + drop(vms); + jail.destroy().await; + return Err(HvError::Backend(format!( + "vm {vm_id} has no process after boot" + ))); + } + }; + vms.insert(vm_id.to_owned(), LiveVm { child, root, net }); + drop(vms); Ok(BootedVm { vm_id: vm_id.to_owned(), topic_id: spec.topic_id.clone(), @@ -644,6 +653,139 @@ mod tests { let _ = std::fs::remove_dir_all(c.chroot_base.parent().unwrap_or(&c.chroot_base)); } + /// Stand in for Firecracker's vsock UDS + the RLM guest agent: answer the + /// `CONNECT` handshake, then `Hello` with `Ready`. Bound by the caller + /// once the jail root exists. + async fn fake_rlm_guest(listener: UnixListener) { + use proof_vm_proto::guest::{read_frame, write_frame}; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + let Ok((stream, _)) = listener.accept().await else { + return; + }; + let mut stream = BufReader::new(stream); + let mut line = String::new(); + let _ = stream.read_line(&mut line).await; + assert_eq!(line, format!("CONNECT {RLM_JOB_PORT}\n")); + stream + .get_mut() + .write_all(b"OK 1073741824\n") + .await + .expect("ok"); + let hello: HostToRlm = read_frame(&mut stream).await.expect("hello"); + assert!(matches!(hello, HostToRlm::Hello { .. })); + write_frame( + stream.get_mut(), + &RlmToHost::Ready { + agent: "fake-rlm-guest".into(), + api_version: API_VERSION, + }, + ) + .await + .expect("ready"); + } + + /// Serve the fake guest as soon as the boot has prepared `root`. + fn serve_fake_guest_when_ready(root: PathBuf) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + for _ in 0..100 { + if root.is_dir() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let listener = UnixListener::bind(vsock::uds_path(&root)).expect("bind fake vsock"); + fake_rlm_guest(listener).await; + }) + } + + fn stand_in_host(tag: &str) -> HostConfig { + let mut c = cfg(tag); + c.boot_timeout = Duration::from_secs(10); + std::fs::write(&c.jailer_bin, b"#!/bin/sh\nexec sleep 30\n").expect("stand-in"); + std::fs::set_permissions(&c.jailer_bin, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + std::fs::write(c.image_dir.join("rlm.ext4"), b"rlm rootfs stand-in").expect("image"); + c + } + + /// The guest is ready and the boot is waiting for the VM registry when + /// the request is cancelled: the guard still owns the jail, so the + /// process, the network, and the directory are released — nothing is + /// registered. With the registry free the same boot completes, is alive, + /// and tears down cleanly. Stand-in process + fake guest, no Firecracker. + #[tokio::test] + async fn a_boot_cancelled_at_the_registry_still_releases_everything() { + let c = stand_in_host("registry"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + let shell = Arc::new(RecordingShell::default()); + let hv = + Arc::new(FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config")); + let held = hv.vms.lock().await; + let guest = serve_fake_guest_when_ready(c.jail_root("topic-a-0001")); + let boot = { + let hv = hv.clone(); + let spec = spec.clone(); + let image = c.image_dir.join("rlm.ext4"); + tokio::spawn(async move { hv.boot_verified("topic-a-0001", &spec, image).await }) + }; + guest.await.expect("guest answered hello"); + // The boot is now parked on the registry lock we hold. + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(!boot.is_finished(), "blocked on the registry"); + boot.abort(); + let _ = boot.await; + drop(held); + let jail_dir = c.jail_dir("topic-a-0001").display().to_string(); + for _ in 0..100 { + if shell + .calls() + .iter() + .any(|l| l.join(" ") == format!("rm -rf {jail_dir}")) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let lines: Vec = shell.calls().iter().map(|l| l.join(" ")).collect(); + assert!( + lines.contains(&"nft delete table inet proof_vm_pfc0".to_owned()), + "{lines:?}" + ); + assert!(lines.contains(&"ip link del pfc0".to_owned()), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!("rm -rf {jail_dir}").as_str()), + "{lines:?}" + ); + assert!(hv.vms.lock().await.is_empty(), "nothing registered"); + let _ = std::fs::remove_dir_all(c.jail_dir("topic-a-0001")); + + // Registry free: the boot completes and the VM is alive until torn down. + let guest = serve_fake_guest_when_ready(c.jail_root("topic-a-0002")); + let vm = hv + .boot_verified("topic-a-0002", &spec, c.image_dir.join("rlm.ext4")) + .await + .expect("boot completes"); + guest.await.expect("guest"); + assert_eq!(vm.topic_id, req.topic_id); + assert!(hv.alive(&vm).await, "stand-in process is running"); + assert_eq!(hv.vms.lock().await.len(), 1); + assert!(hv + .teardown(&vm, RetainPolicy::Destroy) + .await + .expect("teardown")); + assert!(!hv.alive(&vm).await); + assert!(hv.vms.lock().await.is_empty()); + let lines: Vec = shell.calls().iter().map(|l| l.join(" ")).collect(); + assert_eq!( + lines.last().map(String::as_str), + Some(format!("rm -rf {}", c.jail_dir("topic-a-0002").display()).as_str()), + "{lines:?}" + ); + let _ = std::fs::remove_dir_all(c.chroot_base.parent().unwrap_or(&c.chroot_base)); + } + #[tokio::test] async fn owner_key_material_is_read_from_the_host_dir_only() { let mut c = cfg("owner");