From 4959f07bb83a1f3f35f354841ac7cd28131b3fc6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:23:53 +0000 Subject: [PATCH 01/16] feat(proof): eval executor pin ceilings, topic tighten, and EvalExecutorOffer crate Co-authored-by: Mathis --- Cargo.lock | 12 + config/proof-pin.toml | 19 + crates/proof-executor/Cargo.toml | 20 + crates/proof-executor/src/lib.rs | 601 +++++++++++++++++++++++ crates/proof-executor/src/plan.rs | 359 ++++++++++++++ crates/proof-task/src/executor.rs | 161 ++++++ crates/proof-task/src/lib.rs | 11 +- crates/proof-task/src/pin.rs | 189 ++++++- crates/proof-task/src/topic.rs | 76 ++- crates/proof-task/tests/committed_pin.rs | 39 +- 10 files changed, 1480 insertions(+), 7 deletions(-) create mode 100644 crates/proof-executor/Cargo.toml create mode 100644 crates/proof-executor/src/lib.rs create mode 100644 crates/proof-executor/src/plan.rs create mode 100644 crates/proof-task/src/executor.rs diff --git a/Cargo.lock b/Cargo.lock index 40f8a38d1..e306a8db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3473,6 +3473,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "proof-executor" +version = "0.1.0" +dependencies = [ + "hex", + "proof-task", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", +] + [[package]] name = "proof-harvest" version = "0.1.0" diff --git a/config/proof-pin.toml b/config/proof-pin.toml index 4e241695a..dcd0f9b2d 100644 --- a/config/proof-pin.toml +++ b/config/proof-pin.toml @@ -36,6 +36,20 @@ # (open/closed). Miners do not bind it. No baked Qwen; `proxy_model` / # `proxy_models` stay empty. Missing/closed/judge down → 503. # +# Eval executor: the machine the digest-pinned proof-eval image is rented on. +# These are ceilings on the live `EvalExecutorOffer` (operator state, +# `PROOF_EVAL_EXECUTOR_OFFER_FILE`, rotated via POST /v1/admin/proof/executor) +# — a sibling of the RLM judge InferenceOffer, never the same document. +# `gpu_class = "1x"` is the only shape harvest may rent (rent_gpu_count ≠ 1 +# aborts before the rent). An offer may declare a shorter proof deadline than +# `max_proof_deadline_s_ceiling`, never longer; a topic may tighten it again +# (`eval_executor.max_proof_deadline_s`) and may pin the live offer's +# `config_commitment` (`eval_executor.require_offer_commitment`). No per-topic +# machine id. `allowed_lium_template_prefixes` is optional; when set, the +# offer's `lium_template_id` must carry one of the prefixes (the digest-scoped +# harvest template is `proof-eval-<12 hex of the digest>`). Missing / closed / +# shape ≠ 1x → can_score=false → 503. +# # Holdout size / stratum size are the measurement harness (5 scored splits × 24), # not the problem list. @@ -49,6 +63,11 @@ allowed_modes = ["chat", "completions", "embeddings"] max_input_tokens_ceiling = 32768 max_output_tokens_ceiling = 8192 inference_offer_commitment_alg = "sha256" +eval_executor_schema_version = 1 +gpu_class = "1x" +max_proof_deadline_s_ceiling = 7200 +allowed_lium_template_prefixes = ["proof-eval-"] +eval_executor_commitment_alg = "sha256" eval_image = "ghcr.io/cortexlm/proof-eval" eval_image_digest = "sha256:78b614a1f51ce5dd80076c4e343a2b31b85d6c36025e02836cb83929867e7009" proof_git = "https://github.com/CortexLM/cortex" diff --git a/crates/proof-executor/Cargo.toml b/crates/proof-executor/Cargo.toml new file mode 100644 index 000000000..2041d7781 --- /dev/null +++ b/crates/proof-executor/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "proof-executor" +description = "Proof eval executor offer: the 1x Lium machine class the digest-pinned proof-eval image is rented on" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +proof-task = { path = "../proof-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/proof-executor/src/lib.rs b/crates/proof-executor/src/lib.rs new file mode 100644 index 000000000..e30608875 --- /dev/null +++ b/crates/proof-executor/src/lib.rs @@ -0,0 +1,601 @@ +//! Proof eval **executor** offer: the Lium machine class the digest-pinned +//! `proof-eval` image is rented on. +//! +//! Sibling of the RLM judge [`proof_task::InferenceOffer`] — a separate +//! document with its own commitment, admin route, and pin ceilings. The +//! judge is *what* scores; the executor is *where* the proof runs. Git carries +//! only ceilings ([`proof_task::ProofPin`]: `gpu_class`, +//! `max_proof_deadline_s_ceiling`, `allowed_lium_template_prefixes`); the +//! live offer is operator state (`PROOF_EVAL_EXECUTOR_OFFER_FILE`, rotated +//! with `POST /v1/admin/proof/executor`). Miners never bind it. +//! +//! Every refusal here is fail-closed: a missing, closed, or non-`1x` offer +//! means `can_score=false` and submits answer **503**. Nothing here rents. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +mod plan; + +use proof_task::{canonical_json, is_hex64, is_slug, ProofPin, TopicDocument}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub use plan::{ + executor_plan, ExecutorPlan, HarvestOverrides, HARVEST_DEADLINE_SECS_ENV, + HARVEST_GPU_COUNT_ENV, HARVEST_TEMPLATE_ID_ENV, +}; +pub use proof_task::OfferStatus; + +/// Env var naming the live offer file (operator state, never git). +pub const EVAL_EXECUTOR_OFFER_FILE_ENV: &str = "PROOF_EVAL_EXECUTOR_OFFER_FILE"; + +/// Longest `lium_template_id` an offer may carry. +pub const MAX_TEMPLATE_ID_LEN: usize = 128; + +/// Operator live offer (`PROOF_EVAL_EXECUTOR_OFFER_FILE`). Never committed. +/// +/// Every field is public: there is no origin or credential in this document, +/// so `/v1/status` and `/v1/proof/executor` may show it whole. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EvalExecutorOffer { + /// Immutable slug identifying this executor. + pub offer_id: String, + /// Lium template the harvest rents. Either the digest-scoped template + /// name (`proof-eval-<12 hex of the digest>`, resolved or created bound + /// to `repo@digest`) or a raw Lium template UUID. The pin's + /// `allowed_lium_template_prefixes` decides which forms are legal. + pub lium_template_id: String, + /// Machine class. Must equal the pin `gpu_class` (`1x`). + pub machine_shape: String, + /// Longest proof run on this executor, seconds. `<=` the pin ceiling. + pub max_proof_deadline_s: u64, + /// Eval image digest this offer was validated for. Empty = unbound; + /// non-empty must equal the pin digest. + #[serde(default)] + pub eval_image_digest: String, + /// `sha256` hex of canonical JSON of the public knobs + /// ([`executor_config_commitment`]). + pub config_commitment: String, + /// `open` | `closed`. + pub status: OfferStatus, +} + +/// Why an executor offer was refused or cannot score. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ExecutorOfferError { + /// JSON did not parse, or carried an unknown key. + #[error("parse eval executor offer: {0}")] + Parse(String), + /// `offer_id` is not a slug. + #[error("offer_id {0:?} must match [a-z0-9][a-z0-9-]{{1,62}}")] + BadId(String), + /// `lium_template_id` is empty, oversized, or not printable ASCII. + #[error("lium_template_id must be 1..={MAX_TEMPLATE_ID_LEN} printable ASCII chars")] + BadTemplateId, + /// `lium_template_id` is outside the pin allowlist. + #[error("lium_template_id {0:?} is not under the pin allowed_lium_template_prefixes")] + TemplateNotAllowed(String), + /// A digest-scoped template name does not name the pinned digest. + #[error("lium_template_id {0:?} does not carry the pinned eval image digest prefix {1}")] + TemplateDigestMismatch(String, String), + /// Offer shape is not the pin `gpu_class`. + #[error("machine_shape {got:?} is not the pin gpu_class {want:?}; refuse scoring")] + ShapeMismatch { + /// What the offer said. + got: String, + /// The only legal shape. + want: String, + }, + /// Deadline is zero or above the ceiling. + #[error("max_proof_deadline_s = {0} must be 1..={1}")] + BadDeadline(u64, u64), + /// Offer names an eval image digest that is not the pin. + #[error("eval_image_digest does not match the pin")] + DigestMismatch, + /// Declared commitment is not 64 hex or does not match the knobs. + #[error("config_commitment does not match sha256(canonical executor config)")] + CommitmentMismatch, + /// No offer loaded on this host. + #[error("eval executor offer missing; refuse scoring")] + Missing, + /// Offer is present but closed. + #[error("eval executor offer is closed; refuse scoring")] + Closed, + /// Open offer cannot serve this topic (commitment pin mismatch). + #[error("open eval executor offer cannot serve topic eval_executor constraints")] + CannotServeTopic, + /// An operator env override is set but unusable. Fail closed, never + /// silently fall back to the offer. + #[error("{0} is set but not a usable harvest override")] + BadOverride(&'static str), + /// The rent would not be exactly the pinned width. + #[error("abort: executor would rent {0}x GPUs; the Proof executor is exactly {1}x")] + GpuCount(u32, u32), +} + +#[derive(Serialize)] +struct CommitmentMaterial<'a> { + eval_image_digest: &'a str, + lium_template_id: &'a str, + machine_shape: &'a str, + max_proof_deadline_s: u64, +} + +/// `sha256` hex of canonical JSON of the public executor knobs. +/// +/// `offer_id` and `status` are lifecycle, not configuration, and stay out. +pub fn executor_config_commitment( + lium_template_id: &str, + machine_shape: &str, + max_proof_deadline_s: u64, + eval_image_digest: &str, +) -> String { + let material = CommitmentMaterial { + eval_image_digest: eval_image_digest.trim(), + lium_template_id: lium_template_id.trim(), + machine_shape: machine_shape.trim(), + max_proof_deadline_s, + }; + let value = serde_json::to_value(&material).unwrap_or(serde_json::Value::Null); + let mut h = Sha256::new(); + h.update(canonical_json(&value).as_bytes()); + hex::encode(h.finalize()) +} + +/// GPUs behind a machine shape such as `1x` (`None` when not `x`, n >= 1). +pub fn shape_gpu_count(shape: &str) -> Option { + let n: u32 = shape.trim().strip_suffix('x')?.parse().ok()?; + (n >= 1).then_some(n) +} + +/// `8-4-4-4-12` lowercase/uppercase hex: a raw Lium template id rather than a +/// digest-scoped template name. +pub fn is_lium_template_uuid(id: &str) -> bool { + let t = id.trim(); + t.len() == 36 + && t.bytes().enumerate().all(|(i, b)| match i { + 8 | 13 | 18 | 23 => b == b'-', + _ => b.is_ascii_hexdigit(), + }) +} + +/// Template identifier shape, pin allowlist, and (for a digest-scoped name) +/// the pinned digest prefix. +pub fn check_template_id(pin: &ProofPin, template_id: &str) -> Result<(), ExecutorOfferError> { + let id = template_id.trim(); + if id.is_empty() || id.len() > MAX_TEMPLATE_ID_LEN || !id.bytes().all(|b| b.is_ascii_graphic()) + { + return Err(ExecutorOfferError::BadTemplateId); + } + if !pin.allows_template(id) { + return Err(ExecutorOfferError::TemplateNotAllowed(id.to_owned())); + } + if !is_lium_template_uuid(id) && pin.can_rent() { + let hex = pin.eval_image_digest.trim().trim_start_matches("sha256:"); + let prefix = hex.get(..12).unwrap_or(hex); + if !id.contains(prefix) { + return Err(ExecutorOfferError::TemplateDigestMismatch( + id.to_owned(), + prefix.to_owned(), + )); + } + } + Ok(()) +} + +impl EvalExecutorOffer { + /// Parse one operator offer document. + /// + /// # Errors + /// + /// [`ExecutorOfferError::Parse`] on malformed JSON or an unknown key. + pub fn from_json(body: &str) -> Result { + serde_json::from_str(body).map_err(|e| ExecutorOfferError::Parse(e.to_string())) + } + + /// The commitment this offer's knobs hash to. + pub fn expected_commitment(&self) -> String { + executor_config_commitment( + &self.lium_template_id, + &self.machine_shape, + self.max_proof_deadline_s, + &self.eval_image_digest, + ) + } + + /// Structural check against the pin. Does not require `open`. + /// + /// # Errors + /// + /// See [`ExecutorOfferError`]. A closed-but-valid offer is legal to load. + pub fn validate(&self, pin: &ProofPin) -> Result<(), ExecutorOfferError> { + if !is_slug(&self.offer_id) { + return Err(ExecutorOfferError::BadId(self.offer_id.clone())); + } + check_template_id(pin, &self.lium_template_id)?; + if self.machine_shape.trim() != pin.gpu_class.trim() { + return Err(ExecutorOfferError::ShapeMismatch { + got: self.machine_shape.clone(), + want: pin.gpu_class.clone(), + }); + } + if self.max_proof_deadline_s == 0 + || self.max_proof_deadline_s > pin.max_proof_deadline_s_ceiling + { + return Err(ExecutorOfferError::BadDeadline( + self.max_proof_deadline_s, + pin.max_proof_deadline_s_ceiling, + )); + } + let digest = self.eval_image_digest.trim(); + if !digest.is_empty() && !digest.eq_ignore_ascii_case(pin.eval_image_digest.trim()) { + return Err(ExecutorOfferError::DigestMismatch); + } + if !is_hex64(&self.config_commitment) + || !self + .config_commitment + .eq_ignore_ascii_case(&self.expected_commitment()) + { + return Err(ExecutorOfferError::CommitmentMismatch); + } + Ok(()) + } + + /// Whether this executor is open for rents. + pub fn is_open(&self) -> bool { + self.status == OfferStatus::Open + } + + /// GPUs this offer rents (`1` for `1x`). + pub fn gpu_count(&self) -> Option { + shape_gpu_count(&self.machine_shape) + } + + /// Public status payload. Every field is public; still no file paths. + pub fn public_view(&self) -> serde_json::Value { + serde_json::json!({ + "offer_id": self.offer_id, + "lium_template_id": self.lium_template_id, + "machine_shape": self.machine_shape, + "gpu_count": self.gpu_count(), + "max_proof_deadline_s": self.max_proof_deadline_s, + "eval_image_digest": self.eval_image_digest, + "config_commitment": self.config_commitment, + "status": self.status, + }) + } + + /// Whether this open executor may run `topic`. + /// + /// A topic only tightens the deadline (taken as a minimum at plan time) + /// and may pin the live offer's commitment. + /// + /// # Errors + /// + /// [`ExecutorOfferError::Closed`] or [`ExecutorOfferError::CannotServeTopic`]. + pub fn serves_topic(&self, topic: &TopicDocument) -> Result<(), ExecutorOfferError> { + if !self.is_open() { + return Err(ExecutorOfferError::Closed); + } + if let Some(need) = topic.eval_executor.require_offer_commitment.as_deref() { + if !need.trim().eq_ignore_ascii_case(&self.config_commitment) { + return Err(ExecutorOfferError::CannotServeTopic); + } + } + Ok(()) + } + + /// Proof deadline for `topic` on this executor: the offer deadline, + /// tightened by the topic when it names a shorter one. + pub fn effective_deadline_s(&self, topic: &TopicDocument) -> u64 { + topic + .eval_executor + .max_proof_deadline_s + .map_or(self.max_proof_deadline_s, |t| { + t.min(self.max_proof_deadline_s) + }) + } +} + +/// Fail-closed readiness: missing / closed / invalid offer cannot score. +/// +/// # Errors +/// +/// [`ExecutorOfferError::Missing`], [`ExecutorOfferError::Closed`], or a +/// validate error. +pub fn require_open_executor<'a>( + offer: Option<&'a EvalExecutorOffer>, + pin: &ProofPin, +) -> Result<&'a EvalExecutorOffer, ExecutorOfferError> { + let offer = offer.ok_or(ExecutorOfferError::Missing)?; + offer.validate(pin)?; + if !offer.is_open() { + return Err(ExecutorOfferError::Closed); + } + Ok(offer) +} + +#[cfg(test)] +pub(crate) mod fixtures { + use super::*; + + pub const DIGEST_HEX: &str = "78b614a1f51ce5dd80076c4e343a2b31b85d6c36025e02836cb83929867e7009"; + + pub fn pin() -> ProofPin { + ProofPin { + eval_image_digest: format!("sha256:{DIGEST_HEX}"), + topic_pubkey: "ab".repeat(32), + allowed_lium_template_prefixes: vec!["proof-eval-".into()], + ..ProofPin::default() + } + } + + pub fn offer_for(template_id: &str, deadline: u64, pin: &ProofPin) -> EvalExecutorOffer { + let mut o = EvalExecutorOffer { + offer_id: "lium-1x-v0".into(), + lium_template_id: template_id.into(), + machine_shape: "1x".into(), + max_proof_deadline_s: deadline, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: OfferStatus::Open, + }; + o.config_commitment = o.expected_commitment(); + o + } + + pub fn offer() -> EvalExecutorOffer { + offer_for("proof-eval-78b614a1f51c", 7_200, &pin()) + } +} + +#[cfg(test)] +mod tests { + use proof_task::TopicEvalExecutor; + + use super::fixtures::{offer, offer_for, pin}; + use super::*; + + #[test] + fn a_well_formed_open_offer_validates() { + let o = offer(); + o.validate(&pin()).expect("valid"); + assert!(o.is_open()); + assert_eq!(o.gpu_count(), Some(1)); + require_open_executor(Some(&o), &pin()).expect("ready"); + } + + #[test] + fn json_round_trip_and_unknown_key_refused() { + let o = offer(); + let body = serde_json::to_string(&o).expect("json"); + let back = EvalExecutorOffer::from_json(&body).expect("parse"); + assert_eq!(back, o); + back.validate(&pin()).expect("valid after round trip"); + let err = EvalExecutorOffer::from_json(r#"{"offer_id":"x","lium_api_key":"nope"}"#) + .expect_err("unknown key"); + assert!(err.to_string().contains("lium_api_key"), "{err}"); + } + + #[test] + fn commitment_is_stable_and_binds_every_knob() { + let a = executor_config_commitment("proof-eval-78b614a1f51c", "1x", 7_200, "sha256:aa"); + let b = executor_config_commitment("proof-eval-78b614a1f51c", "1x", 7_200, "sha256:aa"); + assert_eq!(a, b); + assert_eq!(a.len(), 64); + for other in [ + executor_config_commitment("proof-eval-000000000000", "1x", 7_200, "sha256:aa"), + executor_config_commitment("proof-eval-78b614a1f51c", "8x", 7_200, "sha256:aa"), + executor_config_commitment("proof-eval-78b614a1f51c", "1x", 3_600, "sha256:aa"), + executor_config_commitment("proof-eval-78b614a1f51c", "1x", 7_200, ""), + ] { + assert_ne!(a, other); + } + let mut o = offer(); + o.config_commitment = "cd".repeat(32); + assert!(matches!( + o.validate(&pin()), + Err(ExecutorOfferError::CommitmentMismatch) + )); + o.config_commitment = "not-hex".into(); + assert!(matches!( + o.validate(&pin()), + Err(ExecutorOfferError::CommitmentMismatch) + )); + } + + #[test] + fn missing_or_closed_offer_cannot_score() { + assert!(matches!( + require_open_executor(None, &pin()), + Err(ExecutorOfferError::Missing) + )); + let mut closed = offer(); + closed.status = OfferStatus::Closed; + closed.validate(&pin()).expect("closed may load"); + assert!(matches!( + require_open_executor(Some(&closed), &pin()), + Err(ExecutorOfferError::Closed) + )); + assert!(matches!( + closed.serves_topic(&TopicDocument::default()), + Err(ExecutorOfferError::Closed) + )); + } + + #[test] + fn any_shape_but_the_pin_gpu_class_is_refused() { + for shape in ["8x", "2x", "1", "", "1X"] { + let mut o = offer(); + o.machine_shape = shape.into(); + o.config_commitment = o.expected_commitment(); + assert!( + matches!( + o.validate(&pin()), + Err(ExecutorOfferError::ShapeMismatch { .. }) + ), + "{shape:?} must not score on a 1x pin" + ); + } + assert_eq!(shape_gpu_count("1x"), Some(1)); + assert_eq!(shape_gpu_count("8x"), Some(8)); + assert_eq!(shape_gpu_count("0x"), None); + assert_eq!(shape_gpu_count("x"), None); + assert_eq!(shape_gpu_count("b200"), None); + } + + #[test] + fn deadline_cannot_loosen_the_pin_ceiling() { + let p = pin(); + let over = offer_for( + "proof-eval-78b614a1f51c", + p.max_proof_deadline_s_ceiling + 1, + &p, + ); + assert!(matches!( + over.validate(&p), + Err(ExecutorOfferError::BadDeadline(7_201, 7_200)) + )); + let zero = offer_for("proof-eval-78b614a1f51c", 0, &p); + assert!(matches!( + zero.validate(&p), + Err(ExecutorOfferError::BadDeadline(0, 7_200)) + )); + let mut tight = p.clone(); + tight.max_proof_deadline_s_ceiling = 3_600; + assert!(matches!( + offer().validate(&tight), + Err(ExecutorOfferError::BadDeadline(7_200, 3_600)) + )); + offer_for("proof-eval-78b614a1f51c", 3_600, &tight) + .validate(&tight) + .expect("at the tightened ceiling"); + } + + #[test] + fn digest_must_match_the_pin_when_present() { + let p = pin(); + let mut o = offer(); + o.eval_image_digest = format!("sha256:{}", "ab".repeat(32)); + o.config_commitment = o.expected_commitment(); + assert!(matches!( + o.validate(&p), + Err(ExecutorOfferError::DigestMismatch) + )); + o.eval_image_digest = String::new(); + o.config_commitment = o.expected_commitment(); + o.validate(&p).expect("unbound digest is legal"); + o.eval_image_digest = p.eval_image_digest.to_ascii_uppercase(); + o.config_commitment = o.expected_commitment(); + o.validate(&p).expect("case-insensitive digest"); + } + + #[test] + fn template_id_obeys_the_pin_allowlist_and_the_pinned_digest() { + let p = pin(); + assert!(matches!( + offer_for("prism-recipe-v10", 600, &p).validate(&p), + Err(ExecutorOfferError::TemplateNotAllowed(_)) + )); + assert!(matches!( + offer_for("proof-eval-000000000000", 600, &p).validate(&p), + Err(ExecutorOfferError::TemplateDigestMismatch(..)) + )); + for bad in [ + "", + "has space", + "tab\tid", + &"a".repeat(MAX_TEMPLATE_ID_LEN + 1), + ] { + let mut o = offer(); + o.lium_template_id = bad.to_owned(); + o.config_commitment = o.expected_commitment(); + assert!( + matches!(o.validate(&p), Err(ExecutorOfferError::BadTemplateId)), + "{bad:?}" + ); + } + // A raw Lium UUID is legal only when the pin allowlist admits it. + let uuid = "f2f5e84c-3b09-4090-be83-1913eabd009e"; + assert!(is_lium_template_uuid(uuid)); + assert!(!is_lium_template_uuid("proof-eval-78b614a1f51c")); + assert!(!is_lium_template_uuid( + "f2f5e84c-3b09-4090-be83-1913eabd009" + )); + assert!(matches!( + offer_for(uuid, 600, &p).validate(&p), + Err(ExecutorOfferError::TemplateNotAllowed(_)) + )); + let mut open = p.clone(); + open.allowed_lium_template_prefixes.clear(); + offer_for(uuid, 600, &open) + .validate(&open) + .expect("uuid under an empty allowlist"); + // Unpinned digest (pre-launch): the name cannot be checked against + // a digest, and the host cannot rent anyway. + let mut unpinned = p.clone(); + unpinned.eval_image_digest.clear(); + offer_for("proof-eval-deadbeef0000", 600, &unpinned) + .validate(&unpinned) + .expect("pre-launch pin skips the digest-name check"); + } + + #[test] + fn bad_offer_id_is_refused() { + let mut o = offer(); + o.offer_id = "Bad Id".into(); + assert!(matches!( + o.validate(&pin()), + Err(ExecutorOfferError::BadId(_)) + )); + } + + #[test] + fn public_view_shows_every_field_and_no_paths() { + let v = offer().public_view(); + assert_eq!(v["offer_id"], "lium-1x-v0"); + assert_eq!(v["lium_template_id"], "proof-eval-78b614a1f51c"); + assert_eq!(v["machine_shape"], "1x"); + assert_eq!(v["gpu_count"], 1); + assert_eq!(v["max_proof_deadline_s"], 7_200); + assert_eq!(v["status"], "open"); + assert!(v["config_commitment"] + .as_str() + .is_some_and(|c| c.len() == 64)); + let dump = v.to_string(); + assert!(!dump.contains("/run/base"), "{dump}"); + assert!(!dump.contains("api_key"), "{dump}"); + } + + #[test] + fn topic_commitment_pin_and_deadline_tighten() { + let o = offer(); + let mut topic = TopicDocument::default(); + o.serves_topic(&topic).expect("no constraints"); + assert_eq!(o.effective_deadline_s(&topic), 7_200); + topic.eval_executor = TopicEvalExecutor { + require_offer_commitment: Some(o.config_commitment.to_ascii_uppercase()), + max_proof_deadline_s: Some(1_800), + }; + o.serves_topic(&topic).expect("matching commitment"); + assert_eq!(o.effective_deadline_s(&topic), 1_800); + topic.eval_executor.max_proof_deadline_s = Some(9_999); + assert_eq!( + o.effective_deadline_s(&topic), + 7_200, + "a topic never loosens the offer deadline" + ); + topic.eval_executor.require_offer_commitment = Some("ab".repeat(32)); + assert!(matches!( + o.serves_topic(&topic), + Err(ExecutorOfferError::CannotServeTopic) + )); + } +} diff --git a/crates/proof-executor/src/plan.rs b/crates/proof-executor/src/plan.rs new file mode 100644 index 000000000..830061b07 --- /dev/null +++ b/crates/proof-executor/src/plan.rs @@ -0,0 +1,359 @@ +//! Resolved rent plan: pin ceilings + live offer + topic tighten + operator +//! env hot-swap, collapsed into what one harvest is allowed to do. +//! +//! The env overrides exist so an operator can swap the template, width, or +//! deadline without a rebuild. They replace the **offer's** values; the pin +//! ceilings still bind, and a value outside them refuses the rent instead of +//! being clamped. An unparseable override is also a refusal — never a silent +//! fall back to the offer. + +use proof_task::{ProofPin, TopicDocument, EVAL_EXECUTOR_GPU_COUNT}; + +use crate::{ + check_template_id, is_lium_template_uuid, require_open_executor, shape_gpu_count, + EvalExecutorOffer, ExecutorOfferError, +}; + +/// Env override: Lium template id / digest-scoped template name to rent. +pub const HARVEST_TEMPLATE_ID_ENV: &str = "PROOF_HARVEST_TEMPLATE_ID"; + +/// Env override: GPUs to rent. Anything but the pinned width aborts. +pub const HARVEST_GPU_COUNT_ENV: &str = "PROOF_HARVEST_GPU_COUNT"; + +/// Env override: proof deadline in seconds. Still `<=` the pin ceiling. +pub const HARVEST_DEADLINE_SECS_ENV: &str = "PROOF_HARVEST_DEADLINE_SECS"; + +/// Operator env hot-swap of the executor plan. `None` = use the offer. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HarvestOverrides { + /// [`HARVEST_TEMPLATE_ID_ENV`]. + pub template_id: Option, + /// [`HARVEST_GPU_COUNT_ENV`]. + pub gpu_count: Option, + /// [`HARVEST_DEADLINE_SECS_ENV`]. + pub deadline_secs: Option, +} + +fn env_value(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +impl HarvestOverrides { + /// Read the three `PROOF_HARVEST_*` variables from the process env. + /// + /// # Errors + /// + /// [`ExecutorOfferError::BadOverride`] when a set variable is unparseable. + pub fn from_env() -> Result { + Self::parse( + env_value(HARVEST_TEMPLATE_ID_ENV).as_deref(), + env_value(HARVEST_GPU_COUNT_ENV).as_deref(), + env_value(HARVEST_DEADLINE_SECS_ENV).as_deref(), + ) + } + + /// Pure core of [`Self::from_env`]. Blank values are unset. + /// + /// # Errors + /// + /// [`ExecutorOfferError::BadOverride`] when a value is present but unparseable. + pub fn parse( + template_id: Option<&str>, + gpu_count: Option<&str>, + deadline_secs: Option<&str>, + ) -> Result { + fn blank(v: Option<&str>) -> Option<&str> { + v.map(str::trim).filter(|s| !s.is_empty()) + } + let gpu_count = match blank(gpu_count) { + Some(raw) => Some( + raw.parse::() + .map_err(|_| ExecutorOfferError::BadOverride(HARVEST_GPU_COUNT_ENV))?, + ), + None => None, + }; + let deadline_secs = match blank(deadline_secs) { + Some(raw) => Some( + raw.parse::() + .map_err(|_| ExecutorOfferError::BadOverride(HARVEST_DEADLINE_SECS_ENV))?, + ), + None => None, + }; + Ok(Self { + template_id: blank(template_id).map(str::to_owned), + gpu_count, + deadline_secs, + }) + } + + /// True when no override is set. + pub fn is_empty(&self) -> bool { + self.template_id.is_none() && self.gpu_count.is_none() && self.deadline_secs.is_none() + } +} + +/// What one harvest rent is allowed to do. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExecutorPlan { + /// Offer this plan was resolved from. + pub offer_id: String, + /// Lium template id or digest-scoped template name to rent. + pub template_id: String, + /// `true` when `template_id` is a raw Lium UUID (rent verbatim) rather + /// than a template name to resolve / create bound to the pinned digest. + pub template_is_uuid: bool, + /// Exact GPUs to rent. Always [`EVAL_EXECUTOR_GPU_COUNT`]. + pub gpu_count: u32, + /// Proof deadline the pod-side `timeout` and the harvest wait enforce. + pub deadline_s: u64, + /// Offer `config_commitment` stamped onto the run request. + pub config_commitment: String, +} + +/// Resolve the plan for scoring `topic` on `offer` under `pin`. +/// +/// # Errors +/// +/// Any [`ExecutorOfferError`]: the offer must be open and valid, the +/// template legal, the width exactly the pin, and the deadline within the +/// ceiling after the topic tighten and the operator override. +pub fn executor_plan( + pin: &ProofPin, + offer: Option<&EvalExecutorOffer>, + topic: &TopicDocument, + overrides: &HarvestOverrides, +) -> Result { + let offer = require_open_executor(offer, pin)?; + offer.serves_topic(topic)?; + + let template_id = overrides + .template_id + .as_deref() + .map(str::trim) + .unwrap_or(offer.lium_template_id.trim()) + .to_owned(); + check_template_id(pin, &template_id)?; + + let want = shape_gpu_count(&pin.gpu_class).unwrap_or(EVAL_EXECUTOR_GPU_COUNT); + let gpu_count = overrides + .gpu_count + .or_else(|| offer.gpu_count()) + .unwrap_or(0); + if gpu_count != want || want != EVAL_EXECUTOR_GPU_COUNT { + return Err(ExecutorOfferError::GpuCount( + gpu_count, + EVAL_EXECUTOR_GPU_COUNT, + )); + } + + let base = overrides + .deadline_secs + .unwrap_or(offer.max_proof_deadline_s); + let deadline_s = topic + .eval_executor + .max_proof_deadline_s + .map_or(base, |t| t.min(base)); + if deadline_s == 0 || deadline_s > pin.max_proof_deadline_s_ceiling { + return Err(ExecutorOfferError::BadDeadline( + deadline_s, + pin.max_proof_deadline_s_ceiling, + )); + } + + Ok(ExecutorPlan { + offer_id: offer.offer_id.clone(), + template_is_uuid: is_lium_template_uuid(&template_id), + template_id, + gpu_count, + deadline_s, + config_commitment: offer.config_commitment.clone(), + }) +} + +#[cfg(test)] +mod tests { + use proof_task::{OfferStatus, TopicEvalExecutor}; + + use super::*; + use crate::fixtures::{offer, offer_for, pin}; + + fn topic() -> TopicDocument { + TopicDocument::default() + } + + #[test] + fn plan_uses_the_open_offer_template_deadline_and_exactly_one_gpu() { + let plan = executor_plan( + &pin(), + Some(&offer()), + &topic(), + &HarvestOverrides::default(), + ) + .expect("plan"); + assert_eq!(plan.offer_id, "lium-1x-v0"); + assert_eq!(plan.template_id, "proof-eval-78b614a1f51c"); + assert!(!plan.template_is_uuid); + assert_eq!(plan.gpu_count, 1); + assert_eq!(plan.deadline_s, 7_200); + assert_eq!(plan.config_commitment, offer().config_commitment); + } + + #[test] + fn missing_closed_or_wrong_shape_offer_has_no_plan() { + let none = HarvestOverrides::default(); + assert!(matches!( + executor_plan(&pin(), None, &topic(), &none), + Err(ExecutorOfferError::Missing) + )); + let mut closed = offer(); + closed.status = OfferStatus::Closed; + assert!(matches!( + executor_plan(&pin(), Some(&closed), &topic(), &none), + Err(ExecutorOfferError::Closed) + )); + let mut wide = offer(); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + assert!(matches!( + executor_plan(&pin(), Some(&wide), &topic(), &none), + Err(ExecutorOfferError::ShapeMismatch { .. }) + )); + } + + #[test] + fn topic_tightens_the_deadline_and_may_pin_the_commitment() { + let mut t = topic(); + t.eval_executor = TopicEvalExecutor { + require_offer_commitment: Some(offer().config_commitment), + max_proof_deadline_s: Some(900), + }; + let plan = + executor_plan(&pin(), Some(&offer()), &t, &HarvestOverrides::default()).expect("plan"); + assert_eq!(plan.deadline_s, 900); + t.eval_executor.require_offer_commitment = Some("ab".repeat(32)); + assert!(matches!( + executor_plan(&pin(), Some(&offer()), &t, &HarvestOverrides::default()), + Err(ExecutorOfferError::CannotServeTopic) + )); + } + + #[test] + fn overrides_swap_template_and_deadline_but_pin_ceilings_still_bind() { + let p = pin(); + let swapped = HarvestOverrides { + template_id: Some("proof-eval-78b614a1f51c-hotfix".into()), + gpu_count: Some(1), + deadline_secs: Some(3_600), + }; + let plan = executor_plan(&p, Some(&offer()), &topic(), &swapped).expect("plan"); + assert_eq!(plan.template_id, "proof-eval-78b614a1f51c-hotfix"); + assert_eq!(plan.deadline_s, 3_600); + + // The override replaces the offer deadline, so a longer one is legal + // up to the ceiling — never past it. + let short_offer = offer_for("proof-eval-78b614a1f51c", 600, &p); + let longer = HarvestOverrides { + deadline_secs: Some(7_200), + ..HarvestOverrides::default() + }; + assert_eq!( + executor_plan(&p, Some(&short_offer), &topic(), &longer) + .expect("plan") + .deadline_s, + 7_200 + ); + let past = HarvestOverrides { + deadline_secs: Some(7_201), + ..HarvestOverrides::default() + }; + assert!(matches!( + executor_plan(&p, Some(&offer()), &topic(), &past), + Err(ExecutorOfferError::BadDeadline(7_201, 7_200)) + )); + let zero = HarvestOverrides { + deadline_secs: Some(0), + ..HarvestOverrides::default() + }; + assert!(matches!( + executor_plan(&p, Some(&offer()), &topic(), &zero), + Err(ExecutorOfferError::BadDeadline(0, 7_200)) + )); + // A topic tighten still applies on top of the override. + let mut t = topic(); + t.eval_executor.max_proof_deadline_s = Some(300); + assert_eq!( + executor_plan(&p, Some(&offer()), &t, &longer) + .expect("plan") + .deadline_s, + 300 + ); + // The template override obeys the allowlist and the pinned digest. + let off_list = HarvestOverrides { + template_id: Some("prism-recipe-v10".into()), + ..HarvestOverrides::default() + }; + assert!(matches!( + executor_plan(&p, Some(&offer()), &topic(), &off_list), + Err(ExecutorOfferError::TemplateNotAllowed(_)) + )); + let mut open = p.clone(); + open.allowed_lium_template_prefixes.clear(); + let uuid = HarvestOverrides { + template_id: Some("f2f5e84c-3b09-4090-be83-1913eabd009e".into()), + ..HarvestOverrides::default() + }; + let plan = executor_plan(&open, Some(&offer()), &topic(), &uuid).expect("plan"); + assert!(plan.template_is_uuid); + } + + #[test] + fn any_gpu_count_but_one_aborts() { + for n in [0u32, 2, 8] { + let over = HarvestOverrides { + gpu_count: Some(n), + ..HarvestOverrides::default() + }; + assert!( + matches!( + executor_plan(&pin(), Some(&offer()), &topic(), &over), + Err(ExecutorOfferError::GpuCount(got, 1)) if got == n + ), + "{n}x must abort" + ); + } + let one = HarvestOverrides { + gpu_count: Some(1), + ..HarvestOverrides::default() + }; + executor_plan(&pin(), Some(&offer()), &topic(), &one).expect("1x is the pin"); + } + + #[test] + fn overrides_parse_blank_as_unset_and_garbage_as_refusal() { + let none = HarvestOverrides::parse(None, Some(" "), Some("")).expect("blank"); + assert!(none.is_empty()); + let set = HarvestOverrides::parse(Some(" tmpl-1 "), Some("1"), Some("600")).expect("set"); + assert_eq!( + set, + HarvestOverrides { + template_id: Some("tmpl-1".into()), + gpu_count: Some(1), + deadline_secs: Some(600), + } + ); + assert!(matches!( + HarvestOverrides::parse(None, Some("one"), None), + Err(ExecutorOfferError::BadOverride(HARVEST_GPU_COUNT_ENV)) + )); + assert!(matches!( + HarvestOverrides::parse(None, None, Some("-5")), + Err(ExecutorOfferError::BadOverride(HARVEST_DEADLINE_SECS_ENV)) + )); + assert_eq!(HARVEST_TEMPLATE_ID_ENV, "PROOF_HARVEST_TEMPLATE_ID"); + assert_eq!(HARVEST_GPU_COUNT_ENV, "PROOF_HARVEST_GPU_COUNT"); + assert_eq!(HARVEST_DEADLINE_SECS_ENV, "PROOF_HARVEST_DEADLINE_SECS"); + } +} diff --git a/crates/proof-task/src/executor.rs b/crates/proof-task/src/executor.rs new file mode 100644 index 000000000..b85d122fc --- /dev/null +++ b/crates/proof-task/src/executor.rs @@ -0,0 +1,161 @@ +//! Eval **executor** ceilings: the machine class the digest-pinned +//! `proof-eval` image is rented on, and what a topic may tighten about it. +//! +//! This is the pin side only. The live `EvalExecutorOffer` (operator state, +//! off git) lives in the `proof-executor` crate. It is a sibling of the RLM +//! judge [`crate::InferenceOffer`], never the same document: the judge is +//! *what* scores, the executor is *where* the proof runs. + +use serde::{Deserialize, Serialize}; + +use crate::{is_hex64, ProofPin, TopicError}; + +/// Pin `eval_executor_schema_version`. +pub const EVAL_EXECUTOR_SCHEMA_VERSION: u32 = 1; + +/// Pin `gpu_class`: the only machine shape a Proof executor may rent. +pub const EVAL_EXECUTOR_GPU_CLASS: &str = "1x"; + +/// GPUs behind [`EVAL_EXECUTOR_GPU_CLASS`]. Harvest aborts any other width. +pub const EVAL_EXECUTOR_GPU_COUNT: u32 = 1; + +/// Pin `max_proof_deadline_s_ceiling`: longest proof deadline an offer or a +/// topic may declare (two hours). +pub const MAX_PROOF_DEADLINE_S_CEILING: u64 = 7_200; + +/// Pin `eval_executor_commitment_alg`. +pub const EVAL_EXECUTOR_COMMITMENT_ALG: &str = "sha256"; + +/// Topic override of the executor contract. **Tighten-only**, and there is +/// deliberately no `machine_id`: a topic names how long a proof may take and +/// which committed executor may run it, never a specific machine. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct TopicEvalExecutor { + /// Optional 64-hex pin of the live executor offer `config_commitment`. + /// Not a miner-facing bind. + #[serde(skip_serializing_if = "Option::is_none")] + pub require_offer_commitment: Option, + /// Proof deadline shorter than the pin ceiling (and the live offer). + #[serde(skip_serializing_if = "Option::is_none")] + pub max_proof_deadline_s: Option, +} + +impl TopicEvalExecutor { + /// True when the topic tightens nothing. Such a topic serializes without + /// an `eval_executor` key, so documents signed before this field existed + /// keep verifying. + pub fn is_empty(&self) -> bool { + self.require_offer_commitment.is_none() && self.max_proof_deadline_s.is_none() + } + + /// Tighten-only check against the pin ceiling. + /// + /// # Errors + /// + /// [`TopicError::BadExecutorCommitment`] or [`TopicError::ExecutorDeadlineCeiling`]. + pub fn validate(&self, pin: &ProofPin) -> Result<(), TopicError> { + if let Some(need) = self.require_offer_commitment.as_deref() { + if !is_hex64(need) { + return Err(TopicError::BadExecutorCommitment); + } + } + if let Some(deadline) = self.max_proof_deadline_s { + if deadline == 0 || deadline > pin.max_proof_deadline_s_ceiling { + return Err(TopicError::ExecutorDeadlineCeiling( + deadline, + pin.max_proof_deadline_s_ceiling, + )); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pin() -> ProofPin { + ProofPin { + topic_pubkey: "ab".repeat(32), + ..ProofPin::default() + } + } + + #[test] + fn locked_executor_constants() { + assert_eq!(EVAL_EXECUTOR_SCHEMA_VERSION, 1); + assert_eq!(EVAL_EXECUTOR_GPU_CLASS, "1x"); + assert_eq!(EVAL_EXECUTOR_GPU_COUNT, 1); + assert_eq!(MAX_PROOF_DEADLINE_S_CEILING, 7_200); + assert_eq!(EVAL_EXECUTOR_COMMITMENT_ALG, "sha256"); + } + + #[test] + fn empty_override_validates_and_is_omitted_from_json() { + let t = TopicEvalExecutor::default(); + assert!(t.is_empty()); + t.validate(&pin()).expect("nothing to tighten"); + assert_eq!(serde_json::to_string(&t).expect("json"), "{}"); + } + + #[test] + fn topic_may_tighten_the_deadline_never_loosen_it() { + let mut t = TopicEvalExecutor { + max_proof_deadline_s: Some(1_800), + ..TopicEvalExecutor::default() + }; + t.validate(&pin()).expect("tighten"); + t.max_proof_deadline_s = Some(MAX_PROOF_DEADLINE_S_CEILING); + t.validate(&pin()).expect("equal to ceiling is legal"); + t.max_proof_deadline_s = Some(MAX_PROOF_DEADLINE_S_CEILING + 1); + assert!(matches!( + t.validate(&pin()), + Err(TopicError::ExecutorDeadlineCeiling(..)) + )); + t.max_proof_deadline_s = Some(0); + assert!(matches!( + t.validate(&pin()), + Err(TopicError::ExecutorDeadlineCeiling(..)) + )); + } + + #[test] + fn a_tightened_pin_ceiling_binds_the_topic() { + let mut p = pin(); + p.max_proof_deadline_s_ceiling = 3_600; + p.validate().expect("pin may tighten its own ceiling"); + let t = TopicEvalExecutor { + max_proof_deadline_s: Some(3_601), + ..TopicEvalExecutor::default() + }; + assert!(matches!( + t.validate(&p), + Err(TopicError::ExecutorDeadlineCeiling(3_601, 3_600)) + )); + } + + #[test] + fn commitment_pin_must_be_hex64() { + let mut t = TopicEvalExecutor { + require_offer_commitment: Some("ab".repeat(32)), + ..TopicEvalExecutor::default() + }; + t.validate(&pin()).expect("hex64"); + t.require_offer_commitment = Some("not-hex".into()); + assert!(matches!( + t.validate(&pin()), + Err(TopicError::BadExecutorCommitment) + )); + } + + #[test] + fn no_per_topic_machine_id() { + let err = serde_json::from_str::( + r#"{"machine_id":"pod-123","max_proof_deadline_s":600}"#, + ) + .expect_err("machine_id is not a topic knob"); + assert!(err.to_string().contains("machine_id"), "{err}"); + } +} diff --git a/crates/proof-task/src/lib.rs b/crates/proof-task/src/lib.rs index 251f589b3..4a0ff18e5 100644 --- a/crates/proof-task/src/lib.rs +++ b/crates/proof-task/src/lib.rs @@ -32,11 +32,16 @@ )] mod canonical; +mod executor; mod inference; mod pin; mod topic; pub use canonical::canonical_json; +pub use executor::{ + TopicEvalExecutor, EVAL_EXECUTOR_COMMITMENT_ALG, EVAL_EXECUTOR_GPU_CLASS, + EVAL_EXECUTOR_GPU_COUNT, EVAL_EXECUTOR_SCHEMA_VERSION, MAX_PROOF_DEADLINE_S_CEILING, +}; pub use inference::{ inference_config_commitment, require_open_offer, resolve_inference, InferenceConfig, InferenceMode, InferenceOffer, InferenceProvider, InferenceProviderKind, OfferError, @@ -127,7 +132,8 @@ pub const QUALITY_FLOOR_NLL_MAX: f64 = 0.02; /// Slice id prefix bound into per-topic measurements. pub const HOLDOUT_SLICE_PREFIX: &str = "proof-holdout"; -pub(crate) fn is_hex64(s: &str) -> bool { +/// Whether `s` (trimmed) is exactly 64 hex characters. +pub fn is_hex64(s: &str) -> bool { let t = s.trim(); t.len() == 64 && t.chars().all(|c| c.is_ascii_hexdigit()) } @@ -139,7 +145,8 @@ pub(crate) fn is_http_origin(url: &str) -> bool { && !u.contains(['\n', ' ']) } -pub(crate) fn is_slug(id: &str) -> bool { +/// Whether `id` matches `[a-z0-9][a-z0-9-]{1,62}` (offer and topic ids). +pub fn is_slug(id: &str) -> bool { let b = id.as_bytes(); (2..=63).contains(&b.len()) && (b[0].is_ascii_lowercase() || b[0].is_ascii_digit()) diff --git a/crates/proof-task/src/pin.rs b/crates/proof-task/src/pin.rs index 9f6105145..06e47d4f0 100644 --- a/crates/proof-task/src/pin.rs +++ b/crates/proof-task/src/pin.rs @@ -14,9 +14,11 @@ use thiserror::Error; use crate::{ is_hex64, is_http_origin, InferenceMode, PinInference, ALLOWED_MODES, BASE_MODEL_FAMILY, CHALLENGE_ID, EPSILON_NLL_MIN, EPSILON_THROUGHPUT_REL_MIN, EPSILON_TOPIC_MAX_REGRESS_MIN, + EVAL_EXECUTOR_COMMITMENT_ALG, EVAL_EXECUTOR_GPU_CLASS, EVAL_EXECUTOR_SCHEMA_VERSION, EVAL_IMAGE, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, INFERENCE_CONFIG_SCHEMA_VERSION, INFERENCE_OFFER_COMMITMENT_ALG, MAX_INPUT_TOKENS_CEILING, MAX_OUTPUT_TOKENS_CEILING, - PROOF_GIT_URL, QUALITY_FLOOR_NLL_MAX, SCORING_VERSION, STRATUM_SIZE, + MAX_PROOF_DEADLINE_S_CEILING, PROOF_GIT_URL, QUALITY_FLOOR_NLL_MAX, SCORING_VERSION, + STRATUM_SIZE, }; /// `config/proof-pin.toml`. @@ -47,6 +49,16 @@ pub struct ProofPin { pub inference_offer_commitment_alg: String, /// Complete provider defaults. Empty model/url is pre-launch fail-closed. pub inference: PinInference, + /// Eval executor schema (`1`). Bounds the live `EvalExecutorOffer`. + pub eval_executor_schema_version: u32, + /// Machine class the proof runs on (`1x`). An offer of any other shape cannot score. + pub gpu_class: String, + /// Longest proof deadline an offer or topic may declare (seconds). + pub max_proof_deadline_s_ceiling: u64, + /// Optional allowlist of `lium_template_id` prefixes. Empty = any id. + pub allowed_lium_template_prefixes: Vec, + /// Hash algorithm for the executor `config_commitment` (`sha256`). + pub eval_executor_commitment_alg: String, /// Eval image reference (no floating tag in prod). pub eval_image: String, /// `sha256:…` digest. Empty until the first green proof-eval CI image. @@ -87,6 +99,11 @@ impl Default for ProofPin { max_output_tokens_ceiling: MAX_OUTPUT_TOKENS_CEILING, inference_offer_commitment_alg: INFERENCE_OFFER_COMMITMENT_ALG.into(), inference: PinInference::default(), + eval_executor_schema_version: EVAL_EXECUTOR_SCHEMA_VERSION, + gpu_class: EVAL_EXECUTOR_GPU_CLASS.into(), + max_proof_deadline_s_ceiling: MAX_PROOF_DEADLINE_S_CEILING, + allowed_lium_template_prefixes: Vec::new(), + eval_executor_commitment_alg: EVAL_EXECUTOR_COMMITMENT_ALG.into(), eval_image: EVAL_IMAGE.into(), eval_image_digest: String::new(), proof_git: PROOF_GIT_URL.into(), @@ -169,6 +186,27 @@ pub enum PinError { /// `[inference].base_url` is set but is not an http(s) origin. #[error("inference.base_url must be empty (secret-backed) or an http(s) origin")] BadInferenceUrl, + /// An eval-executor knob is not the locked value (schema, `gpu_class`, alg). + #[error("{field} = {got:?} is not the locked {want:?}")] + ExecutorLock { + /// Which knob. + field: &'static str, + /// What the pin said. + got: String, + /// Locked value. + want: String, + }, + /// `max_proof_deadline_s_ceiling` is zero or above the crate lock. + #[error("max_proof_deadline_s_ceiling {got} must be 1..={max}")] + DeadlineCeiling { + /// Pin value. + got: u64, + /// Locked ceiling. + max: u64, + }, + /// An `allowed_lium_template_prefixes` entry is empty, oversized, or has whitespace. + #[error("allowed_lium_template_prefixes entries must be 1..=64 chars without whitespace")] + BadTemplatePrefix, /// The topic key is not a 64-hex sr25519 public key. #[error("topic_pubkey must be 64 hex chars (the challenges.toml `proof` row key)")] BadTopicPubkey, @@ -245,6 +283,7 @@ impl ProofPin { }); } self.validate_inference()?; + self.validate_executor()?; if !is_hex64(&self.topic_pubkey) { return Err(PinError::BadTopicPubkey); } @@ -362,6 +401,57 @@ impl ProofPin { Ok(()) } + /// Executor ceilings: a pin may tighten the deadline, never the shape. + fn validate_executor(&self) -> Result<(), PinError> { + for (field, got, want) in [ + ( + "eval_executor_schema_version", + self.eval_executor_schema_version.to_string(), + EVAL_EXECUTOR_SCHEMA_VERSION.to_string(), + ), + ( + "gpu_class", + self.gpu_class.trim().to_owned(), + EVAL_EXECUTOR_GPU_CLASS.to_owned(), + ), + ( + "eval_executor_commitment_alg", + self.eval_executor_commitment_alg.trim().to_owned(), + EVAL_EXECUTOR_COMMITMENT_ALG.to_owned(), + ), + ] { + if got != want { + return Err(PinError::ExecutorLock { field, got, want }); + } + } + if self.max_proof_deadline_s_ceiling == 0 + || self.max_proof_deadline_s_ceiling > MAX_PROOF_DEADLINE_S_CEILING + { + return Err(PinError::DeadlineCeiling { + got: self.max_proof_deadline_s_ceiling, + max: MAX_PROOF_DEADLINE_S_CEILING, + }); + } + if self + .allowed_lium_template_prefixes + .iter() + .any(|p| p.is_empty() || p.len() > 64 || p.contains(char::is_whitespace)) + { + return Err(PinError::BadTemplatePrefix); + } + Ok(()) + } + + /// Whether `template_id` is legal under `allowed_lium_template_prefixes`. + /// An empty allowlist accepts any id. + pub fn allows_template(&self, template_id: &str) -> bool { + self.allowed_lium_template_prefixes.is_empty() + || self + .allowed_lium_template_prefixes + .iter() + .any(|p| template_id.starts_with(p.as_str())) + } + /// True when a live rent is allowed (real digest pin present). /// /// An empty digest is the normal pre-launch state and the reason submits @@ -416,6 +506,11 @@ allowed_modes = ["chat", "completions", "embeddings"] max_input_tokens_ceiling = 32768 max_output_tokens_ceiling = 8192 inference_offer_commitment_alg = "sha256" +eval_executor_schema_version = 1 +gpu_class = "1x" +max_proof_deadline_s_ceiling = 7200 +allowed_lium_template_prefixes = ["proof-eval-"] +eval_executor_commitment_alg = "sha256" eval_image = "{EVAL_IMAGE}" eval_image_digest = "" topic_pubkey = "{}" @@ -434,6 +529,98 @@ stratum_size = 24 assert_eq!(p.topic_pubkey_bytes().expect("key"), [0xcd; 32]); assert!(p.proxy_model.is_empty()); assert!(p.proxy_models.is_empty()); + assert_eq!(p.gpu_class, "1x"); + assert_eq!(p.max_proof_deadline_s_ceiling, 7_200); + assert_eq!(p.allowed_lium_template_prefixes, vec!["proof-eval-"]); + assert!(p.allows_template("proof-eval-78b614a1f51c")); + assert!(!p.allows_template("prism-recipe-v10")); + } + + /// A pin written before the executor keys existed still boots: the + /// defaults are the locked values and an empty allowlist. + #[test] + fn executor_keys_default_when_absent() { + let body = format!( + r#" +challenge_id = "proof" +scoring_version = 1 +base_model_family = "{BASE_MODEL_FAMILY}" +eval_image = "{EVAL_IMAGE}" +topic_pubkey = "{}" +"#, + "cd".repeat(32) + ); + let p = ProofPin::from_toml(&body).expect("parse"); + p.validate().expect("validates"); + assert_eq!(p.eval_executor_schema_version, EVAL_EXECUTOR_SCHEMA_VERSION); + assert_eq!(p.gpu_class, EVAL_EXECUTOR_GPU_CLASS); + assert_eq!(p.max_proof_deadline_s_ceiling, MAX_PROOF_DEADLINE_S_CEILING); + assert_eq!(p.eval_executor_commitment_alg, EVAL_EXECUTOR_COMMITMENT_ALG); + assert!(p.allowed_lium_template_prefixes.is_empty()); + assert!(p.allows_template("any-template-id")); + } + + #[test] + fn executor_shape_schema_and_alg_are_locked() { + let mut p = pin(); + p.gpu_class = "8x".into(); + assert!( + matches!( + p.validate(), + Err(PinError::ExecutorLock { + field: "gpu_class", + .. + }) + ), + "a multi-GPU class must not become the pin" + ); + p = pin(); + p.eval_executor_schema_version = 2; + assert!(matches!( + p.validate(), + Err(PinError::ExecutorLock { + field: "eval_executor_schema_version", + .. + }) + )); + p = pin(); + p.eval_executor_commitment_alg = "blake3".into(); + assert!(matches!( + p.validate(), + Err(PinError::ExecutorLock { + field: "eval_executor_commitment_alg", + .. + }) + )); + } + + #[test] + fn deadline_ceiling_may_tighten_never_loosen() { + let mut p = pin(); + p.max_proof_deadline_s_ceiling = 3_600; + p.validate().expect("tighter ceiling is legal"); + p.max_proof_deadline_s_ceiling = MAX_PROOF_DEADLINE_S_CEILING + 1; + assert!(matches!( + p.validate(), + Err(PinError::DeadlineCeiling { .. }) + )); + p.max_proof_deadline_s_ceiling = 0; + assert!(matches!( + p.validate(), + Err(PinError::DeadlineCeiling { .. }) + )); + } + + #[test] + fn template_prefixes_must_be_usable() { + for bad in ["", "has space", &"x".repeat(65)] { + let mut p = pin(); + p.allowed_lium_template_prefixes = vec![bad.to_owned()]; + assert!( + matches!(p.validate(), Err(PinError::BadTemplatePrefix)), + "{bad:?} must be refused" + ); + } } #[test] diff --git a/crates/proof-task/src/topic.rs b/crates/proof-task/src/topic.rs index a0282c3cf..6d0e7dfaa 100644 --- a/crates/proof-task/src/topic.rs +++ b/crates/proof-task/src/topic.rs @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; -use crate::{canonical_json, is_hex64, ProofPin, TopicInference, TOPIC_DOMAIN}; +use crate::{canonical_json, is_hex64, ProofPin, TopicEvalExecutor, TopicInference, TOPIC_DOMAIN}; /// Only accepted `schema_version`. pub const TOPIC_SCHEMA_VERSION: u32 = 1; @@ -369,6 +369,10 @@ pub struct TopicDocument { pub proxy_model: Option, /// Inference constraints this topic tightens against the pin. pub inference: TopicInference, + /// Executor constraints this topic tightens (deadline, offer commitment). + /// Omitted from the signed payload when empty, so older signatures hold. + #[serde(skip_serializing_if = "TopicEvalExecutor::is_empty")] + pub eval_executor: TopicEvalExecutor, /// Sealed baseline recipe plus its two seal hashes. pub baseline: Baseline, /// Commitment over this topic's holdout records. @@ -406,6 +410,7 @@ impl Default for TopicDocument { epsilon_topic_max_regress: crate::EPSILON_TOPIC_MAX_REGRESS_MIN, proxy_model: None, inference: TopicInference::default(), + eval_executor: TopicEvalExecutor::default(), baseline: default_adamw(crate::FLOPS_BUDGET_MAX), holdout_commitment: String::new(), holdout_size: crate::HOLDOUT_SIZE, @@ -547,6 +552,12 @@ pub enum TopicError { /// Open topic resolved to an incomplete provider config, or an override is unusable. #[error("inference is incomplete or misconfigured (provider, model, mode, tokens, origin)")] IncompleteInference, + /// `eval_executor.require_offer_commitment` is not 64 hex. + #[error("eval_executor.require_offer_commitment must be 64 hex chars")] + BadExecutorCommitment, + /// Topic proof deadline is zero or above the pin ceiling (tighten only). + #[error("eval_executor.max_proof_deadline_s = {0} must be 1..={1}")] + ExecutorDeadlineCeiling(u64, u64), /// The validity window is inverted. #[error("valid_until_epoch {until} is before valid_from_epoch {from}")] BadWindow { @@ -828,6 +839,7 @@ impl TopicDocument { } } self.inference.validate(pin)?; + self.eval_executor.validate(pin)?; let model = crate::resolve_inference(pin, Some(&self.inference), None, None).model; if self.status == TopicStatus::Open && model.trim().is_empty() { return Err(TopicError::IncompleteInference); @@ -1014,6 +1026,68 @@ mod tests { )); } + /// A topic that tightens nothing about the executor signs to the exact + /// bytes it signed before the `eval_executor` field existed. + #[test] + fn empty_eval_executor_does_not_change_the_signed_payload() { + let doc = dt_no_ib(); + assert!(doc.eval_executor.is_empty()); + let payload = + String::from_utf8(topic_signing_payload(&doc).expect("payload")).expect("utf8"); + assert!(!payload.contains("eval_executor"), "{payload}"); + assert!(!payload.contains("max_proof_deadline_s"), "{payload}"); + let parsed = TopicDocument::from_json(&serde_json::to_string(&doc).expect("json")) + .expect("round trip"); + assert_eq!(parsed, doc); + } + + #[test] + fn eval_executor_tighten_is_signed_and_tighten_only() { + let p = pin(); + let mut doc = dt_no_ib(); + doc.eval_executor.max_proof_deadline_s = Some(1_800); + doc.eval_executor.require_offer_commitment = Some("ab".repeat(32)); + doc.validate(&p, &[]).expect("tightened topic validates"); + doc.signature = doc.sign_with(&sk()).expect("sign"); + doc.verify_signature(&p).expect("verifies"); + let payload = + String::from_utf8(topic_signing_payload(&doc).expect("payload")).expect("utf8"); + assert!( + payload.contains("\"max_proof_deadline_s\":1800"), + "{payload}" + ); + + // Loosening the deadline after signing breaks the signature, and a + // deadline above the pin ceiling never validates at publish. + let mut loosened = doc.clone(); + loosened.eval_executor.max_proof_deadline_s = Some(7_200); + assert!(matches!( + loosened.verify_signature(&p), + Err(TopicError::SignatureInvalid) + )); + loosened.eval_executor.max_proof_deadline_s = Some(p.max_proof_deadline_s_ceiling + 1); + assert!(matches!( + loosened.validate(&p, &[]), + Err(TopicError::ExecutorDeadlineCeiling(..)) + )); + doc.eval_executor.require_offer_commitment = Some("nope".into()); + assert!(matches!( + doc.validate(&p, &[]), + Err(TopicError::BadExecutorCommitment) + )); + } + + #[test] + fn a_topic_naming_a_machine_id_is_refused_at_parse() { + let mut v = serde_json::to_value(dt_no_ib()).expect("json"); + v["eval_executor"] = serde_json::json!({ "machine_id": "lium-pod-42" }); + let err = TopicDocument::from_json(&v.to_string()).expect_err("no per-topic machine_id"); + assert!( + matches!(err, TopicError::Parse(ref m) if m.contains("machine_id")), + "{err}" + ); + } + #[test] fn a_topic_signed_by_another_key_is_not_this_subnets_topic() { let p = pin(); diff --git a/crates/proof-task/tests/committed_pin.rs b/crates/proof-task/tests/committed_pin.rs index f8f6d1982..fe2ce4d61 100644 --- a/crates/proof-task/tests/committed_pin.rs +++ b/crates/proof-task/tests/committed_pin.rs @@ -6,9 +6,10 @@ use std::path::{Path, PathBuf}; use proof_task::{ - ProofPin, ALLOWED_MODES, CHALLENGE_ID, EVAL_IMAGE, HOLDOUT_SIZE, - INFERENCE_CONFIG_SCHEMA_VERSION, INFERENCE_OFFER_COMMITMENT_ALG, MAX_INPUT_TOKENS_CEILING, - MAX_OUTPUT_TOKENS_CEILING, STRATUM_SIZE, + ProofPin, ALLOWED_MODES, CHALLENGE_ID, EVAL_EXECUTOR_COMMITMENT_ALG, EVAL_EXECUTOR_GPU_CLASS, + EVAL_EXECUTOR_SCHEMA_VERSION, EVAL_IMAGE, HOLDOUT_SIZE, INFERENCE_CONFIG_SCHEMA_VERSION, + INFERENCE_OFFER_COMMITMENT_ALG, MAX_INPUT_TOKENS_CEILING, MAX_OUTPUT_TOKENS_CEILING, + MAX_PROOF_DEADLINE_S_CEILING, STRATUM_SIZE, }; fn pin_path() -> PathBuf { @@ -106,6 +107,38 @@ fn committed_pin_is_proof_with_a_real_eval_digest() { assert_eq!(p.inference.max_output_tokens, MAX_OUTPUT_TOKENS_CEILING); } +/// The executor ceilings are the 2026-09-08 lock: schema 1, `1x` only, +/// two-hour deadline ceiling, sha256 commitment, and a `proof-eval-` template +/// allowlist that matches the digest-scoped harvest template name. +#[test] +fn committed_pin_locks_the_one_gpu_executor_ceilings() { + let p = pin(); + assert_eq!(p.eval_executor_schema_version, EVAL_EXECUTOR_SCHEMA_VERSION); + assert_eq!(p.gpu_class, EVAL_EXECUTOR_GPU_CLASS); + assert_eq!(p.gpu_class, "1x"); + assert_eq!(p.max_proof_deadline_s_ceiling, MAX_PROOF_DEADLINE_S_CEILING); + assert_eq!(p.max_proof_deadline_s_ceiling, 7_200); + assert_eq!(p.eval_executor_commitment_alg, EVAL_EXECUTOR_COMMITMENT_ALG); + assert_eq!(p.allowed_lium_template_prefixes, vec!["proof-eval-"]); + let digest_hex = p.eval_image_digest.trim_start_matches("sha256:"); + assert!(p.allows_template(&format!("proof-eval-{}", &digest_hex[..12]))); + assert!(!p.allows_template("prism-recipe-v10")); + let text = body(); + for key in [ + "eval_executor_schema_version", + "gpu_class", + "max_proof_deadline_s_ceiling", + "allowed_lium_template_prefixes", + "eval_executor_commitment_alg", + ] { + assert!(text.contains(key), "pin must name {key}"); + } + assert!( + !text.contains("machine_id") && !text.contains("lium_template_id ="), + "the pin carries ceilings, never a live executor offer" + ); +} + #[test] fn topic_pubkey_matches_the_trust_root_proof_row() { assert_eq!(pin().topic_pubkey.to_ascii_lowercase(), proof_row_pubkey()); From 6bcd2eb55e73dbb6f2bf78289debd0bd20f33706 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:26:46 +0000 Subject: [PATCH 02/16] feat(lium): exact_gpu_count spec aborts any rent that is not the pinned width Co-authored-by: Mathis --- crates/prism-lium-types/src/types.rs | 49 ++++++++++++++++++++++++++++ crates/prism-lium/src/client.rs | 49 ++++++++++++++++++++++++++++ crates/prism-lium/src/sim.rs | 2 ++ crates/prism-lium/tests/live_e2e.rs | 1 + 4 files changed, 101 insertions(+) diff --git a/crates/prism-lium-types/src/types.rs b/crates/prism-lium-types/src/types.rs index 85a24b4f8..ccfa8f4e3 100644 --- a/crates/prism-lium-types/src/types.rs +++ b/crates/prism-lium-types/src/types.rs @@ -232,6 +232,9 @@ pub struct InstanceSpec { pub template_id: Option, /// Optional template name to ensure/resolve (e.g. prism-mission-e2e). pub template_name: Option, + /// Refuse any offer whose rent would not be exactly `gpu_count` GPUs (no + /// whole-host / NCU upsizing). The Proof `1x` executor sets this. + pub exact_gpu_count: bool, } impl InstanceSpec { @@ -242,6 +245,13 @@ impl InstanceSpec { .as_deref() .is_some_and(|s| !s.trim().is_empty()) } + + /// Whether `rent_gpu_count` (what `POST /executors/{id}/rent` would send) + /// is legal for this spec. Only an exact-width spec refuses upsizing. + #[must_use] + pub fn accepts_rent_count(&self, rent_gpu_count: u32) -> bool { + !self.exact_gpu_count || rent_gpu_count == self.gpu_count + } } impl Default for InstanceSpec { @@ -259,6 +269,7 @@ impl Default for InstanceSpec { preferred_offer_id: None, template_id: None, template_name: Some("prism-mission-e2e".into()), + exact_gpu_count: false, } } } @@ -916,6 +927,44 @@ mod tests { assert_eq!(offers[0].id, "4a36877c"); } + /// The Proof `1x` executor never upsizes: an NCU B200 that would rent + /// two cards is refused, an idle 8× that splits to one is fine. + #[test] + fn exact_gpu_count_refuses_whole_host_upsizing() { + let ncu = Offer { + id: "4a36877c".into(), + gpu_type: "NVIDIA B200".into(), + gpu_count: 2, + price_per_hour: 5.5, + min_gpu_count_for_rental: Some(1), + available_gpu_count: Some(2), + ncu_profiling_enabled: true, + ..Offer::default() + }; + let idle8 = Offer { + id: "idle".into(), + gpu_type: "NVIDIA B200".into(), + gpu_count: 8, + price_per_hour: 5.85, + available_gpu_count: Some(8), + ..Offer::default() + }; + let exact = InstanceSpec { + gpu_count: 1, + exact_gpu_count: true, + ..InstanceSpec::default() + }; + let loose = InstanceSpec { + gpu_count: 1, + ..InstanceSpec::default() + }; + assert!(!InstanceSpec::default().exact_gpu_count); + assert!(!exact.accepts_rent_count(ncu.rent_count(1))); + assert!(exact.accepts_rent_count(idle8.rent_count(1))); + assert!(loose.accepts_rent_count(ncu.rent_count(1))); + assert!(loose.accepts_rent_count(idle8.rent_count(1))); + } + #[test] fn parse_live_ncu_2x_b200_is_whole_host_rent() { let v = serde_json::json!({ diff --git a/crates/prism-lium/src/client.rs b/crates/prism-lium/src/client.rs index 30febc1b6..f63ce6871 100644 --- a/crates/prism-lium/src/client.rs +++ b/crates/prism-lium/src/client.rs @@ -815,6 +815,14 @@ impl EvalJobBackend for LiumClient { prism_lium_types::effective_gpu_count(selected.gpu_count, &selected.gpu_type); // Split hosts: requested width. NCU / non-split: whole host. let rent_gpu_count = selected.rent_count(spec.gpu_count); + // Exact-width specs (Proof 1x executor) never upsize to a whole host. + if !spec.accepts_rent_count(rent_gpu_count) { + last_err = format!( + "abort: offer {} rents {rent_gpu_count}x, spec requires exactly {}x", + selected.id, spec.gpu_count + ); + continue; + } if pref.matches_pin("RTX 5090") && rent_gpu_count >= 8 && spec.gpu_count < 8 { return Err(LiumError::Api(format!( "abort: refusing {rent_gpu_count}× 5090 rent (no 8×5090 fallback)" @@ -1018,6 +1026,7 @@ mod tests { preferred_offer_id: None, template_id: None, template_name: None, + exact_gpu_count: false, }; let err = c.provision(&spec).await.unwrap_err(); assert!(matches!( @@ -1042,6 +1051,7 @@ mod tests { preferred_offer_id: None, template_id: None, template_name: None, + exact_gpu_count: false, }; let err = c.provision(&spec).await.unwrap_err(); assert!(matches!(err, LiumError::Api(_))); @@ -1106,6 +1116,7 @@ mod tests { preferred_offer_id: None, template_id: None, template_name: None, + exact_gpu_count: false, } } @@ -1595,6 +1606,44 @@ mod tests { assert_eq!(provision_spec().gpu_count, 1); } + /// Proof `1x` executor: the same NCU host that the loose spec upsizes to + /// a 2× whole-host rent is skipped, and no rent is ever POSTed. + #[tokio::test] + async fn provision_exact_gpu_count_aborts_instead_of_renting_two() { + let server = MockServer::start().await; + mount_common( + &server, + serde_json::json!([{ + "id": "4a36877c", + "machine_name": "NVIDIA B200", + "gpu_count": 2, + "available_gpu_count": 2, + "min_gpu_count_for_rental": 1, + "ncu_profiling_enabled": true, + "price_per_gpu": 5.5 + }]), + ) + .await; + Mock::given(method("POST")) + .and(path("/executors/4a36877c/rent")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"id": "pod-ncu"})), + ) + .expect(0) + .mount(&server) + .await; + let c = LiumClient::with_base_url("test-key", server.uri()).unwrap(); + let mut spec = provision_spec(); + spec.exact_gpu_count = true; + let err = c.provision(&spec).await.unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("abort") && msg.contains("2x") && msg.contains("exactly 1x"), + "got {msg}" + ); + server.verify().await; + } + #[tokio::test] async fn provision_ncu_over_price_is_named_error_not_split() { let server = MockServer::start().await; diff --git a/crates/prism-lium/src/sim.rs b/crates/prism-lium/src/sim.rs index f044852ba..4c3e8f436 100644 --- a/crates/prism-lium/src/sim.rs +++ b/crates/prism-lium/src/sim.rs @@ -218,6 +218,7 @@ mod tests { preferred_offer_id: None, template_id: None, template_name: None, + exact_gpu_count: false, }; let inst = b.provision(&spec).await.unwrap(); assert!(!b.verify_terminated(&inst.id).await.unwrap()); @@ -283,6 +284,7 @@ mod tests { preferred_offer_id: None, template_id: None, template_name: None, + exact_gpu_count: false, }; let e = b.provision(&spec).await.unwrap_err(); assert!(matches!( diff --git a/crates/prism-lium/tests/live_e2e.rs b/crates/prism-lium/tests/live_e2e.rs index d62130950..c7f66aea8 100644 --- a/crates/prism-lium/tests/live_e2e.rs +++ b/crates/prism-lium/tests/live_e2e.rs @@ -164,6 +164,7 @@ async fn live_rent_ssh_eval_terminate() { // Exercise digest-only `PRISM_POD_IMAGE_REF` resolution, not a // historical provider-side template. template_name: None, + exact_gpu_count: false, }; let mut att = serde_json::json!({ "offer_id": offer.id, From f3eefb89f5afe1ea3e83b838afbaa2f0fabd41bf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:27:50 +0000 Subject: [PATCH 03/16] feat(harvest-pod): per-run deadline caps the pod timeout and classifies deadline exits Co-authored-by: Mathis --- crates/harvest-pod/src/lib.rs | 59 +++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/harvest-pod/src/lib.rs b/crates/harvest-pod/src/lib.rs index dac8a8a66..1f3026d19 100644 --- a/crates/harvest-pod/src/lib.rs +++ b/crates/harvest-pod/src/lib.rs @@ -79,6 +79,35 @@ pub struct RunExtras { pub holdout_tar: Vec, /// On-disk proxy-model tar. Missing / empty skips. pub proxy_tar_path: Option, + /// Per-run proof deadline in seconds. When set it caps the pod-side + /// `timeout` (and the SSH wait) at `min(deadline, configured run + /// timeout)`; the image is killed at the deadline, never given more. + pub deadline_secs: Option, +} + +/// Exit codes GNU `timeout` reports when the entrypoint hit the deadline: +/// `124` on the TERM timeout, `137` when `--kill-after` had to SIGKILL. +pub const DEADLINE_EXIT_CODES: [u32; 2] = [124, 137]; + +/// Seconds the entrypoint gets for one run: the configured run timeout, +/// tightened by a per-run deadline when one is staged. +#[must_use] +pub fn effective_run_timeout_secs(run_timeout_secs: u64, deadline_secs: Option) -> u64 { + deadline_secs + .filter(|d| *d > 0) + .map_or(run_timeout_secs, |d| d.min(run_timeout_secs)) +} + +/// Whether pod stdout reports an `exit=` line with a deadline exit code +/// (the run was cut at the `timeout`, not a scoring failure). +#[must_use] +pub fn hit_deadline(stdout: &str) -> bool { + stdout.lines().any(|l| { + l.trim() + .strip_prefix("exit=") + .and_then(|rc| rc.trim().parse::().ok()) + .is_some_and(|rc| DEADLINE_EXIT_CODES.contains(&rc)) + }) } /// Seconds allowed to stream `bytes` over SSH. Floor is the short-op @@ -524,13 +553,14 @@ impl EvalPod for LiumEvalPod { // `allow_fail`: a non-zero image exit still has to be harvested, since // the log tail is the only diagnosis the operator gets. + let run_secs = effective_run_timeout_secs(self.run_timeout_secs, extras.deadline_secs); let out = ssh_exec_allow_fail( &target, &key, - &self.program.run_cmd(self.run_timeout_secs), + &self.program.run_cmd(run_secs), 1, SSH_RETRY_SECS, - self.run_timeout_secs.saturating_add(SSH_SHORT_TIMEOUT_SECS), + run_secs.saturating_add(SSH_SHORT_TIMEOUT_SECS), ) .await .map_err(|e| format!("run eval image: {e}"))?; @@ -692,6 +722,31 @@ mod tests { assert!(err.contains("refusing to stage the holdout"), "{err}"); } + #[test] + fn a_staged_deadline_only_tightens_the_run_timeout() { + assert_eq!(effective_run_timeout_secs(5400, None), 5400); + assert_eq!(effective_run_timeout_secs(5400, Some(0)), 5400); + assert_eq!(effective_run_timeout_secs(5400, Some(1800)), 1800); + assert_eq!( + effective_run_timeout_secs(5400, Some(9000)), + 5400, + "a deadline never extends the configured run timeout" + ); + let cmd = PROGRAM.run_cmd(effective_run_timeout_secs(5400, Some(1800))); + assert!(cmd.contains("timeout --kill-after=60 1800"), "{cmd}"); + assert!(RunExtras::default().deadline_secs.is_none()); + } + + #[test] + fn deadline_exit_codes_are_recognised_in_stdout() { + assert!(hit_deadline("boot\nexit=124\ntail of run.log\n")); + assert!(hit_deadline("exit=137\n")); + assert!(!hit_deadline("exit=2\n")); + assert!(!hit_deadline("DEMO_METRICS={\"a\":1}\nDEMO_EVAL_OK\n")); + assert!(!hit_deadline("the log said exit=124 once")); + assert_eq!(DEADLINE_EXIT_CODES, [124, 137]); + } + #[test] fn stage_timeout_grows_with_archive_size() { assert_eq!(stage_timeout_secs(0), 120); From 5921d7b9cc94099e346389f87c7880a558518074 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:31:00 +0000 Subject: [PATCH 04/16] feat(proof-eval): executor offer gates live readiness and the harvest scorer Co-authored-by: Mathis --- Cargo.lock | 1 + crates/proof-eval/Cargo.toml | 1 + crates/proof-eval/src/lib.rs | 227 ++++++++++++++++++++++++++++++++++- 3 files changed, 224 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e306a8db6..2695d239d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3463,6 +3463,7 @@ dependencies = [ "async-trait", "hex", "prism-lium-types", + "proof-executor", "proof-score", "proof-store", "proof-task", diff --git a/crates/proof-eval/Cargo.toml b/crates/proof-eval/Cargo.toml index ce4db9bad..a00c65020 100644 --- a/crates/proof-eval/Cargo.toml +++ b/crates/proof-eval/Cargo.toml @@ -11,6 +11,7 @@ publish = false [dependencies] async-trait = "0.1" prism-lium-types = { path = "../prism-lium-types" } +proof-executor = { path = "../proof-executor" } proof-score = { path = "../proof-score" } proof-store = { path = "../proof-store" } proof-task = { path = "../proof-task" } diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 182fcf22b..bd542d972 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -25,6 +25,7 @@ use std::collections::{BTreeMap, BTreeSet}; use async_trait::async_trait; use prism_lium_types::{EvalReceipt, NoScoreGate}; +use proof_executor::{require_open_executor, EvalExecutorOffer, ExecutorOfferError}; use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind, SealedBaseline}; use proof_store::ArtifactManifest; use proof_task::{ @@ -124,6 +125,33 @@ pub enum EvalError { /// Live score needs operator-staged holdout shard bytes. #[error("PROOF_HOLDOUT_STORE missing or incomplete; refuse scoring")] HoldoutStoreMissing, + /// No live `EvalExecutorOffer` on this host (Lium path). + #[error("eval executor offer missing; refuse scoring")] + ExecutorOfferMissing, + /// Live `EvalExecutorOffer` is closed. + #[error("eval executor offer is closed; refuse scoring")] + ExecutorOfferClosed, + /// Live `EvalExecutorOffer` failed pin validation or cannot serve the topic. + #[error("eval executor offer: {0}")] + ExecutorOffer(String), + /// The eval run was cut at the executor proof deadline. Not a zero — a 503 + /// whose body carries the pod's log tail. + #[error("proof deadline of {deadline_s}s exceeded; stdout_tail: {stdout_tail}")] + ProofDeadlineExceeded { + /// Deadline the run was held to (offer, topic tighten, operator override). + deadline_s: u64, + /// Last bytes of pod stdout (run log tail), for the operator. + stdout_tail: String, + }, +} + +/// Map an executor refusal onto the eval error the HTTP layer answers 503 with. +pub fn map_executor_err(e: ExecutorOfferError) -> EvalError { + match e { + ExecutorOfferError::Missing => EvalError::ExecutorOfferMissing, + ExecutorOfferError::Closed => EvalError::ExecutorOfferClosed, + other => EvalError::ExecutorOffer(other.to_string()), + } } /// Schema version of the metrics+verdict document the eval image emits. @@ -231,12 +259,16 @@ impl ProofEvalDocument { #[async_trait] pub trait LiveScorer: Send + Sync { /// Score one artifact on one topic's verified holdout. + /// + /// `offer` is the RLM judge the image calls; `executor` is the open + /// `1x` machine class the image is rented on. Both are host state. #[allow(clippy::too_many_arguments)] async fn score( &self, pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, + executor: &EvalExecutorOffer, frozen_digest: &str, artifact_digest: &str, holdout: &[HoldoutRecord], @@ -378,12 +410,17 @@ fn map_offer_err(e: OfferError) -> EvalError { } /// Whether this host can produce a verdict at all. +/// +/// The RLM judge offer is required on every backend. The eval executor offer +/// is a Lium-path requirement: sim rents nothing, so there is no machine to +/// bind; a live host with no open `1x` executor cannot score. pub fn scoring_readiness( pin: &ProofPin, backend: EvalBackend, live: Option<&dyn LiveScorer>, has_open_sealed_topic: bool, offer: Option<&InferenceOffer>, + executor: Option<&EvalExecutorOffer>, judge_api_key: Option<&str>, ) -> Result<(), EvalError> { if !has_open_sealed_topic { @@ -398,7 +435,9 @@ pub fn scoring_readiness( } let scorer = live.ok_or(EvalError::LiveHarvestUnavailable)?; scorer.ready()?; - judge_api_key_ready(judge_api_key) + judge_api_key_ready(judge_api_key)?; + require_open_executor(executor, pin).map_err(map_executor_err)?; + Ok(()) } } } @@ -646,6 +685,7 @@ pub async fn eval_after_freeze( pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, + executor: Option<&EvalExecutorOffer>, frozen_digest: &str, artifact_digest: &str, holdout: &[HoldoutRecord], @@ -658,10 +698,24 @@ pub async fn eval_after_freeze( if frozen_digest.trim().is_empty() || holdout.is_empty() { return Err(EvalError::HoldoutSealed); } - scoring_readiness(pin, backend, live, true, Some(offer), judge_api_key)?; + scoring_readiness( + pin, + backend, + live, + true, + Some(offer), + executor, + judge_api_key, + )?; offer .serves_topic(pin, topic) .map_err(|e| EvalError::InferenceOffer(e.to_string()))?; + if backend == EvalBackend::Lium { + executor + .ok_or(EvalError::ExecutorOfferMissing)? + .serves_topic(topic) + .map_err(map_executor_err)?; + } let resolved = resolve_inference( pin, Some(&topic.inference), @@ -689,11 +743,13 @@ pub async fn eval_after_freeze( } EvalBackend::Lium => { let scorer = live.ok_or(EvalError::LiveHarvestUnavailable)?; + let executor = executor.ok_or(EvalError::ExecutorOfferMissing)?; scorer .score( pin, topic, offer, + executor, frozen_digest, artifact_digest, holdout, @@ -800,6 +856,23 @@ mod tests { } } + /// Open `1x` executor bound to `pin`'s digest (template name carries the + /// digest prefix, as the digest-scoped harvest template does). + fn executor(pin: &ProofPin) -> EvalExecutorOffer { + let hex = pin.eval_image_digest.trim_start_matches("sha256:"); + let mut o = EvalExecutorOffer { + offer_id: "lium-1x-v0".into(), + lium_template_id: format!("proof-eval-{}", hex.get(..12).unwrap_or("unpinned")), + machine_shape: "1x".into(), + max_proof_deadline_s: 3_600, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: proof_executor::OfferStatus::Open, + }; + o.config_commitment = o.expected_commitment(); + o + } + struct Harvest { reproduced: bool, } @@ -811,6 +884,7 @@ mod tests { pin: &ProofPin, topic: &TopicDocument, _offer: &InferenceOffer, + _executor: &EvalExecutorOffer, frozen: &str, artifact: &str, _holdout: &[HoldoutRecord], @@ -841,6 +915,7 @@ mod tests { &pin(""), &t, &offer(), + Some(&executor(&pin(""))), "d", "art", &recs, @@ -861,6 +936,7 @@ mod tests { &pin(&format!("sha256:{}", "ab".repeat(32))), &t, &offer(), + Some(&executor(&pin(&format!("sha256:{}", "ab".repeat(32))))), "d", "art", &recs, @@ -887,6 +963,7 @@ mod tests { &p, &t, &offer(), + Some(&executor(&p)), "digest-a", "art", &recs, @@ -930,7 +1007,15 @@ mod tests { let live = pin(&format!("sha256:{}", "ab".repeat(32))); let o = offer(); assert!(matches!( - scoring_readiness(&live, EvalBackend::Sim, None, false, Some(&o), None), + scoring_readiness( + &live, + EvalBackend::Sim, + None, + false, + Some(&o), + Some(&executor(&live)), + None + ), Err(EvalError::NoOpenTopic) )); scoring_readiness( @@ -939,6 +1024,7 @@ mod tests { None, true, Some(&o), + Some(&executor(&ProofPin::default())), None, ) .expect("sim"); @@ -949,7 +1035,8 @@ mod tests { None, true, None, - None + Some(&executor(&ProofPin::default())), + None, ), Err(EvalError::InferenceOfferMissing) )); @@ -960,12 +1047,21 @@ mod tests { None, true, Some(&o), + Some(&executor(&ProofPin::default())), None, ), Err(EvalError::EvalImageUnpinned) )); assert!(matches!( - scoring_readiness(&live, EvalBackend::Lium, None, true, Some(&o), None), + scoring_readiness( + &live, + EvalBackend::Lium, + None, + true, + Some(&o), + Some(&executor(&live)), + None + ), Err(EvalError::LiveHarvestUnavailable) )); scoring_readiness( @@ -974,6 +1070,7 @@ mod tests { Some(&Harvest { reproduced: true }), true, Some(&o), + Some(&executor(&live)), Some("test-judge-key"), ) .expect("ready"); @@ -984,12 +1081,130 @@ mod tests { Some(&Harvest { reproduced: true }), true, Some(&o), + Some(&executor(&live)), None, ), Err(EvalError::InferenceAuthMissing) )); } + /// Lium: missing / closed / non-`1x` executor is a refusal after every + /// other live prerequisite holds. Sim never rents, so it does not care. + #[test] + fn readiness_requires_an_open_one_gpu_executor_on_lium_only() { + let live = pin(&format!("sha256:{}", "ab".repeat(32))); + let o = offer(); + let harvest = Harvest { reproduced: true }; + assert!(matches!( + scoring_readiness( + &live, + EvalBackend::Lium, + Some(&harvest), + true, + Some(&o), + None, + Some("test-judge-key"), + ), + Err(EvalError::ExecutorOfferMissing) + )); + let mut closed = executor(&live); + closed.status = proof_executor::OfferStatus::Closed; + assert!(matches!( + scoring_readiness( + &live, + EvalBackend::Lium, + Some(&harvest), + true, + Some(&o), + Some(&closed), + Some("test-judge-key"), + ), + Err(EvalError::ExecutorOfferClosed) + )); + let mut wide = executor(&live); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + let err = scoring_readiness( + &live, + EvalBackend::Lium, + Some(&harvest), + true, + Some(&o), + Some(&wide), + Some("test-judge-key"), + ) + .expect_err("8x cannot score"); + assert!( + matches!(err, EvalError::ExecutorOffer(ref m) if m.contains("machine_shape")), + "{err}" + ); + scoring_readiness( + &ProofPin::default(), + EvalBackend::Sim, + None, + true, + Some(&o), + None, + None, + ) + .expect("sim rents nothing"); + } + + #[tokio::test] + async fn live_eval_refuses_without_an_executor_that_serves_the_topic() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let mut t = topic(); + let missing = eval_after_freeze( + &p, + &t, + &offer(), + None, + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Lium, + Some(&Harvest { reproduced: true }), + Some("test-judge-key"), + None, + ) + .await + .expect_err("no executor"); + assert!( + matches!(missing, EvalError::ExecutorOfferMissing), + "{missing}" + ); + + t.eval_executor.require_offer_commitment = Some("cd".repeat(32)); + let pinned_elsewhere = eval_after_freeze( + &p, + &t, + &offer(), + Some(&executor(&p)), + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Lium, + Some(&Harvest { reproduced: true }), + Some("test-judge-key"), + None, + ) + .await + .expect_err("topic pins another executor"); + assert!( + matches!(pinned_elsewhere, EvalError::ExecutorOffer(ref m) if m.contains("cannot serve")), + "{pinned_elsewhere}" + ); + assert!(EvalError::ProofDeadlineExceeded { + deadline_s: 600, + stdout_tail: "exit=124".into(), + } + .to_string() + .contains("600s"),); + } + #[test] fn baseline_commitment_is_bound_to_the_vector() { let mut splits = BTreeMap::new(); @@ -1114,6 +1329,7 @@ mod tests { &p, &t, &offer(), + Some(&executor(&p)), "digest-a", "art", &recs, @@ -1146,6 +1362,7 @@ mod tests { &p, &t, &offer(), + Some(&executor(&p)), "digest-a", "art", &recs, From 2ed5d8dd93628321f2bbc87978e65930fb13d7bc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:39:08 +0000 Subject: [PATCH 05/16] feat(proof-harvest): rent the executor template at exactly 1x and hold the run to the proof deadline Co-authored-by: Mathis --- Cargo.lock | 1 + crates/harvest-pod/src/lib.rs | 33 +- crates/proof-harvest/Cargo.toml | 4 +- crates/proof-harvest/src/lib.rs | 600 ++++++++++++++++++++++++++++++-- 4 files changed, 594 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2695d239d..7bbb8950a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3495,6 +3495,7 @@ dependencies = [ "hex", "prism-lium-types", "proof-eval", + "proof-executor", "proof-task", "serde", "serde_json", diff --git a/crates/harvest-pod/src/lib.rs b/crates/harvest-pod/src/lib.rs index 1f3026d19..ad567f747 100644 --- a/crates/harvest-pod/src/lib.rs +++ b/crates/harvest-pod/src/lib.rs @@ -79,10 +79,6 @@ pub struct RunExtras { pub holdout_tar: Vec, /// On-disk proxy-model tar. Missing / empty skips. pub proxy_tar_path: Option, - /// Per-run proof deadline in seconds. When set it caps the pod-side - /// `timeout` (and the SSH wait) at `min(deadline, configured run - /// timeout)`; the image is killed at the deadline, never given more. - pub deadline_secs: Option, } /// Exit codes GNU `timeout` reports when the entrypoint hit the deadline: @@ -90,7 +86,8 @@ pub struct RunExtras { pub const DEADLINE_EXIT_CODES: [u32; 2] = [124, 137]; /// Seconds the entrypoint gets for one run: the configured run timeout, -/// tightened by a per-run deadline when one is staged. +/// tightened by a per-run proof deadline when one is given. The image is +/// killed at the deadline, never given more. #[must_use] pub fn effective_run_timeout_secs(run_timeout_secs: u64, deadline_secs: Option) -> u64 { deadline_secs @@ -284,18 +281,25 @@ pub trait EvalPod: Send + Sync { /// Boot the digest-pinned image and return the instance id. async fn boot(&self, spec: &InstanceSpec) -> Result; - /// Deliver `request` bytes (and optional env file), run the image, return stdout. + /// Deliver `request` bytes (and optional env file) onto the pod. /// /// `env_file` is written to [`ENV_FILE`] over stdin when non-empty. Empty /// skips that stage so challenges without a judge host stay env-free. /// `extras` stages holdout bytes and a proxy tar path when present. - async fn run( + /// Staging is separate from [`Self::run`] so a slow multi-GB upload never + /// eats into a proof deadline. + async fn stage( &self, instance_id: &str, request: &[u8], env_file: &[u8], extras: &RunExtras, - ) -> Result; + ) -> Result<(), String>; + + /// Run the image entrypoint on what [`Self::stage`] delivered and return + /// stdout. `deadline_secs` caps the pod-side `timeout` at + /// `min(deadline, configured run timeout)` ([`effective_run_timeout_secs`]). + async fn run(&self, instance_id: &str, deadline_secs: Option) -> Result; /// Terminate. `Ok(true)` only when the provider confirms the pod is gone. async fn shutdown(&self, instance_id: &str) -> Result; @@ -507,13 +511,13 @@ impl EvalPod for LiumEvalPod { Ok(inst.id) } - async fn run( + async fn stage( &self, instance_id: &str, request: &[u8], env_file: &[u8], extras: &RunExtras, - ) -> Result { + ) -> Result<(), String> { let key = resolve_private_key(None).map_err(|e| e.to_string())?; let target = self.target(instance_id).await?; @@ -550,10 +554,16 @@ impl EvalPod for LiumEvalPod { if let Some(path) = extras.proxy_tar_path.as_deref() { self.stage_tree_file(&target, &key, PROXY_DIR, path).await?; } + Ok(()) + } + + async fn run(&self, instance_id: &str, deadline_secs: Option) -> Result { + let key = resolve_private_key(None).map_err(|e| e.to_string())?; + let target = self.target(instance_id).await?; // `allow_fail`: a non-zero image exit still has to be harvested, since // the log tail is the only diagnosis the operator gets. - let run_secs = effective_run_timeout_secs(self.run_timeout_secs, extras.deadline_secs); + let run_secs = effective_run_timeout_secs(self.run_timeout_secs, deadline_secs); let out = ssh_exec_allow_fail( &target, &key, @@ -734,7 +744,6 @@ mod tests { ); let cmd = PROGRAM.run_cmd(effective_run_timeout_secs(5400, Some(1800))); assert!(cmd.contains("timeout --kill-after=60 1800"), "{cmd}"); - assert!(RunExtras::default().deadline_secs.is_none()); } #[test] diff --git a/crates/proof-harvest/Cargo.toml b/crates/proof-harvest/Cargo.toml index d21bb086e..5460b5b83 100644 --- a/crates/proof-harvest/Cargo.toml +++ b/crates/proof-harvest/Cargo.toml @@ -13,15 +13,17 @@ async-trait = "0.1" harvest-pod = { path = "../harvest-pod" } prism-lium-types = { path = "../prism-lium-types" } proof-eval = { path = "../proof-eval" } +proof-executor = { path = "../proof-executor" } proof-task = { path = "../proof-task" } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" hex = "0.4" +tokio = { version = "1", features = ["time"] } tracing = "0.1" [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "time"] } [lints] workspace = true diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 06261568a..f0fbf208f 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -6,6 +6,14 @@ //! must enforce — e.g. a 12.5 Gbit/s cap), reads back the metrics document, //! and tears the pod down. Nothing here computes a score. Miners do not bind //! or train against the offer. There is no sim fallback. +//! +//! **Where** the image runs is the live `EvalExecutorOffer`: the pod is +//! rented on that offer's Lium template, at exactly the pinned `1x` width +//! (any other rent width aborts before the rent), and the run is held to the +//! resolved proof deadline (offer, tightened by the topic, or an operator +//! `PROOF_HARVEST_*` override) both by the pod-side `timeout` and by this +//! crate's wait. A run cut at the deadline is a **503** carrying the pod's +//! stdout tail, never a zero. #![forbid(unsafe_code)] #![allow( @@ -19,15 +27,19 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use harvest_pod::{ - harvest_template_name, truncate_tail, EvalPod, PodProgram, RunExtras, HOLDOUT_DIR, PROXY_DIR, + harvest_template_name, hit_deadline, truncate_tail, EvalPod, PodProgram, RunExtras, + HOLDOUT_DIR, PROXY_DIR, }; use prism_lium_types::InstanceSpec; use proof_eval::{ - secret_backed_base_url, EvalError, LiveScorer, ProofEvalDocument, PROOF_METRICS_SCHEMA, + map_executor_err, secret_backed_base_url, EvalError, LiveScorer, ProofEvalDocument, + PROOF_METRICS_SCHEMA, }; +use proof_executor::{executor_plan, EvalExecutorOffer, ExecutorPlan, HarvestOverrides}; use proof_task::{ resolve_inference, HoldoutRecord, InferenceOffer, ProofPin, TopicDocument, CHALLENGE_ID, }; @@ -40,9 +52,14 @@ pub const METRICS_MARKER: &str = "PROOF_METRICS="; /// Marker the eval image prints on a completed run. pub const OK_MARKER: &str = "PROOF_EVAL_OK"; -/// Bytes of pod stdout retained on a missing [`OK_MARKER`] refuse. +/// Bytes of pod stdout retained on a missing [`OK_MARKER`] or deadline refuse. const STDOUT_TAIL_BYTES: usize = 8 * 1024; +/// Seconds the harvest waits past the proof deadline for the pod-side +/// `timeout --kill-after=60` and the SSH round trip to report back before it +/// stops waiting and tears the pod down anyway. +pub const DEADLINE_WAIT_GRACE_SECS: u64 = 300; + /// Directory the request and metrics sidecar live in, on the pod. pub const POD_WORKDIR: &str = "/tmp/proof_eval"; @@ -92,6 +109,12 @@ pub struct HarvestRequest { pub max_output_tokens: u32, /// Judge config commitment. pub config_commitment: String, + /// Live executor offer the pod was rented on (host stamp). + pub executor_offer_id: String, + /// Executor `config_commitment` (template, shape, deadline, digest). + pub executor_commitment: String, + /// Proof deadline this run is held to, seconds. + pub max_proof_deadline_s: u64, /// Eval image digest, so the image can stamp its own provenance. pub eval_image_digest: String, /// Commitment the records below must hash to. @@ -108,15 +131,14 @@ pub struct HarvestRequest { pub holdout: Vec, } -/// Rent limits for one harvest. +/// Rent limits for one harvest. Width is not a limit: it is the executor +/// plan's exact `1x`, and any other rent aborts. #[derive(Debug, Clone)] pub struct HarvestLimits { /// Max pod lifetime hours. pub max_lifetime_hours: f64, /// Max USD per GPU-hour. pub max_price_per_hour: f64, - /// GPUs requested. - pub gpu_count: u32, } impl Default for HarvestLimits { @@ -124,7 +146,6 @@ impl Default for HarvestLimits { Self { max_lifetime_hours: 6.0, max_price_per_hour: 12.0, - gpu_count: 1, } } } @@ -353,6 +374,11 @@ pub struct LiumProofHarvest { /// Defaults to the process temp dir. Tests override this so leftover /// scans do not race sibling unit tests that share `/tmp`. proxy_tar_dir: PathBuf, + /// Operator `PROOF_HARVEST_*` hot-swap. `None` reads the process env at + /// score time; tests inject a value so they never touch the env. + overrides: Option, + /// Seconds past the deadline the wait tolerates ([`DEADLINE_WAIT_GRACE_SECS`]). + deadline_wait_grace_secs: u64, } impl LiumProofHarvest { @@ -367,7 +393,40 @@ impl LiumProofHarvest { proxy_model_dir: None, holdout_store: None, proxy_tar_dir: std::env::temp_dir(), + overrides: None, + deadline_wait_grace_secs: DEADLINE_WAIT_GRACE_SECS, + } + } + + /// Pin the operator overrides instead of reading `PROOF_HARVEST_*` env. + #[must_use] + pub fn with_harvest_overrides(mut self, overrides: Option) -> Self { + self.overrides = overrides; + self + } + + /// Tighten the post-deadline wait grace (tests). + #[cfg(test)] + #[must_use] + fn with_deadline_wait_grace_secs(mut self, secs: u64) -> Self { + self.deadline_wait_grace_secs = secs; + self + } + + fn plan( + &self, + pin: &ProofPin, + topic: &TopicDocument, + executor: &EvalExecutorOffer, + ) -> Result { + let overrides = match &self.overrides { + Some(o) => o.clone(), + None => HarvestOverrides::from_env().map_err(map_executor_err)?, + }; + if !overrides.is_empty() { + tracing::info!(?overrides, "PROOF_HARVEST_* override in effect"); } + executor_plan(pin, Some(executor), topic, &overrides).map_err(map_executor_err) } /// Inject the judge API key staged into `teacher.env` on the pod. @@ -432,25 +491,87 @@ impl LiumProofHarvest { )) } - fn spec(&self, pin: &ProofPin, frozen_digest: &str) -> InstanceSpec { + /// Rent spec for one run: the executor plan's template at exactly its + /// width. A raw Lium UUID is rented verbatim; a digest-scoped template + /// name is resolved (or created) bound to `eval_image@digest`. + fn spec(&self, pin: &ProofPin, frozen_digest: &str, plan: &ExecutorPlan) -> InstanceSpec { + let template_name = if plan.template_is_uuid { + harvest_template_name(&pin.eval_image, &pin.eval_image_digest) + } else { + plan.template_id.clone() + }; InstanceSpec { name: format!("proof-{}", &frozen_digest[..12.min(frozen_digest.len())]), max_lifetime_hours: self.limits.max_lifetime_hours, max_price_per_hour: self.limits.max_price_per_hour, - gpu_count: self.limits.gpu_count, + gpu_count: plan.gpu_count, image_digest: Some(pin.eval_image_digest.clone()), docker_image: Some(pin.eval_image.clone()), startup_commands: None, ssh_public_keys: self.ssh_public_keys.clone(), ssh_key_name: Some(SSH_KEY_NAME.to_owned()), preferred_offer_id: None, - template_id: None, - template_name: Some(harvest_template_name( - &pin.eval_image, - &pin.eval_image_digest, - )), + template_id: plan.template_is_uuid.then(|| plan.template_id.clone()), + template_name: Some(template_name), + exact_gpu_count: true, } } + + /// Stage, then run under the deadline, then always tear down. The run + /// wait is bounded here too: a pod that never reports back is not a + /// reason to hold the submission open past the deadline. + async fn run_on_pod( + &self, + instance: &str, + body: &[u8], + env: &[u8], + extras: &RunExtras, + deadline_s: u64, + ) -> Result { + let outcome = match self.pod.stage(instance, body, env, extras).await { + Err(e) => Err(EvalError::Backend(e)), + Ok(()) => { + let wait = + Duration::from_secs(deadline_s.saturating_add(self.deadline_wait_grace_secs)); + match tokio::time::timeout(wait, self.pod.run(instance, Some(deadline_s))).await { + Ok(Ok(stdout)) => Ok(stdout), + Ok(Err(e)) => Err(EvalError::Backend(e)), + Err(_elapsed) => Err(EvalError::ProofDeadlineExceeded { + deadline_s, + stdout_tail: format!( + "harvest wait exceeded the {deadline_s}s deadline (+{}s grace); \ + no pod stdout", + self.deadline_wait_grace_secs + ), + }), + } + } + }; + match self.pod.shutdown(instance).await { + Ok(true) => {} + Ok(false) => { + return Err(EvalError::Integrity(format!( + "pod {instance} terminate not verified" + ))) + } + Err(e) => return Err(EvalError::Backend(e)), + } + let stdout = outcome?; + if hit_deadline(&stdout) { + let stdout_tail = truncate_tail(&stdout, STDOUT_TAIL_BYTES); + tracing::warn!( + instance, + deadline_s, + stdout_tail = %stdout_tail, + "eval run cut at the proof deadline; refusing" + ); + return Err(EvalError::ProofDeadlineExceeded { + deadline_s, + stdout_tail, + }); + } + Ok(stdout) + } } #[async_trait] @@ -460,6 +581,7 @@ impl LiveScorer for LiumProofHarvest { pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, + executor: &EvalExecutorOffer, frozen_digest: &str, artifact_digest: &str, holdout: &[HoldoutRecord], @@ -472,6 +594,10 @@ impl LiveScorer for LiumProofHarvest { return Err(EvalError::HoldoutSealed); } self.ready()?; + // Executor first: no judge env, no proxy tar, no rent unless the + // open 1x executor, the topic tighten, and any operator override + // collapse into a legal plan. + let plan = self.plan(pin, topic, executor)?; offer .serves_topic(pin, topic) .map_err(|e| EvalError::InferenceOffer(e.to_string()))?; @@ -512,6 +638,9 @@ impl LiveScorer for LiumProofHarvest { max_input_tokens: max_in, max_output_tokens: max_out, config_commitment: offer.config_commitment.clone(), + executor_offer_id: plan.offer_id.clone(), + executor_commitment: plan.config_commitment.clone(), + max_proof_deadline_s: plan.deadline_s, eval_image_digest: pin.eval_image_digest.clone(), holdout_commitment: topic.holdout_commitment.clone(), constraints: topic.constraints, @@ -523,23 +652,21 @@ impl LiveScorer for LiumProofHarvest { let body = serde_json::to_vec(&request) .map_err(|e| EvalError::Backend(format!("encode request: {e}")))?; + tracing::info!( + executor_offer_id = %plan.offer_id, + template_id = %plan.template_id, + gpu_count = plan.gpu_count, + deadline_s = plan.deadline_s, + "proof harvest rent plan" + ); let instance = self .pod - .boot(&self.spec(pin, frozen_digest)) + .boot(&self.spec(pin, frozen_digest, &plan)) .await .map_err(EvalError::Backend)?; - let run = self.pod.run(&instance, &body, &env, &extras).await; - let shutdown = self.pod.shutdown(&instance).await; - match shutdown { - Ok(true) => {} - Ok(false) => { - return Err(EvalError::Integrity(format!( - "pod {instance} terminate not verified" - ))) - } - Err(e) => return Err(EvalError::Backend(e)), - } - let stdout = run.map_err(EvalError::Backend)?; + let stdout = self + .run_on_pod(&instance, &body, &env, &extras, plan.deadline_s) + .await?; if !PROGRAM.ran_to_completion(&stdout) { let stdout_tail = truncate_tail(&stdout, STDOUT_TAIL_BYTES); tracing::warn!( @@ -625,6 +752,9 @@ mod tests { max_input_tokens: 4_096, max_output_tokens: 256, config_commitment: "ab".repeat(32), + executor_offer_id: "lium-1x-v0".into(), + executor_commitment: "cd".repeat(32), + max_proof_deadline_s: 3_600, eval_image_digest: String::new(), holdout_commitment: topic.holdout_commitment.clone(), constraints: topic.constraints, @@ -639,8 +769,11 @@ mod tests { assert_eq!(v["challenge_id"], "proof"); assert_eq!(v["provider_kind"], "openai_compatible"); assert_eq!(v["mode"], "chat"); + assert_eq!(v["executor_offer_id"], "lium-1x-v0"); + assert_eq!(v["max_proof_deadline_s"], 3_600); assert!(v.get("proxy_model").is_none()); assert!(v.get("api_key").is_none()); + assert!(v.get("lium_api_key").is_none()); } #[test] @@ -665,6 +798,9 @@ mod tests { extras: std::sync::Mutex, proxy_tar_len: std::sync::Mutex, booted: std::sync::Mutex, + spec: std::sync::Mutex>, + run_deadline: std::sync::Mutex>>, + shutdowns: std::sync::Mutex, } impl CapturePod { @@ -675,24 +811,36 @@ mod tests { extras: std::sync::Mutex::new(RunExtras::default()), proxy_tar_len: std::sync::Mutex::new(0), booted: std::sync::Mutex::new(false), + spec: std::sync::Mutex::new(None), + run_deadline: std::sync::Mutex::new(None), + shutdowns: std::sync::Mutex::new(0), }) } + + fn spec(&self) -> InstanceSpec { + self.spec + .lock() + .expect("spec") + .clone() + .expect("booted spec") + } } #[async_trait] impl EvalPod for CapturePod { - async fn boot(&self, _spec: &InstanceSpec) -> Result { + async fn boot(&self, spec: &InstanceSpec) -> Result { *self.booted.lock().expect("boot") = true; + *self.spec.lock().expect("spec") = Some(spec.clone()); Ok("pod-1".into()) } - async fn run( + async fn stage( &self, _instance_id: &str, request: &[u8], env_file: &[u8], extras: &RunExtras, - ) -> Result { + ) -> Result<(), String> { *self.request.lock().expect("req") = request.to_vec(); *self.env.lock().expect("env") = env_file.to_vec(); *self.extras.lock().expect("extras") = extras.clone(); @@ -702,10 +850,20 @@ mod tests { .and_then(|p| std::fs::metadata(p).ok()) .map_or(0, |m| m.len()); *self.proxy_tar_len.lock().expect("proxy len") = n; + Ok(()) + } + + async fn run( + &self, + _instance_id: &str, + deadline_secs: Option, + ) -> Result { + *self.run_deadline.lock().expect("deadline") = Some(deadline_secs); Err("captured".into()) } async fn shutdown(&self, _instance_id: &str) -> Result { + *self.shutdowns.lock().expect("shutdowns") += 1; Ok(true) } } @@ -714,12 +872,28 @@ mod tests { let mut p = ProofPin { eval_image_digest: format!("sha256:{}", "ab".repeat(32)), topic_pubkey: "ab".repeat(32), + allowed_lium_template_prefixes: vec!["proof-eval-".into()], ..ProofPin::default() }; p.inference.model = "master-proxy-v0".into(); p } + /// Open `1x` executor on the digest-scoped template of [`harvest_pin`]. + fn harvest_executor() -> EvalExecutorOffer { + let mut o = EvalExecutorOffer { + offer_id: "lium-1x-v0".into(), + lium_template_id: "proof-eval-abababababab".into(), + machine_shape: "1x".into(), + max_proof_deadline_s: 3_600, + eval_image_digest: harvest_pin().eval_image_digest, + config_commitment: String::new(), + status: proof_executor::OfferStatus::Open, + }; + o.config_commitment = o.expected_commitment(); + o + } + fn harvest_offer() -> InferenceOffer { let config = proof_task::InferenceConfig { mode: proof_task::InferenceMode::Chat, @@ -806,6 +980,8 @@ mod tests { .with_judge_api_key(key) .with_proxy_model_dir(Some(proxy)) .with_holdout_store(Some(store)) + // Deterministic: never read PROOF_HARVEST_* from the test process env. + .with_harvest_overrides(Some(HarvestOverrides::default())) } #[tokio::test] @@ -820,6 +996,7 @@ mod tests { &harvest_pin(), &topic, &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, @@ -878,6 +1055,7 @@ mod tests { &harvest_pin(), &topic, &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, @@ -905,6 +1083,7 @@ mod tests { &harvest_pin(), &topic, &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, @@ -936,6 +1115,7 @@ mod tests { &harvest_pin(), &harvest_topic(&recs), &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, @@ -1010,6 +1190,7 @@ mod tests { &harvest_pin(), &harvest_topic(&recs), &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &missing, @@ -1207,6 +1388,7 @@ mod tests { &harvest_pin(), &topic, &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, @@ -1249,12 +1431,20 @@ mod tests { Ok("pod-1".into()) } - async fn run( + async fn stage( &self, _instance_id: &str, _request: &[u8], _env_file: &[u8], _extras: &RunExtras, + ) -> Result<(), String> { + Ok(()) + } + + async fn run( + &self, + _instance_id: &str, + _deadline_secs: Option, ) -> Result { Ok(self.stdout.clone()) } @@ -1264,6 +1454,353 @@ mod tests { } } + fn deadline_stdout(rc: u32) -> String { + format!( + "{}\nexit={rc}\nTraceback: still training step 4200 when the proof deadline hit\n", + "boot ok\n".repeat(3) + ) + } + + #[tokio::test] + async fn harvest_rents_the_executor_template_at_exactly_one_gpu_under_its_deadline() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(HarvestOverrides::default())); + let mut topic = harvest_topic(&recs); + topic.eval_executor.max_proof_deadline_s = Some(1_800); + let err = harvest + .score( + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("capture"); + assert!(matches!(err, EvalError::Backend(_)), "{err}"); + let spec = pod.spec(); + assert_eq!(spec.gpu_count, 1); + assert!(spec.exact_gpu_count, "any other rent width must abort"); + assert_eq!( + spec.template_id, None, + "a template name is resolved, not rented verbatim" + ); + assert_eq!( + spec.template_name.as_deref(), + Some("proof-eval-abababababab") + ); + assert_eq!( + spec.image_digest.as_deref(), + Some(harvest_pin().eval_image_digest.as_str()) + ); + assert_eq!( + *pod.run_deadline.lock().expect("deadline"), + Some(Some(1_800)), + "topic tightens the offer deadline and the pod run is held to it" + ); + assert_eq!(*pod.shutdowns.lock().expect("shutdowns"), 1); + let req: serde_json::Value = + serde_json::from_slice(&pod.request.lock().expect("req")).expect("json"); + assert_eq!(req["executor_offer_id"], "lium-1x-v0"); + assert_eq!( + req["executor_commitment"], + harvest_executor().config_commitment + ); + assert_eq!(req["max_proof_deadline_s"], 1_800); + } + + #[tokio::test] + async fn harvest_rents_a_uuid_template_verbatim() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let pod = CapturePod::new(); + let mut pin = harvest_pin(); + pin.allowed_lium_template_prefixes.clear(); + let mut executor = harvest_executor(); + executor.lium_template_id = "f2f5e84c-3b09-4090-be83-1913eabd009e".into(); + executor.config_commitment = executor.expected_commitment(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(HarvestOverrides::default())); + let _ = harvest + .score( + &pin, + &harvest_topic(&recs), + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await; + let spec = pod.spec(); + assert_eq!( + spec.template_id.as_deref(), + Some("f2f5e84c-3b09-4090-be83-1913eabd009e") + ); + assert_eq!( + spec.template_name.as_deref(), + Some("proof-eval-abababababab") + ); + assert!(spec.exact_gpu_count); + } + + #[tokio::test] + async fn harvest_refuses_a_closed_wide_or_topic_mismatched_executor_before_boot() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let mut closed = harvest_executor(); + closed.status = proof_executor::OfferStatus::Closed; + let mut wide = harvest_executor(); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + let mut pinned_topic = harvest_topic(&recs); + pinned_topic.eval_executor.require_offer_commitment = Some("cd".repeat(32)); + for (label, executor, topic, want) in [ + ("closed", closed, harvest_topic(&recs), "closed"), + ("8x", wide, harvest_topic(&recs), "machine_shape"), + ( + "topic pins another executor", + harvest_executor(), + pinned_topic, + "cannot serve", + ), + ] { + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(HarvestOverrides::default())); + let err = harvest + .score( + &harvest_pin(), + &topic, + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err(label); + assert!(err.to_string().contains(want), "{label}: {err}"); + assert!( + !*pod.booted.lock().expect("booted"), + "{label} must not rent" + ); + } + } + + #[tokio::test] + async fn harvest_env_override_aborts_any_width_but_one_and_never_loosens_the_ceiling() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + for (overrides, want) in [ + ( + HarvestOverrides { + gpu_count: Some(8), + ..HarvestOverrides::default() + }, + "abort: executor would rent 8x", + ), + ( + HarvestOverrides { + deadline_secs: Some(7_201), + ..HarvestOverrides::default() + }, + "max_proof_deadline_s = 7201", + ), + ( + HarvestOverrides { + template_id: Some("prism-recipe-v10".into()), + ..HarvestOverrides::default() + }, + "allowed_lium_template_prefixes", + ), + ] { + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(overrides)); + let err = harvest + .score( + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("override refused"); + assert!( + matches!(err, EvalError::ExecutorOffer(ref m) if m.contains(want)), + "{err}" + ); + assert!(!*pod.booted.lock().expect("booted")); + } + + // A legal hot-swap: same width, shorter deadline, another allowed template. + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(HarvestOverrides { + template_id: Some("proof-eval-abababababab-hotfix".into()), + gpu_count: Some(1), + deadline_secs: Some(600), + })); + let _ = harvest + .score( + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await; + assert!(*pod.booted.lock().expect("booted")); + assert_eq!( + pod.spec().template_name.as_deref(), + Some("proof-eval-abababababab-hotfix") + ); + assert_eq!(*pod.run_deadline.lock().expect("deadline"), Some(Some(600))); + } + + #[tokio::test] + async fn a_run_cut_at_the_deadline_is_a_503_with_the_stdout_tail() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + for rc in harvest_pod::DEADLINE_EXIT_CODES { + let (proxy, store) = live_asset_dirs(&recs); + let pod = StdoutPod::new(deadline_stdout(rc)); + let harvest = LiumProofHarvest::new( + pod.clone(), + HarvestLimits::default(), + vec!["ssh-ed25519 AAAAtest proof".into()], + ) + .with_judge_api_key(Some("sk-live-not-a-real-secret".into())) + .with_proxy_model_dir(Some(proxy)) + .with_holdout_store(Some(store)) + .with_harvest_overrides(Some(HarvestOverrides::default())); + let err = harvest + .score( + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("deadline"); + match err { + EvalError::ProofDeadlineExceeded { + deadline_s, + stdout_tail, + } => { + assert_eq!(deadline_s, 3_600); + assert!(stdout_tail.contains(&format!("exit={rc}")), "{stdout_tail}"); + assert!(stdout_tail.contains("step 4200"), "{stdout_tail}"); + } + other => panic!("exit={rc}: expected deadline refuse, got {other}"), + } + assert!(*pod.booted.lock().expect("booted")); + } + } + + struct HangingPod { + shutdowns: std::sync::Mutex, + } + + #[async_trait] + impl EvalPod for HangingPod { + async fn boot(&self, _spec: &InstanceSpec) -> Result { + Ok("pod-hang".into()) + } + + async fn stage( + &self, + _instance_id: &str, + _request: &[u8], + _env_file: &[u8], + _extras: &RunExtras, + ) -> Result<(), String> { + Ok(()) + } + + async fn run( + &self, + _instance_id: &str, + _deadline_secs: Option, + ) -> Result { + tokio::time::sleep(Duration::from_secs(30)).await; + Ok("PROOF_EVAL_OK\n".into()) + } + + async fn shutdown(&self, _instance_id: &str) -> Result { + *self.shutdowns.lock().expect("shutdowns") += 1; + Ok(true) + } + } + + #[tokio::test] + async fn a_pod_that_never_reports_back_is_torn_down_at_the_deadline() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (proxy, store) = live_asset_dirs(&recs); + let pod = Arc::new(HangingPod { + shutdowns: std::sync::Mutex::new(0), + }); + let mut executor = harvest_executor(); + executor.max_proof_deadline_s = 1; + executor.config_commitment = executor.expected_commitment(); + let harvest = LiumProofHarvest::new( + pod.clone(), + HarvestLimits::default(), + vec!["ssh-ed25519 AAAAtest proof".into()], + ) + .with_judge_api_key(Some("sk-live-not-a-real-secret".into())) + .with_proxy_model_dir(Some(proxy)) + .with_holdout_store(Some(store)) + .with_harvest_overrides(Some(HarvestOverrides::default())) + .with_deadline_wait_grace_secs(0); + let err = harvest + .score( + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("wait elapsed"); + assert!( + matches!( + err, + EvalError::ProofDeadlineExceeded { deadline_s: 1, ref stdout_tail } + if stdout_tail.contains("harvest wait exceeded") + ), + "{err}" + ); + assert_eq!( + *pod.shutdowns.lock().expect("shutdowns"), + 1, + "the pod is terminated even when its run never returned" + ); + } + #[tokio::test] async fn harvest_refuses_stdout_without_ok_marker() { let recs = synthetic_holdout(STRATUM_SIZE, 1); @@ -1283,6 +1820,7 @@ mod tests { &harvest_pin(), &topic, &harvest_offer(), + &harvest_executor(), "digest-abcdef", "artifact", &recs, From 987882746728133a27b4954991d6c2b4f4843ca5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:45:32 +0000 Subject: [PATCH 06/16] feat(proof-http): public executor route, admin rotate, and executor-gated status/submit Co-authored-by: Mathis --- Cargo.lock | 1 + crates/proof-http/Cargo.toml | 1 + crates/proof-http/src/lib.rs | 445 ++++++++++++++++++++- crates/proof-http/tests/live_submit_e2e.rs | 15 + crates/proof-store/src/lib.rs | 7 + 5 files changed, 466 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bbb8950a..955100709 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3524,6 +3524,7 @@ dependencies = [ "hex", "http-body-util", "proof-eval", + "proof-executor", "proof-score", "proof-store", "proof-task", diff --git a/crates/proof-http/Cargo.toml b/crates/proof-http/Cargo.toml index b58f750c2..cefd57a83 100644 --- a/crates/proof-http/Cargo.toml +++ b/crates/proof-http/Cargo.toml @@ -12,6 +12,7 @@ publish = false axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } hex = "0.4" proof-eval = { path = "../proof-eval" } +proof-executor = { path = "../proof-executor" } proof-score = { path = "../proof-score" } proof-store = { path = "../proof-store" } proof-task = { path = "../proof-task" } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 8607144a6..4f6f49109 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -5,10 +5,12 @@ //! GET /v1/status //! GET /v1/proof/topics //! GET /v1/proof/topics/{id} -//! POST /v1/submissions miner submit (topic_id required) +//! GET /v1/proof/executor public EvalExecutorOffer + pin ceilings +//! POST /v1/submissions miner submit (topic_id required) //! GET /v1/submissions //! GET /v1/submissions/{id} -//! POST /v1/admin/proof/topics operator publish (signed document) +//! POST /v1/admin/proof/topics operator publish (signed document) +//! POST /v1/admin/proof/executor operator rotate the live executor offer //! ``` #![forbid(unsafe_code)] @@ -20,7 +22,7 @@ clippy::too_many_arguments )] -use std::sync::Arc; +use std::sync::{Arc, PoisonError, RwLock}; use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; @@ -32,6 +34,7 @@ use proof_eval::{ contamination_evidence, eval_after_freeze, force_sim, scoring_readiness, secret_backed_base_url, supported_custom, EvalBackend, EvalError, LiveScorer, }; +use proof_executor::{require_open_executor, EvalExecutorOffer}; use proof_score::{ judge_topic, primary_from_harness, AgentVerdict, GateFail, HarnessMetrics, MinerTopicRun, ProofKind, ProofVerdict, @@ -46,6 +49,16 @@ use proof_task::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +/// Live executor offer slot: rotated at runtime by the admin route, read on +/// every status / submit. In-memory like the rest of the submission state; +/// the boot value comes from `PROOF_EVAL_EXECUTOR_OFFER_FILE`. +pub type ExecutorSlot = Arc>>; + +/// Build an [`ExecutorSlot`] holding `offer`. +pub fn executor_slot(offer: Option) -> ExecutorSlot { + Arc::new(RwLock::new(offer)) +} + /// Shared HTTP state. #[derive(Clone)] pub struct AppState { @@ -60,6 +73,9 @@ pub struct AppState { pub live_scorer: Option>, /// Live RLM judge backend (operator state). Missing/closed → can_score false. pub offer: Option, + /// Live `1x` eval executor (operator state). On the Lium path + /// missing/closed/shape ≠ pin → can_score false. + pub executor: ExecutorSlot, /// Judge API key from `PROOF_INFERENCE_API_KEY_FILE`. Never on `/v1/status`. pub judge_api_key: Option, /// Operator bearer hashes (sha256 hex). Empty → admin 503. @@ -73,6 +89,31 @@ impl AppState { self.live_scorer.as_deref() } + /// Snapshot of the live executor offer (a poisoned lock still reads). + pub fn executor_offer(&self) -> Option { + self.executor + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn set_executor_offer(&self, offer: EvalExecutorOffer) { + *self + .executor + .write() + .unwrap_or_else(PoisonError::into_inner) = Some(offer); + } + + fn executor_pin_view(&self) -> serde_json::Value { + serde_json::json!({ + "schema_version": self.pin.eval_executor_schema_version, + "gpu_class": self.pin.gpu_class, + "max_proof_deadline_s_ceiling": self.pin.max_proof_deadline_s_ceiling, + "allowed_lium_template_prefixes": self.pin.allowed_lium_template_prefixes, + "commitment_alg": self.pin.eval_executor_commitment_alg, + }) + } + fn can_score(&self) -> bool { let open = self.store.any_open_scorable(self.epoch).unwrap_or(false); if scoring_readiness( @@ -81,6 +122,7 @@ impl AppState { self.live(), open, self.offer.as_ref(), + self.executor_offer().as_ref(), self.judge_api_key.as_deref(), ) .is_err() @@ -108,9 +150,11 @@ pub fn proof_router(state: AppState) -> Router { .route("/v1/status", get(status)) .route("/v1/proof/topics", get(list_topics)) .route("/v1/proof/topics/{id}", get(get_topic)) + .route("/v1/proof/executor", get(get_executor)) .route("/v1/submissions", post(submit).get(list_subs)) .route("/v1/submissions/{id}", get(get_sub)) .route("/v1/admin/proof/topics", post(publish_topic)) + .route("/v1/admin/proof/executor", post(rotate_executor)) .with_state(state) } @@ -139,6 +183,8 @@ async fn status(State(st): State) -> impl IntoResponse { "max_input_tokens": st.pin.inference.max_input_tokens, "max_output_tokens": st.pin.inference.max_output_tokens, }, + "eval_executor": st.executor_offer().as_ref().map(EvalExecutorOffer::public_view), + "executor": st.executor_pin_view(), "eval_backend": st.backend, "force_sim": force_sim(), "sim_stub_win": st.backend == EvalBackend::Sim, @@ -166,6 +212,20 @@ async fn get_topic( Ok(Json(doc)) } +/// Public executor contract: the live offer (every field is public), whether +/// it can rent right now, and the pin ceilings it is bound by. Always 200 — +/// a missing offer is `eval_executor: null` with the reason, never a 404. +async fn get_executor(State(st): State) -> impl IntoResponse { + let offer = st.executor_offer(); + let readiness = require_open_executor(offer.as_ref(), &st.pin).map(|_| ()); + Json(serde_json::json!({ + "eval_executor": offer.as_ref().map(EvalExecutorOffer::public_view), + "ready": readiness.is_ok(), + "reason": readiness.err().map(|e| e.to_string()), + "pin": st.executor_pin_view(), + })) +} + #[derive(Debug, Deserialize)] struct SubmitBody { miner_hotkey: String, @@ -243,12 +303,16 @@ async fn submit( let nonce = nonce_from(&hotkey, &topic_id, &artifact); let submission_digest = freeze_submission_digest(&hotkey, &topic_id, &artifact, &nonce); + // One snapshot of the executor for this request: rotation mid-submit + // must not score under one offer and stamp another. + let executor = st.executor_offer(); scoring_readiness( &st.pin, st.backend, st.live(), st.store.any_open_scorable(st.epoch).unwrap_or(false), st.offer.as_ref(), + executor.as_ref(), st.judge_api_key.as_deref(), ) .map_err(|e| eval_err(&e))?; @@ -258,6 +322,13 @@ async fn submit( offer .serves_topic(&st.pin, &topic) .map_err(|e| offer_err(&e))?; + if st.backend == EvalBackend::Lium { + executor + .as_ref() + .ok_or(EvalError::ExecutorOfferMissing) + .and_then(|x| x.serves_topic(&topic).map_err(proof_eval::map_executor_err)) + .map_err(|e| eval_err(&e))?; + } let resolved = resolve_inference( &st.pin, Some(&topic.inference), @@ -295,6 +366,7 @@ async fn submit( }; return persist_pre_eval_reject( &st, + executor.as_ref(), body, &topic, hotkey, @@ -309,6 +381,7 @@ async fn submit( &st.pin, &topic, offer, + executor.as_ref(), &submission_digest, &artifact, &holdout, @@ -332,6 +405,7 @@ async fn submit( let receipt_json = serde_json::to_string(&eval.receipt).unwrap_or_default(); persist_scored( &st, + executor.as_ref(), body, hotkey, artifact, @@ -346,6 +420,7 @@ async fn submit( #[allow(clippy::too_many_arguments)] fn persist_pre_eval_reject( st: &AppState, + executor: Option<&EvalExecutorOffer>, body: SubmitBody, topic: &TopicDocument, hotkey: String, @@ -394,6 +469,10 @@ fn persist_pre_eval_reject( .as_ref() .map(|o| o.config_commitment.clone()) .unwrap_or_default(), + executor_offer_id: executor.map(|x| x.offer_id.clone()).unwrap_or_default(), + executor_commitment: executor + .map(|x| x.config_commitment.clone()) + .unwrap_or_default(), manifest: body.manifest, nonce, submission_digest, @@ -429,6 +508,7 @@ fn persist_pre_eval_reject( #[allow(clippy::too_many_arguments)] fn persist_scored( st: &AppState, + executor: Option<&EvalExecutorOffer>, body: SubmitBody, hotkey: String, artifact: String, @@ -472,6 +552,10 @@ fn persist_scored( .as_ref() .map(|o| o.config_commitment.clone()) .unwrap_or_default(), + executor_offer_id: executor.map(|x| x.offer_id.clone()).unwrap_or_default(), + executor_commitment: executor + .map(|x| x.config_commitment.clone()) + .unwrap_or_default(), manifest: body.manifest, nonce, submission_digest, @@ -548,6 +632,35 @@ async fn publish_topic( Ok((StatusCode::CREATED, Json(doc))) } +/// Rotate the live executor offer. The body is the same document as +/// `PROOF_EVAL_EXECUTOR_OFFER_FILE`; it must validate against the pin +/// (shape, deadline ceiling, template allowlist, digest, commitment) or it is +/// a 400 and the previous offer stays. Posting `status: closed` is how an +/// operator takes the executor down without a restart. +async fn rotate_executor( + State(st): State, + headers: HeaderMap, + Json(offer): Json, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + offer + .validate(&st.pin) + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + st.set_executor_offer(offer.clone()); + Ok(( + StatusCode::CREATED, + Json(serde_json::json!({ + "eval_executor": offer.public_view(), + "can_score": st.can_score(), + })), + )) +} + fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { let Some(raw) = headers .get(axum::http::header::AUTHORIZATION) @@ -793,6 +906,7 @@ mod tests { pin: &ProofPin, topic: &TopicDocument, _offer: &InferenceOffer, + _executor: &EvalExecutorOffer, frozen: &str, artifact: &str, _holdout: &[proof_task::HoldoutRecord], @@ -841,12 +955,16 @@ mod tests { } let judge_api_key = (backend == EvalBackend::Lium && live.is_some()).then(|| "test-judge-key".to_owned()); + // Lium + a wired harvest is the live path: it also needs the open 1x + // executor. Sim rents nothing, so the slot stays empty there. + let executor = (backend == EvalBackend::Lium && live.is_some()).then(|| test_executor(&p)); proof_router(AppState { store, pin: p, backend, live_scorer: live, offer: with_offer.then(offer), + executor: executor_slot(executor), // Lium + a wired harvest is the live path: a missing key is the // Testeur blocker. Sim does not call the judge, so it stays None. judge_api_key, @@ -859,6 +977,46 @@ mod tests { app_full(token, EvalBackend::Sim, "", None, true, true, true) } + /// Open `1x` executor on the digest-scoped template of `pin`. + fn test_executor(pin: &ProofPin) -> EvalExecutorOffer { + let hex = pin.eval_image_digest.trim_start_matches("sha256:"); + let mut o = EvalExecutorOffer { + offer_id: "lium-1x-v0".into(), + lium_template_id: format!("proof-eval-{}", hex.get(..12).unwrap_or("unpinned")), + machine_shape: "1x".into(), + max_proof_deadline_s: 3_600, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: proof_executor::OfferStatus::Open, + }; + o.config_commitment = o.expected_commitment(); + o + } + + /// Lium host with a wired harvest whose executor slot holds `executor`. + fn app_lium_with_executor(executor: Option) -> Router { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let store = MemoryStore::new(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, meas) = seal_topic(&p, unsigned_topic(&recs)); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, meas.into_sealed()) + .expect("baseline"); + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Lium, + live_scorer: Some(Arc::new(StubScorer::win())), + offer: Some(offer()), + executor: executor_slot(executor), + judge_api_key: Some("test-judge-key".into()), + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + async fn json_req( app: Router, method: &str, @@ -928,6 +1086,283 @@ mod tests { assert!(!dump.contains("base_url"), "{dump}"); assert!(!dump.contains("api_key"), "{dump}"); assert!(!dump.contains("evil.example"), "{dump}"); + // Sim rents nothing: no executor offer, still scorable; the pin + // ceilings are public regardless. + assert!(body["eval_executor"].is_null(), "{body}"); + assert_eq!(body["executor"]["gpu_class"], "1x"); + assert_eq!(body["executor"]["max_proof_deadline_s_ceiling"], 7_200); + } + + #[tokio::test] + async fn status_and_executor_route_expose_the_public_executor_contract() { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let app = app_lium_with_executor(Some(test_executor(&p))); + let (st, body) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(body["can_score"], true, "{body}"); + assert_eq!(body["eval_executor"]["offer_id"], "lium-1x-v0"); + assert_eq!(body["eval_executor"]["machine_shape"], "1x"); + assert_eq!(body["eval_executor"]["gpu_count"], 1); + assert_eq!(body["eval_executor"]["max_proof_deadline_s"], 3_600); + assert_eq!( + body["eval_executor"]["lium_template_id"], + "proof-eval-abababababab" + ); + assert_eq!(body["eval_executor"]["status"], "open"); + assert_eq!(body["executor"]["gpu_class"], "1x"); + assert_eq!(body["executor"]["schema_version"], 1); + + let (st, view) = json_req( + app, + "GET", + "/v1/proof/executor", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(view["ready"], true, "{view}"); + assert!(view["reason"].is_null(), "{view}"); + assert_eq!(view["eval_executor"]["offer_id"], "lium-1x-v0"); + assert_eq!(view["pin"]["max_proof_deadline_s_ceiling"], 7_200); + let dump = view.to_string(); + assert!(!dump.contains("api_key"), "{dump}"); + assert!(!dump.contains("/run/base"), "{dump}"); + } + + #[tokio::test] + async fn missing_closed_or_wide_executor_is_can_score_false_and_submit_503() { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let mut closed = test_executor(&p); + closed.status = proof_executor::OfferStatus::Closed; + let mut wide = test_executor(&p); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + for (label, executor, want) in [ + ("missing", None, "executor offer missing"), + ("closed", Some(closed), "closed"), + ("8x", Some(wide), "machine_shape"), + ] { + let app = app_lium_with_executor(executor); + let (st, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["live_harvest_wired"], true, "{label}: {status}"); + assert_eq!(status["can_score"], false, "{label}: {status}"); + + let (st, view) = json_req( + app.clone(), + "GET", + "/v1/proof/executor", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(view["ready"], false, "{label}: {view}"); + assert!( + view["reason"].as_str().unwrap_or_default().contains(want), + "{label}: {view}" + ); + + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("x", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{label}: {body}"); + assert!( + body["error"].as_str().unwrap_or_default().contains(want), + "{label}: {body}" + ); + let (_, list) = + json_req(app, "GET", "/v1/submissions", serde_json::json!({}), None).await; + assert!( + list["items"].as_array().is_some_and(Vec::is_empty), + "{label} banked rows: {list}" + ); + } + } + + #[tokio::test] + async fn admin_rotate_executor_requires_bearer_validates_and_takes_effect() { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let app = app_lium_with_executor(None); + let good = serde_json::to_value(test_executor(&p)).expect("json"); + + let (st, _) = json_req( + app.clone(), + "POST", + "/v1/admin/proof/executor", + good.clone(), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + + let mut wide = test_executor(&p); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/admin/proof/executor", + serde_json::to_value(&wide).expect("json"), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("machine_shape"), + "{body}" + ); + let mut forged = test_executor(&p); + forged.config_commitment = "cd".repeat(32); + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/admin/proof/executor", + serde_json::to_value(&forged).expect("json"), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + let (_, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!( + status["can_score"], false, + "refused rotations leave the slot empty" + ); + + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/admin/proof/executor", + good, + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eval_executor"]["offer_id"], "lium-1x-v0"); + assert_eq!(created["can_score"], true, "{created}"); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("after-rotate", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + let id = created["id"].as_str().expect("id"); + let (_, row) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + None, + ) + .await; + assert_eq!(row["executor_offer_id"], "lium-1x-v0", "{row}"); + assert_eq!( + row["executor_commitment"], + test_executor(&p).config_commitment, + "{row}" + ); + + // Closing is the same route: the host stops scoring without a restart. + let mut closed = test_executor(&p); + closed.status = proof_executor::OfferStatus::Closed; + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/admin/proof/executor", + serde_json::to_value(&closed).expect("json"), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{body}"); + assert_eq!(body["can_score"], false, "{body}"); + let (st, body) = json_req( + app, + "POST", + "/v1/submissions", + submit_body("after-close", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + } + + #[tokio::test] + async fn topic_pinning_another_executor_commitment_is_503() { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let store = MemoryStore::new(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (mut topic, meas) = seal_topic(&p, unsigned_topic(&recs)); + topic.eval_executor.require_offer_commitment = Some("cd".repeat(32)); + topic.eval_executor.max_proof_deadline_s = Some(900); + topic.signature = topic.sign_with(&sk()).expect("sign"); + topic.validate(&p, &[]).expect("tightened topic is legal"); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, meas.into_sealed()) + .expect("baseline"); + let scorer = Arc::new(StubScorer::win()); + let app = proof_router(AppState { + store, + pin: p.clone(), + backend: EvalBackend::Lium, + live_scorer: Some(scorer.clone()), + offer: Some(offer()), + executor: executor_slot(Some(test_executor(&p))), + judge_api_key: Some("test-judge-key".into()), + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }); + let (st, body) = json_req( + app, + "POST", + "/v1/submissions", + submit_body("pinned", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("cannot serve"), + "{body}" + ); + assert_eq!(scorer.hits.load(Ordering::SeqCst), 0, "no rent"); } #[tokio::test] @@ -1329,6 +1764,7 @@ mod tests { backend: EvalBackend::Lium, live_scorer: Some(Arc::new(StubScorer::win())), offer: Some(offer()), + executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), epoch: 0, @@ -1417,6 +1853,7 @@ mod tests { backend: EvalBackend::Sim, live_scorer: None, offer: Some(offer()), + executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), epoch: 0, @@ -1457,6 +1894,7 @@ mod tests { backend: EvalBackend::Sim, live_scorer: None, offer: Some(staging_offer()), + executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), epoch: 0, @@ -1664,6 +2102,7 @@ mod tests { backend: EvalBackend::Sim, live_scorer: None, offer: Some(staging_offer()), + executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), epoch: 0, diff --git a/crates/proof-http/tests/live_submit_e2e.rs b/crates/proof-http/tests/live_submit_e2e.rs index 218938cb7..35974965a 100644 --- a/crates/proof-http/tests/live_submit_e2e.rs +++ b/crates/proof-http/tests/live_submit_e2e.rs @@ -78,6 +78,21 @@ async fn live_host_submit_scores_or_fails_closed() { let dump = status.to_string(); assert!(!dump.contains("api_key"), "{dump}"); assert!(!dump.contains("content_sha256"), "{dump}"); + assert_eq!(status["executor"]["gpu_class"], "1x", "{status}"); + + // Public executor contract: always 200, `ready` says whether the live 1x + // offer can rent, `reason` names the refusal when it cannot. + let (st, executor) = get(&client, &format!("{base}/v1/proof/executor")).await; + assert_eq!(st, 200, "{executor}"); + assert!(executor["ready"].is_boolean(), "{executor}"); + assert_eq!(executor["pin"]["gpu_class"], "1x", "{executor}"); + if executor["ready"] == false { + assert!( + executor["reason"].as_str().is_some_and(|r| !r.is_empty()), + "silent not-ready executor: {executor}" + ); + } + assert!(!executor.to_string().contains("api_key"), "{executor}"); let (st, topics) = get(&client, &format!("{base}/v1/proof/topics")).await; assert_eq!(st, 200, "{topics}"); diff --git a/crates/proof-store/src/lib.rs b/crates/proof-store/src/lib.rs index 1be8cb37f..88ca8f4e6 100644 --- a/crates/proof-store/src/lib.rs +++ b/crates/proof-store/src/lib.rs @@ -82,6 +82,13 @@ pub struct Submission { /// Judge `config_commitment` stamped from the host offer. #[serde(default)] pub config_commitment: String, + /// Live `EvalExecutorOffer` id the run was rented on (host stamp; empty + /// on sim, which rents nothing). + #[serde(default)] + pub executor_offer_id: String, + /// Executor `config_commitment` stamped from the host offer. + #[serde(default)] + pub executor_commitment: String, /// Declared training fingerprints. #[serde(default)] pub manifest: ArtifactManifest, From f55b5a37f498242ad3db36e87cec2f82d3990401 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:48:19 +0000 Subject: [PATCH 07/16] feat(proof-challenge): load PROOF_EVAL_EXECUTOR_OFFER_FILE and wire the executor slot Co-authored-by: Mathis --- Cargo.lock | 1 + bins/proof-challenge/src/main.rs | 122 +++++++++++++++++++++++++++++- crates/proof-challenge/Cargo.toml | 1 + crates/proof-challenge/src/lib.rs | 6 +- 4 files changed, 126 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 955100709..be2c47341 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3426,6 +3426,7 @@ dependencies = [ "crypto", "hex", "proof-eval", + "proof-executor", "proof-http", "proof-score", "proof-store", diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index ccc85d4ab..42b57f71a 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -19,9 +19,9 @@ use challenge_keys::load_challenge_secret; use clap::Parser; use prism_lium::LiumClient; use proof_challenge::{ - hash_admin_token, parse_holdout_file, proof_router, AppState, BaselineMeasurement, EvalBackend, - InferenceOffer, LiveScorer, MemoryStore, ProofPin, TopicDocument, CHALLENGE_ID, - SCORING_VERSION, + executor_slot, hash_admin_token, parse_holdout_file, proof_router, AppState, + BaselineMeasurement, EvalBackend, EvalExecutorOffer, HarvestOverrides, InferenceOffer, + LiveScorer, MemoryStore, ProofPin, TopicDocument, CHALLENGE_ID, SCORING_VERSION, }; use proof_eval::supported_custom; use proof_harvest::{HarvestLimits, LiumProofHarvest}; @@ -71,6 +71,11 @@ struct Cli { /// Provider API key file. Never logged, never on `/v1/status`. #[arg(long, env = "PROOF_INFERENCE_API_KEY_FILE")] inference_api_key_file: Option, + /// Live `1x` `EvalExecutorOffer` JSON (Lium template + proof deadline). + /// Operator state; never a git pin. Rotated at runtime via + /// `POST /v1/admin/proof/executor`. Missing/closed/shape ≠ 1x → 503. + #[arg(long, env = "PROOF_EVAL_EXECUTOR_OFFER_FILE")] + eval_executor_offer_file: Option, /// Local measurement weights staged onto the eval pod (no HF bake). #[arg(long, env = "PROOF_PROXY_MODEL_DIR")] proxy_model_dir: Option, @@ -151,6 +156,36 @@ fn run(cli: &Cli) -> Result<(), String> { ), _ => {} } + let executor = match load_executor(&pin, cli.eval_executor_offer_file.as_deref()) { + Ok(x) => { + tracing::info!( + offer_id = %x.offer_id, + lium_template_id = %x.lium_template_id, + machine_shape = %x.machine_shape, + max_proof_deadline_s = x.max_proof_deadline_s, + status = ?x.status, + "eval executor offer loaded" + ); + Some(x) + } + Err(e) => { + if backend == EvalBackend::Lium { + tracing::warn!( + "eval executor offer unavailable ({e}); live submits will 503 until \ + PROOF_EVAL_EXECUTOR_OFFER_FILE holds an open 1x offer or one is posted \ + to /v1/admin/proof/executor" + ); + } + None + } + }; + match HarvestOverrides::from_env() { + Ok(o) if !o.is_empty() => { + tracing::info!(?o, "PROOF_HARVEST_* override set; pin ceilings still bind"); + } + Ok(_) => {} + Err(e) => tracing::warn!("{e}; every live harvest will refuse until it is fixed"), + } let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -181,6 +216,7 @@ fn run(cli: &Cli) -> Result<(), String> { backend, live_scorer, offer, + executor: executor_slot(executor), judge_api_key, admin_hashes: Arc::new(load_admin_hashes(cli.admin_tokens_file.as_deref())), epoch: 0, @@ -359,6 +395,14 @@ fn load_offer(pin: &ProofPin, path: Option<&Path>) -> Result) -> Result { + let p = path.ok_or("PROOF_EVAL_EXECUTOR_OFFER_FILE not set")?; + let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; + let offer = EvalExecutorOffer::from_json(&body).map_err(|e| e.to_string())?; + offer.validate(pin).map_err(|e| e.to_string())?; + Ok(offer) +} + fn load_admin_hashes(path: Option<&Path>) -> Vec { let Some(p) = path else { return Vec::new(); @@ -519,4 +563,76 @@ mod tests { } static OFFER_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// Compose sets `PROOF_EVAL_EXECUTOR_OFFER_FILE`. Missing / unparseable / + /// non-`1x` is `can_score=false` / submit 503 — never `exit 1`. A valid + /// `1x` offer on the committed pin's digest-scoped template loads. + #[test] + fn compose_executor_offer_env_parses_and_bad_files_are_not_boot_errors() { + let _guard = OFFER_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::env::set_var( + "PROOF_EVAL_EXECUTOR_OFFER_FILE", + "/run/base/proof/eval_executor_offer.json", + ); + let cli = Cli::try_parse_from(["proof-challenge"]) + .unwrap_or_else(|e| panic!("PROOF_EVAL_EXECUTOR_OFFER_FILE broke parsing: {e}")); + assert_eq!( + cli.eval_executor_offer_file.as_deref(), + Some(Path::new("/run/base/proof/eval_executor_offer.json")) + ); + std::env::remove_var("PROOF_EVAL_EXECUTOR_OFFER_FILE"); + + let pin_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/proof-pin.toml"); + let pin = load_pin(Some(&pin_path)).expect("committed pin"); + let missing = load_executor(&pin, Some(Path::new("/nonexistent/executor.json"))) + .expect_err("missing file is unavailable, not a panic"); + assert!(missing.contains("read"), "{missing}"); + assert!(load_executor(&pin, None) + .expect_err("unset") + .contains("PROOF_EVAL_EXECUTOR_OFFER_FILE")); + + let dir = std::env::temp_dir().join(format!( + "proof-executor-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let hex = pin.eval_image_digest.trim_start_matches("sha256:"); + let mut good = EvalExecutorOffer { + offer_id: "lium-1x-v0".into(), + lium_template_id: format!("proof-eval-{}", &hex[..12]), + machine_shape: "1x".into(), + max_proof_deadline_s: 7_200, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: proof_challenge::OfferStatus::Open, + }; + good.config_commitment = good.expected_commitment(); + let good_path = dir.join("good.json"); + std::fs::write(&good_path, serde_json::to_vec(&good).expect("json")).expect("write"); + let loaded = load_executor(&pin, Some(&good_path)).expect("valid 1x offer loads"); + assert_eq!(loaded, good); + + let mut wide = good.clone(); + wide.machine_shape = "8x".into(); + wide.config_commitment = wide.expected_commitment(); + let wide_path = dir.join("wide.json"); + std::fs::write(&wide_path, serde_json::to_vec(&wide).expect("json")).expect("write"); + let err = load_executor(&pin, Some(&wide_path)).expect_err("8x is refused"); + assert!(err.contains("machine_shape"), "{err}"); + + let junk_path = dir.join("junk.json"); + std::fs::write( + &junk_path, + b"{\"offer_id\":\"x\",\"lium_api_key\":\"nope\"}", + ) + .expect("write"); + let err = load_executor(&pin, Some(&junk_path)).expect_err("unknown key"); + assert!(err.contains("lium_api_key"), "{err}"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/proof-challenge/Cargo.toml b/crates/proof-challenge/Cargo.toml index 028f1fbd1..385b49884 100644 --- a/crates/proof-challenge/Cargo.toml +++ b/crates/proof-challenge/Cargo.toml @@ -13,6 +13,7 @@ bundle = { path = "../bundle" } challenge-common = { path = "../challenge-common" } hex = "0.4" proof-eval = { path = "../proof-eval" } +proof-executor = { path = "../proof-executor" } proof-http = { path = "../proof-http" } proof-score = { path = "../proof-score" } proof-store = { path = "../proof-store" } diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index 70d18f397..df2ff8516 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -19,7 +19,11 @@ pub use proof_eval::{ force_sim, resolve_eval_backend, scoring_readiness, sim_stub_win, supported_custom, BaselineMeasurement, EvalBackend, LiveScorer, }; -pub use proof_http::{hash_admin_token, proof_router, AppState}; +pub use proof_executor::{ + EvalExecutorOffer, ExecutorOfferError, HarvestOverrides, OfferStatus, + EVAL_EXECUTOR_OFFER_FILE_ENV, +}; +pub use proof_http::{executor_slot, hash_admin_token, proof_router, AppState, ExecutorSlot}; pub use proof_store::{ArtifactManifest, MemoryStore}; pub use proof_task::{ HoldoutRecord, InferenceOffer, OfferError, ProofPin, TopicDocument, BASE_MODEL_FAMILY, From 3ceb90eea6a60459e6a2b91602e5dec795b401d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:50:06 +0000 Subject: [PATCH 08/16] feat(xtask): proof-executor-offer ceremony builds a pin-validated 1x offer Co-authored-by: Mathis --- Cargo.lock | 1 + xtask/Cargo.toml | 1 + xtask/src/main.rs | 55 +++++++++ xtask/src/proof_executor_offer.rs | 191 ++++++++++++++++++++++++++++++ 4 files changed, 248 insertions(+) create mode 100644 xtask/src/proof_executor_offer.rs diff --git a/Cargo.lock b/Cargo.lock index be2c47341..802848b56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6134,6 +6134,7 @@ dependencies = [ "frame-metadata", "hex", "parity-scale-codec", + "proof-executor", "proof-task", "regex", "reqwest 0.12.28", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index b66b85b2b..941d6de07 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -19,6 +19,7 @@ frame-metadata = { version = "16", features = ["std", "decode"] } hex = "0.4" parity-scale-codec = { version = "3.7", features = ["derive"] } regex = "1.11" +proof-executor = { path = "../crates/proof-executor" } proof-task = { path = "../crates/proof-task" } reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls-native-roots"] } scale-info = { version = "2.11", default-features = false, features = ["std"] } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 444799517..e1e5d3362 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -10,6 +10,7 @@ //! - `external-docs-check` — fail if external miner docs `protocol_version` ≠ bundle, or D19 drifts //! - `proof-holdout` — select a Proof per-topic holdout set and print its commitment //! - `proof-topic` — sign a Proof YAML/JSON topic draft (fills holdout + signature) +//! - `proof-executor-offer` — build + validate a Proof `1x` EvalExecutorOffer (computes the commitment) #![allow(clippy::print_stdout, clippy::print_stderr)] mod consensus_lint; @@ -18,6 +19,7 @@ mod external_docs_check; mod loc_cap; mod metadata_snapshot; mod natural_pack; +mod proof_executor_offer; mod proof_holdout; mod proof_topic; mod spec_check; @@ -135,6 +137,36 @@ enum Command { #[arg(long)] synthetic: bool, }, + /// Build and validate a Proof `1x` `EvalExecutorOffer` (computes `config_commitment`). + /// + /// Operator state for `PROOF_EVAL_EXECUTOR_OFFER_FILE` / `POST /v1/admin/proof/executor`. + /// Never written under a tracked path; never carries a secret. + ProofExecutorOffer { + /// Pin the offer must validate against. + #[arg(long, default_value = "config/proof-pin.toml")] + pin: PathBuf, + /// Immutable slug for this executor. + #[arg(long)] + offer_id: String, + /// Lium template id or digest-scoped template name. Defaults to `proof-eval-<12 hex>` of the pin digest. + #[arg(long)] + template_id: Option, + /// Machine shape (must be the pin `gpu_class`, `1x`). + #[arg(long, default_value = "1x")] + machine_shape: String, + /// Proof deadline in seconds (`<=` pin `max_proof_deadline_s_ceiling`). + #[arg(long)] + max_proof_deadline_s: u64, + /// Leave `eval_image_digest` empty instead of binding the pin digest. + #[arg(long)] + unbound_digest: bool, + /// Publish `status: closed`. + #[arg(long)] + closed: bool, + /// Write the JSON here (outside the repo). Stdout when omitted. + #[arg(long)] + out: Option, + }, } fn workspace_root() -> Result { @@ -240,5 +272,28 @@ fn dispatch(command: Command, root: &Path) -> Result<(), String> { holdout, synthetic, }), + Command::ProofExecutorOffer { + pin, + offer_id, + template_id, + machine_shape, + max_proof_deadline_s, + unbound_digest, + closed, + out, + } => proof_executor_offer::run(&proof_executor_offer::ExecutorOfferArgs { + pin: if pin.is_absolute() { + pin + } else { + root.join(pin) + }, + offer_id, + template_id, + machine_shape, + max_proof_deadline_s, + bind_digest: !unbound_digest, + closed, + out, + }), } } diff --git a/xtask/src/proof_executor_offer.rs b/xtask/src/proof_executor_offer.rs new file mode 100644 index 000000000..059974e42 --- /dev/null +++ b/xtask/src/proof_executor_offer.rs @@ -0,0 +1,191 @@ +//! Proof eval executor offer ceremony helper. +//! +//! Builds the operator `EvalExecutorOffer` document (the `1x` Lium template +//! the digest-pinned `proof-eval` image is rented on, plus its proof +//! deadline), computes `config_commitment`, and validates it against +//! `config/proof-pin.toml` before writing. The document is operator state: +//! `PROOF_EVAL_EXECUTOR_OFFER_FILE` at boot, `POST /v1/admin/proof/executor` +//! to rotate. It never enters git and it never carries a secret. + +use std::fs; +use std::path::{Path, PathBuf}; + +use proof_executor::{EvalExecutorOffer, OfferStatus}; +use proof_task::ProofPin; + +/// Arguments for the executor offer ceremony. +#[derive(Debug)] +pub struct ExecutorOfferArgs { + /// Pin the offer must validate against (`config/proof-pin.toml`). + pub pin: PathBuf, + /// Immutable slug for this executor. + pub offer_id: String, + /// Lium template id, or the digest-scoped template name + /// (`proof-eval-<12 hex>`). Defaults to the pin's digest-scoped name. + pub template_id: Option, + /// Machine shape. Must be the pin `gpu_class` (`1x`). + pub machine_shape: String, + /// Proof deadline in seconds (`<=` pin ceiling). + pub max_proof_deadline_s: u64, + /// Bind the offer to the pin's eval image digest. + pub bind_digest: bool, + /// Publish closed (host cannot score until reopened). + pub closed: bool, + /// Where to write the JSON (never inside the repo). Stdout when omitted. + pub out: Option, +} + +/// Build, validate, and emit the offer. +/// +/// # Errors +/// +/// Unreadable / invalid pin, or an offer the pin refuses (wrong shape, +/// deadline over the ceiling, template outside the allowlist, …). +pub fn run(args: &ExecutorOfferArgs) -> Result<(), String> { + let body = fs::read_to_string(&args.pin) + .map_err(|e| format!("read pin {}: {e}", args.pin.display()))?; + let pin = ProofPin::from_toml(&body).map_err(|e| e.to_string())?; + pin.validate().map_err(|e| e.to_string())?; + let offer = build(&pin, args)?; + let json = serde_json::to_string_pretty(&offer).map_err(|e| e.to_string())?; + match &args.out { + Some(out) => { + refuse_repo_path(out)?; + if let Some(parent) = out.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("mkdir {}: {e}", parent.display()))?; + } + fs::write(out, format!("{json}\n")) + .map_err(|e| format!("write {}: {e}", out.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(out, fs::Permissions::from_mode(0o600)); + } + println!( + "executor offer {} ({}, {}, {}s) → {}", + offer.offer_id, + offer.lium_template_id, + offer.machine_shape, + offer.max_proof_deadline_s, + out.display() + ); + } + None => println!("{json}"), + } + Ok(()) +} + +fn build(pin: &ProofPin, args: &ExecutorOfferArgs) -> Result { + let hex = pin.eval_image_digest.trim().trim_start_matches("sha256:"); + let template_id = match args.template_id.as_deref().map(str::trim) { + Some(t) if !t.is_empty() => t.to_owned(), + _ => { + let prefix = hex.get(..12).ok_or( + "pin has no eval_image_digest; pass --template-id explicitly (pre-launch pin)", + )?; + format!("proof-eval-{prefix}") + } + }; + let mut offer = EvalExecutorOffer { + offer_id: args.offer_id.trim().to_owned(), + lium_template_id: template_id, + machine_shape: args.machine_shape.trim().to_owned(), + max_proof_deadline_s: args.max_proof_deadline_s, + eval_image_digest: if args.bind_digest { + pin.eval_image_digest.trim().to_owned() + } else { + String::new() + }, + config_commitment: String::new(), + status: if args.closed { + OfferStatus::Closed + } else { + OfferStatus::Open + }, + }; + offer.config_commitment = offer.expected_commitment(); + offer.validate(pin).map_err(|e| e.to_string())?; + Ok(offer) +} + +fn refuse_repo_path(out: &Path) -> Result<(), String> { + let text = out.to_string_lossy(); + if text.contains("/config/") || text.starts_with("config/") || text.contains("/docs/") { + return Err(format!( + "refusing to write a live executor offer under a tracked path: {text}" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn committed_pin() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../config/proof-pin.toml") + } + + fn args(shape: &str, deadline: u64) -> ExecutorOfferArgs { + ExecutorOfferArgs { + pin: committed_pin(), + offer_id: "lium-1x-v0".into(), + template_id: None, + machine_shape: shape.into(), + max_proof_deadline_s: deadline, + bind_digest: true, + closed: false, + out: None, + } + } + + fn pin() -> ProofPin { + let p = + ProofPin::from_toml(&fs::read_to_string(committed_pin()).expect("pin")).expect("parse"); + p.validate().expect("valid"); + p + } + + #[test] + fn builds_a_valid_one_gpu_offer_on_the_committed_pin() { + let p = pin(); + let offer = build(&p, &args("1x", 7_200)).expect("offer"); + assert_eq!(offer.machine_shape, "1x"); + assert!(offer.lium_template_id.starts_with("proof-eval-")); + assert_eq!(offer.eval_image_digest, p.eval_image_digest); + assert_eq!(offer.config_commitment, offer.expected_commitment()); + assert!(offer.is_open()); + offer.validate(&p).expect("validates"); + let round: EvalExecutorOffer = + serde_json::from_str(&serde_json::to_string(&offer).expect("json")).expect("parse"); + assert_eq!(round, offer); + } + + #[test] + fn refuses_a_shape_or_deadline_the_pin_does_not_allow() { + let p = pin(); + let err = build(&p, &args("8x", 7_200)).expect_err("8x"); + assert!(err.contains("machine_shape"), "{err}"); + let err = build(&p, &args("1x", 7_201)).expect_err("over ceiling"); + assert!(err.contains("max_proof_deadline_s"), "{err}"); + let mut off_list = args("1x", 600); + off_list.template_id = Some("prism-recipe-v10".into()); + let err = build(&p, &off_list).expect_err("allowlist"); + assert!(err.contains("allowed_lium_template_prefixes"), "{err}"); + } + + #[test] + fn closed_offers_and_tracked_paths() { + let p = pin(); + let mut closed = args("1x", 600); + closed.closed = true; + assert!(!build(&p, &closed).expect("closed").is_open()); + assert!(refuse_repo_path(Path::new("config/eval_executor_offer.json")).is_err()); + assert!(refuse_repo_path(Path::new("/x/docs/offer.json")).is_err()); + refuse_repo_path(Path::new( + "/root/.base-secrets/proof/eval_executor_offer.json", + )) + .expect("operator path"); + } +} From dc93dd76bc71da6c6d90ad141064bb2082e349db Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 11:55:00 +0000 Subject: [PATCH 09/16] docs(proof): document the EvalExecutorOffer contract, admin route, and harvest 1x enforcement Co-authored-by: Mathis --- AGENTS.md | 7 ++-- deploy/env/proof-challenge.env.example | 19 +++++++++ deploy/scripts/proof-operator-path.sh | 22 +++++++++-- deploy/secrets/README.md | 3 +- docker-compose.yml | 3 ++ docs/COMPLETENESS.md | 1 + docs/OPERATOR_SECURITY.md | 1 + docs/PROOF.md | 55 +++++++++++++++++++++++--- docs/external-miner/proof.md | 20 ++++++---- docs/runbooks/proof-submit-e2e.md | 10 ++++- 10 files changed, 121 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 29b9da854..1c9a37f6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,12 +71,13 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 2. Edge / failure probes: bad harness, sanitize reject, quota, wrong routes/auth. 3. **Bounty — pair + report:** `ctx bounty pair --hotkey --account-id --accept-terms`, then `POST /v1/pair` (terms + signature) and `POST /v1/reports`. Operator bearer `POST /v1/admin/adjudicate` (`valid` / `already_fixed_not_prod` / `invalid_malicious` / `duplicate`). Scoring **reads** CortexLM/backend public JSON (`BOUNTY_BACKEND_PUBLIC_URL`); do not serve `/v1/public/*` from this repo. 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). -5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer. Contamination / empty manifest persist **rejected** without rent. `GET /v1/proof/topics` must never leak holdout records. -6. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). +5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent. `GET /v1/proof/topics` must never leak holdout records. +6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. +7. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. -**Bounty product rules:** pay is precision x severity, an unpriced `valid` row is not creditable, and the triage-noise ratio stays off the visible score. **Proof product rules (do not weaken):** topics are operator-published signed documents, not a git catalog; a topic may tighten a floor never loosen it; a baseline must be sealed to open; each topic is `wta` (winner takes the topic mass) or `discovery` (pass floor + novelty); global miner score is the **sum** of per-topic masses, not a mean of binary lattices; empty open set / empty eval digest fails closed (`503`); `custom` unknown ids refuse at publish; `harness_success_rate` is listed and fail-closes until the real harness exists. +**Bounty product rules:** pay is precision x severity, an unpriced `valid` row is not creditable, and the triage-noise ratio stays off the visible score. **Proof product rules (do not weaken):** topics are operator-published signed documents, not a git catalog; a topic may tighten a floor never loosen it; a baseline must be sealed to open; each topic is `wta` (winner takes the topic mass) or `discovery` (pass floor + novelty); global miner score is the **sum** of per-topic masses, not a mean of binary lattices; empty open set / empty eval digest fails closed (`503`); `custom` unknown ids refuse at publish; `harness_success_rate` is listed and fail-closes until the real harness exists; the eval executor is exactly `1x` (pin `gpu_class`), a topic may only tighten `eval_executor.max_proof_deadline_s` / pin `require_offer_commitment`, and there is no per-topic `machine_id`. Local smoke automates the weights seal step via `weights-smoke` inside `./deploy/scripts/local-e2e.sh --smoke` (see [`deploy/AGENTS.md`](deploy/AGENTS.md) and [`docs/runbooks/local-testnet-e2e.md`](docs/runbooks/local-testnet-e2e.md)). diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index ec1827ed9..583859c57 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -69,3 +69,22 @@ PROOF_SIM_STUB_WIN=false # omit one. Never log. Prefer the file form on a droplet. # PROOF_INFERENCE_BASE_URL= # PROOF_INFERENCE_BASE_URL_FILE=/run/base/proof/inference_base_url + +# Live 1x EvalExecutorOffer (operator state, never git, no secret): the Lium +# template the digest-pinned proof-eval image is rented on and the proof +# deadline the run is held to. A sibling of the InferenceOffer, not the same +# document. Build it with: +# cargo run -p xtask -- proof-executor-offer --offer-id lium-1x-v0 \ +# --max-proof-deadline-s 7200 --out deploy/secrets/proof/eval_executor_offer.json +# Rotate live with POST /v1/admin/proof/executor (operator bearer). Missing, +# closed, or any machine_shape other than the pin gpu_class (1x) → +# can_score=false and submits 503. Any rent that would not be exactly 1 GPU +# aborts before the rent. +# PROOF_EVAL_EXECUTOR_OFFER_FILE=/run/base/proof/eval_executor_offer.json +# +# Optional hot-swap of the executor plan without a rebuild. Each replaces the +# offer's value; the pin ceilings still bind (a value outside them refuses the +# rent, it is never clamped). Unparseable values also refuse. Leave unset. +# PROOF_HARVEST_TEMPLATE_ID= +# PROOF_HARVEST_GPU_COUNT=1 +# PROOF_HARVEST_DEADLINE_SECS= diff --git a/deploy/scripts/proof-operator-path.sh b/deploy/scripts/proof-operator-path.sh index c8cfb560e..4f3cc5c4c 100755 --- a/deploy/scripts/proof-operator-path.sh +++ b/deploy/scripts/proof-operator-path.sh @@ -80,14 +80,30 @@ cargo run -p xtask -- proof-topic \\ # -H 'content-type: application/json' \\ # --data-binary @${SECRETS}/topics.json -# 6. Point the host at the operator files (never in git): +# 6. Build the live 1x EvalExecutorOffer (Lium template + proof deadline; +# no secret inside, still never in git). The pin refuses any other shape. +cargo run -p xtask -- proof-executor-offer \\ + --offer-id lium-1x-v0 \\ + --max-proof-deadline-s 7200 \\ + --out '${SECRETS}/eval_executor_offer.json' +# Rotate or close live without a restart: +# curl -sS -X POST "\$PROOF_BASE/v1/admin/proof/executor" \\ +# -H "authorization: Bearer \$PROOF_ADMIN_TOKEN" \\ +# -H 'content-type: application/json' \\ +# --data-binary @${SECRETS}/eval_executor_offer.json + +# 7. Point the host at the operator files (never in git): # PROOF_TOPICS_FILE=${SECRETS}/topics.json # PROOF_HOLDOUT_FILE=${SECRETS}/holdouts.json # PROOF_BASELINE_FILE=${SECRETS}/baselines.json +# PROOF_EVAL_EXECUTOR_OFFER_FILE=${SECRETS}/eval_executor_offer.json # LIUM_API_KEY=… LIUM_SSH_PUBLIC_KEY_FILE=… # Restart proof-challenge, then: -# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,baseline_sealed}' +# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,baseline_sealed,eval_executor}' +# curl -sS "\$PROOF_BASE/v1/proof/executor" | jq '{ready,reason}' # can_score is true only with: real digest + harvest wired + open topic + -# sealed baseline + verified holdout. Empty digest stays 503. +# sealed baseline + verified holdout + open 1x executor offer. Empty digest +# stays 503. Harvest rents exactly 1 GPU on the offer template and cuts the +# run at max_proof_deadline_s (503 + stdout_tail). EOF diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 8336d98b6..4b115dfd3 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -34,10 +34,11 @@ chmod 0400 deploy/secrets/gateway_admin_token | `proof/topics.json` | proof-challenge | Signed topic documents (JSON array). **Never commit secrets**; the documents themselves are operator-published. Mode **0400**, uid **65532** | | `proof/holdouts.json` | proof-challenge | Per-topic holdout records (array or map keyed by `topic_id`). **Never commit.** Verified at boot against each topic's `holdout_commitment`. Mode **0400**, uid **65532** | | `proof/baselines.json` | proof-challenge | Sealed baseline measurements keyed by topic id. **Never commit.** Mode **0400**, uid **65532** | -| `proof/admin_tokens` | proof-challenge | One operator bearer per line for `POST /v1/admin/proof/topics` | +| `proof/admin_tokens` | proof-challenge | One operator bearer per line for `POST /v1/admin/proof/topics` and `POST /v1/admin/proof/executor` | | `proof/inference_offer.json` | proof-challenge | Live RLM judge `InferenceOffer` (provider kind, origin, mode, model_ref, token caps, `config_commitment`, status). Consumed by proof-eval; **not** a miner training proxy. **Never commit.** Missing/closed → `can_score=false` / 503. Mode **0400**, uid **65532** | | `proof/inference_api_key` | proof-challenge | Provider API key for the eval image. **Never commit, never log.** Mode **0400**, uid **65532** | | `proof/inference_base_url` | proof-challenge | Optional secret-backed origin (`PROOF_INFERENCE_BASE_URL_FILE`) when pin `[inference].base_url` and the topic omit one. **Never commit, never log.** Mode **0400**, uid **65532** | +| `proof/eval_executor_offer.json` | proof-challenge | Live `1x` `EvalExecutorOffer` (`offer_id`, `lium_template_id`, `machine_shape`, `max_proof_deadline_s`, `eval_image_digest`, `config_commitment`, status). Sibling of the judge offer, no secret inside; still operator state, **never commit**. Build with `cargo run -p xtask -- proof-executor-offer …`; rotate live via `POST /v1/admin/proof/executor`. Missing/closed/shape ≠ pin `gpu_class` → `can_score=false` / 503. Mode **0400**, uid **65532** | | `bounty/admin_tokens` | bounty-challenge | Operator bearer for `POST /v1/admin/adjudicate` | | `bounty/session_secret` | bounty-challenge | Pairing session HMAC secret. Empty/missing no longer crashes boot (`/health` stays up; pairing will not survive restart). `remote-deploy.sh` fills a 32-byte value from urandom when the file is missing or 0-length. | diff --git a/docker-compose.yml b/docker-compose.yml index ed18f2f30..406caf9bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -262,6 +262,9 @@ services: PROOF_ADMIN_TOKENS_FILE: /run/base/proof/admin_tokens PROOF_INFERENCE_OFFER_FILE: /run/base/proof/inference_offer.json PROOF_INFERENCE_API_KEY_FILE: /run/base/proof/inference_api_key + # Live 1x EvalExecutorOffer (Lium template + proof deadline). Operator + # state, no secret; missing/closed → can_score=false / submit 503. + PROOF_EVAL_EXECUTOR_OFFER_FILE: /run/base/proof/eval_executor_offer.json BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} env_file: - path: ./deploy/env/proof-challenge.env diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 11d40a19d..60d1aae70 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -79,6 +79,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Compose / images | **done** | Default compose + `images.yml` target `proof-challenge`. | | Eval pin | **done** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, digest `sha256:78b614a1…` (publish-proof-eval-image run 33892650063, commit `51f937c7`). No HF bake; `proxy_model` stays empty. Live submits still **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. Do not re-pin a guessed sha256. | | Inference offer | **v0** | Digest-pinned RLM **judge** backend (`proof-eval` / harvest call it). Pin `[inference]` defaults plus schema v1 / ceilings / modes / commitment. `config_commitment` hashes config knobs **and** `provider.base_url`; a topic that spoofs origin is **503** before lattice. Topic `require_judge_offer_commitment` is optional and not a miner bind. Live `InferenceOffer` is operator state. Auth is `PROOF_INFERENCE_API_KEY_FILE` staged as harvest `teacher.env` (never git, never `/v1/status`). Missing/closed/judge down / missing key → `can_score=false` / 503. No baked Qwen; architecture ≠ HF stays retired. | +| Eval executor offer | **v0** | `crates/proof-executor`: live `1x` `EvalExecutorOffer` (Lium template, `machine_shape`, `max_proof_deadline_s`, digest, `config_commitment`, status) — a sibling of the judge offer, not the same document. Pin ceilings `eval_executor_schema_version` / `gpu_class = "1x"` / `max_proof_deadline_s_ceiling = 7200` / optional `allowed_lium_template_prefixes` / `eval_executor_commitment_alg`. Public on `GET /v1/status` + `GET /v1/proof/executor`; rotated via `POST /v1/admin/proof/executor` (in-memory until restart; boot from `PROOF_EVAL_EXECUTOR_OFFER_FILE`). Topic tighten-only `eval_executor.{require_offer_commitment, max_proof_deadline_s}`, no per-topic `machine_id`. Lium path: missing/closed/shape ≠ `1x` → `can_score=false` / 503; harvest rents the offer template at exactly `1x` (`rent_gpu_count ≠ 1` aborts pre-rent) and holds the run to the deadline (pod `timeout` + harvest wait; exceed → 503 + `stdout_tail`). `PROOF_HARVEST_TEMPLATE_ID` / `_GPU_COUNT` / `_DEADLINE_SECS` hot-swap under the pin ceilings. Sim does not consult it. No live Lium rent in CI. | | Topics | **done** | sr25519 under the `proof` trust-root key (`base-proof-topic-v1`). Admin `POST /v1/admin/proof/topics`. A topic must be sealed to `open`. | | Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. | | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index e78b538d9..5a1328747 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -15,6 +15,7 @@ Use this before every promote and after every incident. Architecture: [`ARCHITEC - [ ] Cloudflare / DO / Phala tokens live only in operator secret stores, not in docs or CI logs. - [ ] Proof miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`) is never written to git, compose env committed files, or logs. Control-plane Lium mounts under `deploy/secrets/lium` are files, mode **0400**, uid **65532**. - [ ] The eval-image InferenceOffer / `proxy_model` is the **RLM judge agent**, not a miner training proxy. Never commit judge credentials. +- [ ] The Proof `EvalExecutorOffer` (`deploy/secrets/proof/eval_executor_offer.json`) is operator state, never git; it carries no secret but names the live Lium template. Only a `1x` offer may be open; `POST /v1/admin/proof/executor` is operator-bearer only. --- diff --git a/docs/PROOF.md b/docs/PROOF.md index 21aae85ec..7320a63b1 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -66,6 +66,34 @@ baseline + an open topic are on the host. either → `can_score=false` / submit **503** (do not rent). After this source fix, republish `proof-eval` and re-pin the new digest before the next 1× GPU rent — do not invent a sha256. +- **Where** the image runs is the live `EvalExecutorOffer` — a sibling of + the judge `InferenceOffer`, never the same document. The pin carries only + ceilings: `eval_executor_schema_version = 1`, `gpu_class = "1x"`, + `max_proof_deadline_s_ceiling = 7200`, optional + `allowed_lium_template_prefixes` (`["proof-eval-"]`, the digest-scoped + harvest template name `proof-eval-<12 hex>`), and + `eval_executor_commitment_alg = sha256`. The live offer + (`PROOF_EVAL_EXECUTOR_OFFER_FILE`; `POST /v1/admin/proof/executor` to + rotate or close) names `offer_id`, `lium_template_id`, `machine_shape`, + `max_proof_deadline_s` (≤ ceiling), `eval_image_digest` (must equal the pin + when non-empty), `config_commitment` = sha256 of the canonical public knobs, + and `status`. Every field is public: `GET /v1/status` (`eval_executor`, + pin `executor`) and `GET /v1/proof/executor` show it whole. Missing / + closed / `machine_shape ≠ 1x` → `can_score=false` → **503** on the Lium + path (sim rents nothing and does not consult it). Harvest rents exactly + that template at exactly `1x` — a rent that would upsize to a whole host + (`rent_gpu_count ≠ 1`) aborts before the rent — and holds the run to the + resolved deadline both pod-side (`timeout`) and in its own wait; a run cut + at the deadline is a **503** carrying the pod's `stdout_tail`, never a + zero. A topic may only tighten: `eval_executor.max_proof_deadline_s` + (shorter) and `eval_executor.require_offer_commitment` (64-hex pin of the + live offer, not a miner bind). There is **no per-topic `machine_id`** + (publish reject). Operator hot-swap without a rebuild: + `PROOF_HARVEST_TEMPLATE_ID` / `PROOF_HARVEST_GPU_COUNT` / + `PROOF_HARVEST_DEADLINE_SECS` replace the offer's values; the pin ceilings + still bind, and an unparseable or out-of-ceiling value refuses the rent + rather than clamping. Ceremony: + `cargo run -p xtask -- proof-executor-offer --offer-id --max-proof-deadline-s --out `. - A baseline must be sealed (`script_sha256` + `metrics_commitment`) to open. Nobody is paid for beating a number nobody measured. - 8000 bps is split equally across currently `open` topics. Each topic then @@ -145,14 +173,24 @@ Trust-root keygen is the throwaway owner path in - `GET /health`, `GET /v1/status` — `can_score`, `eval_backend`, `force_sim`, `live_harvest_wired`, `baseline_sealed`, public pin `inference` judge - defaults (no origin), public `inference_offer` (RLM judge backend). Never - leak origins, keys, or holdout records. + defaults (no origin), public `inference_offer` (RLM judge backend), public + `eval_executor` (live `1x` executor offer) and pin `executor` ceilings. + Never leak origins, keys, or holdout records. - `GET /v1/proof/topics`, `GET /v1/proof/topics/{id}` +- `GET /v1/proof/executor` — always **200**: `eval_executor` (public offer or + `null`), `ready`, `reason` when not ready, and the pin ceilings. - `POST /v1/admin/proof/topics` — operator bearer; verify sig/schema/floors/seal before `open` +- `POST /v1/admin/proof/executor` — operator bearer; body is the offer + document. Pin-validated (**400** keeps the previous offer); `status: closed` + takes the executor down live. In-memory until restart, like submissions — + update `PROOF_EVAL_EXECUTOR_OFFER_FILE` to persist. - `POST /v1/submissions` **requires** `topic_id`. Missing/unknown/not-open → - **400**. Miners do **not** bind the judge offer. Zero open / unsealed - baseline / empty digest / missing or closed RLM judge backend / agent down - → **503**. Refusals must **not** persist rows. + **400**. Miners do **not** bind the judge offer or the executor offer. Zero + open / unsealed baseline / empty digest / missing or closed RLM judge + backend / missing, closed, or non-`1x` executor / agent down / run cut at + the proof deadline → **503**. Refusals must **not** persist rows. Scored + rows stamp `executor_offer_id` + `executor_commitment` next to the judge + `inference_offer_id` + `config_commitment`. - Submit fields miners must send: `claim` (what the recipe achieved), `declared_flops` (≤ topic budget), `artifact_digest` of a **reproducible train/eval recipe** (code under budget, not weights-only), plus `manifest`. @@ -248,3 +286,10 @@ model with no topic `model` is **400** at publish. (`offer_id`, `provider_kind`, `mode`, `model_ref`, token caps, `config_commitment`, `status`). It never leaks `base_url`, API keys, or file paths. Miners do not call this backend. + +It also exposes `eval_executor` (`offer_id`, `lium_template_id`, +`machine_shape`, `gpu_count`, `max_proof_deadline_s`, `eval_image_digest`, +`config_commitment`, `status`) and the pin `executor` ceilings +(`gpu_class`, `max_proof_deadline_s_ceiling`, `allowed_lium_template_prefixes`, +`schema_version`, `commitment_alg`). The executor offer holds no secret, so it +is shown whole. Miners do not rent it and do not pass it. diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 5e3c1019e..a82352993 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -48,10 +48,11 @@ Holdout records are not included in public topic responses. `GET /challenge/proof/v1/status` shows `can_score`, `eval_backend`, `force_sim`, `live_harvest_wired`, `baseline_sealed`, public pin `inference` -judge defaults (provider, model, mode, token caps — never the origin), and -the public RLM judge `inference_offer` (id, kind, mode, model_ref, token -caps, commitment, status). It never leaks holdout records, teacher hosts, -origins, or keys. +judge defaults (provider, model, mode, token caps — never the origin), the +public RLM judge `inference_offer` (id, kind, mode, model_ref, token caps, +commitment, status), and the public `eval_executor` — the `1x` Lium machine +class your recipe is re-run on (template, shape, proof deadline, commitment, +status). It never leaks holdout records, teacher hosts, origins, or keys. Muon, token superposition, and “decentralized training without InfiniBand” are *examples* of solutions or of topics — they are not the product. @@ -72,9 +73,11 @@ curl -sS https://network.cortex.foundation/challenge/proof/v1/status `GET /challenge/proof/v1/status` shows `can_score`, `eval_backend`, `force_sim`, `live_harvest_wired`, `baseline_sealed`, `eval_image_digest`, -public pin `inference` (no origin), public RLM judge `inference_offer`, and +public pin `inference` (no origin), public RLM judge `inference_offer`, +public `eval_executor` (plus the pin `executor` ceilings), and `open_topics`. It never leaks holdout records, teacher hosts, origins, or -keys. +keys. `GET /challenge/proof/v1/proof/executor` shows the executor alone with +`ready` and a `reason` when it cannot rent. `can_score: false` means submits **503**. Nothing is stored and nothing is rented. @@ -83,6 +86,7 @@ rented. |--------------|----------------| | `eval_image_digest` | Must be a `sha256:…` pin (live pin is `sha256:78b614a1…`). Empty → **503** | | `inference_offer` | Public RLM **judge** backend (id, kind, mode, model_ref, token caps, commitment, status). Missing/closed/misconfigured → **503**. You do not pass an offer id | +| `eval_executor` | Public `1x` **executor**: the Lium machine class your recipe is re-run on (`lium_template_id`, `machine_shape`, `max_proof_deadline_s`, commitment, status). Your recipe must finish inside `max_proof_deadline_s` (≤ pin ceiling 7200 s; a topic may name a shorter one) on **one** GPU — the host never rents more. Missing/closed/any shape but `1x` → **503**. You do not pass or rent it | | `open_topics` empty | No currently `open` signed topic with a sealed baseline → **503** | | `baseline_sealed: false` | An open topic without `script_sha256` + `metrics_commitment` → **503** | | `live_harvest_wired: false` | Live RLM harvest is not connected → **503** | @@ -162,7 +166,7 @@ code against the public split and checks the claim against those public numbers. A claim the code cannot support is `unreproduced_claim` / reject. ```bash -ctx proof status # can_score, inference_offer, eval_image_digest +ctx proof status # can_score, inference_offer, eval_executor, eval_image_digest ctx proof topics # pick an open topic_id; read flops_budget + payout_mode ctx proof submit \ @@ -257,6 +261,8 @@ Refusals (**400** / **503**) do **not** persist a submission row. | **503** unsealed baseline | Topic open without both seal hashes | no | no | | **503** live harvest down / unparseable agent verdict | Host cannot judge | no | no | | **503** missing / closed RLM judge backend | Live `InferenceOffer` not scoring | no | no | +| **503** missing / closed / non-`1x` executor | Live `eval_executor` cannot rent the `1x` machine | no | no | +| **503** `proof deadline … exceeded` | Your recipe did not finish inside `max_proof_deadline_s`; the body carries the run's `stdout_tail` | no | no (pod torn down) | | **201** `rejected` + `contamination_evidence_missing` | Empty manifest | **yes** (rejected) | **no** | | **201** `rejected` + contamination | Holdout shard / corpus id in `manifest` | **yes** (rejected) | **no** | diff --git a/docs/runbooks/proof-submit-e2e.md b/docs/runbooks/proof-submit-e2e.md index e4fac7b66..f51b9f7aa 100644 --- a/docs/runbooks/proof-submit-e2e.md +++ b/docs/runbooks/proof-submit-e2e.md @@ -93,10 +93,18 @@ curl -sS "$BASE/v1/status" # "can_score": true, # "baseline_sealed": true, # "open_topics": ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"], -# "inference_offer": { "offer_id": "openrouter-glm53flash-v0", "status": "open", ... } +# "inference_offer": { "offer_id": "openrouter-glm53flash-v0", "status": "open", ... }, +# "eval_executor": null, # sim rents nothing; Lium needs an open 1x offer +# "executor": { "gpu_class": "1x", "max_proof_deadline_s_ceiling": 7200, ... } # } # Never contains api_key, base_url, or holdout records. +curl -sS "$BASE/v1/proof/executor" +# { "eval_executor": null | {...}, "ready": false, "reason": "eval executor offer missing; refuse scoring", "pin": {...} } +# Always 200. On a Lium host `ready: false` means submits 503 until an open 1x +# offer is posted to /v1/admin/proof/executor (operator bearer) or staged in +# the executor offer file (see deploy/env/proof-challenge.env.example). + curl -sS "$BASE/v1/proof/topics" # { "items": [ { "id": "dt-no-ib-v0", ... }, { "id": "muon-vs-adamw-10m-v0", ... } ] } # Never contains content_sha256. From 35157ca65a6888df654fcc9b5b3f384e2691a580 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:05:26 +0000 Subject: [PATCH 10/16] chore(proof): clippy fixes and boot_executor helper Co-authored-by: Mathis --- bins/proof-challenge/src/main.rs | 71 ++++++++++++++++++------------- crates/harvest-pod/src/lib.rs | 2 + crates/proof-executor/src/plan.rs | 3 +- crates/proof-harvest/src/lib.rs | 42 +++++++++--------- xtask/src/main.rs | 2 +- 5 files changed, 66 insertions(+), 54 deletions(-) diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 42b57f71a..81b6c79ba 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -156,36 +156,7 @@ fn run(cli: &Cli) -> Result<(), String> { ), _ => {} } - let executor = match load_executor(&pin, cli.eval_executor_offer_file.as_deref()) { - Ok(x) => { - tracing::info!( - offer_id = %x.offer_id, - lium_template_id = %x.lium_template_id, - machine_shape = %x.machine_shape, - max_proof_deadline_s = x.max_proof_deadline_s, - status = ?x.status, - "eval executor offer loaded" - ); - Some(x) - } - Err(e) => { - if backend == EvalBackend::Lium { - tracing::warn!( - "eval executor offer unavailable ({e}); live submits will 503 until \ - PROOF_EVAL_EXECUTOR_OFFER_FILE holds an open 1x offer or one is posted \ - to /v1/admin/proof/executor" - ); - } - None - } - }; - match HarvestOverrides::from_env() { - Ok(o) if !o.is_empty() => { - tracing::info!(?o, "PROOF_HARVEST_* override set; pin ceilings still bind"); - } - Ok(_) => {} - Err(e) => tracing::warn!("{e}; every live harvest will refuse until it is fixed"), - } + let executor = boot_executor(&pin, backend, cli.eval_executor_offer_file.as_deref()); let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -395,6 +366,46 @@ fn load_offer(pin: &ProofPin, path: Option<&Path>) -> Result, +) -> Option { + match HarvestOverrides::from_env() { + Ok(o) if !o.is_empty() => { + tracing::info!(?o, "PROOF_HARVEST_* override set; pin ceilings still bind"); + } + Ok(_) => {} + Err(e) => tracing::warn!("{e}; every live harvest will refuse until it is fixed"), + } + match load_executor(pin, path) { + Ok(x) => { + tracing::info!( + offer_id = %x.offer_id, + lium_template_id = %x.lium_template_id, + machine_shape = %x.machine_shape, + max_proof_deadline_s = x.max_proof_deadline_s, + status = ?x.status, + "eval executor offer loaded" + ); + Some(x) + } + Err(e) => { + if backend == EvalBackend::Lium { + tracing::warn!( + "eval executor offer unavailable ({e}); live submits will 503 until \ + PROOF_EVAL_EXECUTOR_OFFER_FILE holds an open 1x offer or one is posted \ + to /v1/admin/proof/executor" + ); + } + None + } + } +} + fn load_executor(pin: &ProofPin, path: Option<&Path>) -> Result { let p = path.ok_or("PROOF_EVAL_EXECUTOR_OFFER_FILE not set")?; let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; diff --git a/crates/harvest-pod/src/lib.rs b/crates/harvest-pod/src/lib.rs index ad567f747..a962e632c 100644 --- a/crates/harvest-pod/src/lib.rs +++ b/crates/harvest-pod/src/lib.rs @@ -83,6 +83,8 @@ pub struct RunExtras { /// Exit codes GNU `timeout` reports when the entrypoint hit the deadline: /// `124` on the TERM timeout, `137` when `--kill-after` had to SIGKILL. +/// `137` is also what an external SIGKILL (OOM) reports; either way the run +/// did not finish inside its budget and the harvest refuses with the tail. pub const DEADLINE_EXIT_CODES: [u32; 2] = [124, 137]; /// Seconds the entrypoint gets for one run: the configured run timeout, diff --git a/crates/proof-executor/src/plan.rs b/crates/proof-executor/src/plan.rs index 830061b07..a58235cd9 100644 --- a/crates/proof-executor/src/plan.rs +++ b/crates/proof-executor/src/plan.rs @@ -132,8 +132,7 @@ pub fn executor_plan( let template_id = overrides .template_id .as_deref() - .map(str::trim) - .unwrap_or(offer.lium_template_id.trim()) + .map_or(offer.lium_template_id.trim(), str::trim) .to_owned(); check_template_id(pin, &template_id)?; diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index f0fbf208f..61212c2e3 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -528,23 +528,22 @@ impl LiumProofHarvest { extras: &RunExtras, deadline_s: u64, ) -> Result { - let outcome = match self.pod.stage(instance, body, env, extras).await { - Err(e) => Err(EvalError::Backend(e)), - Ok(()) => { - let wait = - Duration::from_secs(deadline_s.saturating_add(self.deadline_wait_grace_secs)); - match tokio::time::timeout(wait, self.pod.run(instance, Some(deadline_s))).await { - Ok(Ok(stdout)) => Ok(stdout), - Ok(Err(e)) => Err(EvalError::Backend(e)), - Err(_elapsed) => Err(EvalError::ProofDeadlineExceeded { - deadline_s, - stdout_tail: format!( - "harvest wait exceeded the {deadline_s}s deadline (+{}s grace); \ - no pod stdout", - self.deadline_wait_grace_secs - ), - }), - } + let outcome = if let Err(e) = self.pod.stage(instance, body, env, extras).await { + Err(EvalError::Backend(e)) + } else { + let wait = + Duration::from_secs(deadline_s.saturating_add(self.deadline_wait_grace_secs)); + match tokio::time::timeout(wait, self.pod.run(instance, Some(deadline_s))).await { + Ok(Ok(stdout)) => Ok(stdout), + Ok(Err(e)) => Err(EvalError::Backend(e)), + Err(_elapsed) => Err(EvalError::ProofDeadlineExceeded { + deadline_s, + stdout_tail: format!( + "harvest wait exceeded the {deadline_s}s deadline (+{}s grace); \ + no pod stdout", + self.deadline_wait_grace_secs + ), + }), } }; match self.pod.shutdown(instance).await { @@ -799,7 +798,8 @@ mod tests { proxy_tar_len: std::sync::Mutex, booted: std::sync::Mutex, spec: std::sync::Mutex>, - run_deadline: std::sync::Mutex>>, + /// Deadline the harvest handed to `run` (the harvest always passes one). + run_deadline: std::sync::Mutex>, shutdowns: std::sync::Mutex, } @@ -858,7 +858,7 @@ mod tests { _instance_id: &str, deadline_secs: Option, ) -> Result { - *self.run_deadline.lock().expect("deadline") = Some(deadline_secs); + *self.run_deadline.lock().expect("deadline") = deadline_secs; Err("captured".into()) } @@ -1501,7 +1501,7 @@ mod tests { ); assert_eq!( *pod.run_deadline.lock().expect("deadline"), - Some(Some(1_800)), + Some(1_800), "topic tightens the offer deadline and the pod run is held to it" ); assert_eq!(*pod.shutdowns.lock().expect("shutdowns"), 1); @@ -1672,7 +1672,7 @@ mod tests { pod.spec().template_name.as_deref(), Some("proof-eval-abababababab-hotfix") ); - assert_eq!(*pod.run_deadline.lock().expect("deadline"), Some(Some(600))); + assert_eq!(*pod.run_deadline.lock().expect("deadline"), Some(600)); } #[tokio::test] diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e1e5d3362..ac95ebc3d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -10,7 +10,7 @@ //! - `external-docs-check` — fail if external miner docs `protocol_version` ≠ bundle, or D19 drifts //! - `proof-holdout` — select a Proof per-topic holdout set and print its commitment //! - `proof-topic` — sign a Proof YAML/JSON topic draft (fills holdout + signature) -//! - `proof-executor-offer` — build + validate a Proof `1x` EvalExecutorOffer (computes the commitment) +//! - `proof-executor-offer` — build + validate a Proof `1x` `EvalExecutorOffer` (computes the commitment) #![allow(clippy::print_stdout, clippy::print_stderr)] mod consensus_lint; From f7f1004c6fa02d90e1ddba725aa4b06d81e899ba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:09:57 +0000 Subject: [PATCH 11/16] docs(proof): state executor isolation invariants; carry topic scope on the rent plan Co-authored-by: Mathis --- crates/proof-executor/src/lib.rs | 7 +++++++ crates/proof-executor/src/plan.rs | 14 +++++++++++++- crates/proof-harvest/src/lib.rs | 8 ++++++++ docs/PROOF.md | 6 ++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/proof-executor/src/lib.rs b/crates/proof-executor/src/lib.rs index e30608875..c3a80c208 100644 --- a/crates/proof-executor/src/lib.rs +++ b/crates/proof-executor/src/lib.rs @@ -11,6 +11,13 @@ //! //! Every refusal here is fail-closed: a missing, closed, or non-`1x` offer //! means `can_score=false` and submits answer **503**. Nothing here rents. +//! +//! Isolation invariants this contract assumes and never weakens: the +//! control-plane host runs neither the eval image nor the RLM judge; the +//! harvest is the **only** path from the control plane to a rented GPU; an +//! offer names a remote machine class, never a host process or a specific +//! machine. The contract is challenge-agnostic — it carries no topic ids, +//! benchmark names, or model names, only ceilings the operator publishes. #![forbid(unsafe_code)] #![allow( diff --git a/crates/proof-executor/src/plan.rs b/crates/proof-executor/src/plan.rs index a58235cd9..8c376d4f8 100644 --- a/crates/proof-executor/src/plan.rs +++ b/crates/proof-executor/src/plan.rs @@ -100,6 +100,10 @@ impl HarvestOverrides { pub struct ExecutorPlan { /// Offer this plan was resolved from. pub offer_id: String, + /// Topic the rent is scoped to. Carried so a topic-scoped attach (the + /// per-topic judge VM the pod reports to) can key on the plan without + /// changing its shape; this crate does not interpret the id. + pub topic_id: String, /// Lium template id or digest-scoped template name to rent. pub template_id: String, /// `true` when `template_id` is a raw Lium UUID (rent verbatim) rather @@ -164,6 +168,7 @@ pub fn executor_plan( Ok(ExecutorPlan { offer_id: offer.offer_id.clone(), + topic_id: topic.id.clone(), template_is_uuid: is_lium_template_uuid(&template_id), template_id, gpu_count, @@ -180,7 +185,10 @@ mod tests { use crate::fixtures::{offer, offer_for, pin}; fn topic() -> TopicDocument { - TopicDocument::default() + TopicDocument { + id: "any-open-topic-v0".into(), + ..TopicDocument::default() + } } #[test] @@ -193,6 +201,10 @@ mod tests { ) .expect("plan"); assert_eq!(plan.offer_id, "lium-1x-v0"); + assert_eq!( + plan.topic_id, "any-open-topic-v0", + "topic scope rides on the plan" + ); assert_eq!(plan.template_id, "proof-eval-78b614a1f51c"); assert!(!plan.template_is_uuid); assert_eq!(plan.gpu_count, 1); diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 61212c2e3..25c43c73c 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -14,6 +14,13 @@ //! `PROOF_HARVEST_*` override) both by the pod-side `timeout` and by this //! crate's wait. A run cut at the deadline is a **503** carrying the pod's //! stdout tail, never a zero. +//! +//! This crate is the **only** path from the control plane to a rented GPU. +//! The control-plane host runs neither the eval image nor the RLM judge; the +//! judge is a remote offer the image calls from the pod, and the executor is +//! a remote machine class. Nothing here knows a topic beyond the signed +//! document it is handed: no topic ids, benchmarks, or model names are +//! compiled in. #![forbid(unsafe_code)] #![allow( @@ -653,6 +660,7 @@ impl LiveScorer for LiumProofHarvest { tracing::info!( executor_offer_id = %plan.offer_id, + topic_id = %plan.topic_id, template_id = %plan.template_id, gpu_count = plan.gpu_count, deadline_s = plan.deadline_s, diff --git a/docs/PROOF.md b/docs/PROOF.md index 7320a63b1..e27f1a4ae 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -94,6 +94,12 @@ baseline + an open topic are on the host. still bind, and an unparseable or out-of-ceiling value refuses the rent rather than clamping. Ceremony: `cargo run -p xtask -- proof-executor-offer --offer-id --max-proof-deadline-s --out `. + Isolation: the control-plane host runs neither the eval image nor the RLM + judge; the harvest is the **only** path to the rented GPU, and the executor + offer names a remote machine class, never a host process. The contract is + challenge-agnostic — no topic ids, benchmark names, or model names are + compiled in; the rent plan carries the topic id only as scope for a + topic-scoped attach. - A baseline must be sealed (`script_sha256` + `metrics_commitment`) to open. Nobody is paid for beating a number nobody measured. - 8000 bps is split equally across currently `open` topics. Each topic then From 23892811e3941dc156ac1a5d01e2a5e4d8f704e3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:57:13 +0000 Subject: [PATCH 12/16] fix(harvest-pod): resolved deadline is the pod timeout; deadline provenance from the wrapper Greptile P1 'Honor Proof Deadline': effective_run_timeout_secs no longer clamps a resolved deadline to the host fallback, so an approved 7200 s proof gets 7200 s on a host whose fallback is 5400 s. The fallback (default now the pin ceiling) applies only when no deadline was resolved. Greptile P2 'Identify SIGKILL Source': run_cmd records whether the timeout wrapper ended the run (exit 124, or 137 after the full budget) with an EVAL_DEADLINE_HIT line; hit_deadline keys on that, and a bare exit=137 (external SIGKILL such as OOM) is reported by killed_externally instead of being called a deadline. Co-authored-by: Mathis --- bins/proof-challenge/src/main.rs | 10 +- crates/harvest-pod/src/lib.rs | 165 ++++++++++++++++++++++++++----- 2 files changed, 147 insertions(+), 28 deletions(-) diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 81b6c79ba..a55122b0b 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -59,8 +59,14 @@ struct Cli { /// Operator holdout records (JSON array or map keyed by topic id). Never in git. #[arg(long, env = "PROOF_HOLDOUT_FILE")] holdout_file: Option, - /// Seconds the eval image gets to score one artifact on the pod. - #[arg(long, env = "PROOF_EVAL_TIMEOUT_SECS", default_value_t = 5400)] + /// Fallback seconds the eval image gets when no executor deadline was + /// resolved. A resolved `max_proof_deadline_s` is the pod timeout and is + /// never clamped by this value; the default equals the pin ceiling. + #[arg( + long, + env = "PROOF_EVAL_TIMEOUT_SECS", + default_value_t = proof_task::MAX_PROOF_DEADLINE_S_CEILING + )] eval_timeout_secs: u64, /// Sealed baseline measurements (JSON map keyed by topic id). #[arg(long, env = "PROOF_BASELINE_FILE")] diff --git a/crates/harvest-pod/src/lib.rs b/crates/harvest-pod/src/lib.rs index a962e632c..ed1b2be88 100644 --- a/crates/harvest-pod/src/lib.rs +++ b/crates/harvest-pod/src/lib.rs @@ -81,34 +81,57 @@ pub struct RunExtras { pub proxy_tar_path: Option, } -/// Exit codes GNU `timeout` reports when the entrypoint hit the deadline: -/// `124` on the TERM timeout, `137` when `--kill-after` had to SIGKILL. -/// `137` is also what an external SIGKILL (OOM) reports; either way the run -/// did not finish inside its budget and the harvest refuses with the tail. -pub const DEADLINE_EXIT_CODES: [u32; 2] = [124, 137]; - -/// Seconds the entrypoint gets for one run: the configured run timeout, -/// tightened by a per-run proof deadline when one is given. The image is -/// killed at the deadline, never given more. +/// Exit code GNU `timeout` reports when the entrypoint hit the deadline and +/// died on TERM. Unambiguous: only the wrapper produces it. +pub const DEADLINE_EXIT_CODE: u32 = 124; + +/// `128 + SIGKILL`. Ambiguous on its own: the wrapper's `--kill-after` and an +/// external kill (GPU OOM, host pressure) both report it. The run command +/// therefore records provenance itself ([`DEADLINE_MARKER`]) instead of the +/// harvest guessing from the code. +pub const SIGKILL_EXIT_CODE: u32 = 137; + +/// Line the run command prints when the **wrapper** ended the run: exit +/// `124`, or exit `137` after at least the full timeout had elapsed (the +/// `--kill-after` path). A `137` before the deadline is an external kill and +/// does not print it. +pub const DEADLINE_MARKER: &str = "EVAL_DEADLINE_HIT"; + +/// Seconds the entrypoint gets for one run. +/// +/// A resolved proof deadline **is** the pod timeout: the run is killed at +/// the deadline and never clamped below it by the host's fallback +/// `run_timeout_secs` (which only applies when no deadline was resolved). +/// An approved 7200 s proof therefore gets 7200 s even on a host whose +/// fallback is shorter. #[must_use] pub fn effective_run_timeout_secs(run_timeout_secs: u64, deadline_secs: Option) -> u64 { - deadline_secs - .filter(|d| *d > 0) - .map_or(run_timeout_secs, |d| d.min(run_timeout_secs)) + deadline_secs.filter(|d| *d > 0).unwrap_or(run_timeout_secs) } -/// Whether pod stdout reports an `exit=` line with a deadline exit code -/// (the run was cut at the `timeout`, not a scoring failure). -#[must_use] -pub fn hit_deadline(stdout: &str) -> bool { - stdout.lines().any(|l| { +fn exit_code(stdout: &str) -> Option { + stdout.lines().find_map(|l| { l.trim() .strip_prefix("exit=") .and_then(|rc| rc.trim().parse::().ok()) - .is_some_and(|rc| DEADLINE_EXIT_CODES.contains(&rc)) }) } +/// Whether the run was cut by the deadline wrapper: the [`DEADLINE_MARKER`] +/// line, or the unambiguous `exit=124`. +#[must_use] +pub fn hit_deadline(stdout: &str) -> bool { + stdout.lines().any(|l| l.trim_end() == DEADLINE_MARKER) + || exit_code(stdout) == Some(DEADLINE_EXIT_CODE) +} + +/// Whether the entrypoint was SIGKILLed by something other than the deadline +/// wrapper (e.g. GPU out-of-memory): `exit=137` without the deadline marker. +#[must_use] +pub fn killed_externally(stdout: &str) -> bool { + exit_code(stdout) == Some(SIGKILL_EXIT_CODE) && !hit_deadline(stdout) +} + /// Seconds allowed to stream `bytes` over SSH. Floor is the short-op /// budget; large archives get one extra second per MiB (capped at 1h). #[must_use] @@ -180,6 +203,11 @@ impl PodProgram { /// A non-interactive SSH session often has a login-less PATH that misses /// the image's install location; exit 127 with no `OK` marker is that miss, /// not a scoring failure. + /// + /// Deadline provenance is recorded on the pod: [`DEADLINE_MARKER`] is + /// printed when the wrapper ended the run (`exit=124`, or `exit=137` + /// once the full timeout had elapsed — the `--kill-after` path). A + /// `137` earlier than that is an external SIGKILL and stays unmarked. #[must_use] pub fn run_cmd(&self, timeout_secs: u64) -> String { let Self { @@ -204,11 +232,14 @@ impl PodProgram { bin={bin}; \ if [ -x \"/usr/bin/$bin\" ]; then resolved=\"/usr/bin/$bin\"; \ else resolved=$(command -v \"$bin\" 2>/dev/null || echo \"/usr/bin/$bin\"); fi; \ + t0=$(date +%s); \ timeout --kill-after=60 {timeout_secs} \"$resolved\"{args} \ --request {REQUEST_FILE} --out metrics.json > run.log 2>&1; \ - rc=$?; \ + rc=$?; t1=$(date +%s); \ if [ -f metrics.json ]; then printf '{metrics_marker}'; cat metrics.json; printf '\\n'; fi; \ if [ $rc -eq 0 ]; then echo {ok_marker}; else echo \"exit=$rc\"; fi; \ + if [ $rc -eq {DEADLINE_EXIT_CODE} ] || {{ [ $rc -eq {SIGKILL_EXIT_CODE} ] && \ + [ $((t1 - t0)) -ge {timeout_secs} ]; }}; then echo {DEADLINE_MARKER}; fi; \ tail -c 8192 run.log 2>/dev/null || true" ) } @@ -734,28 +765,110 @@ mod tests { assert!(err.contains("refusing to stage the holdout"), "{err}"); } + /// The resolved deadline is the pod timeout. A host whose fallback run + /// timeout is 5400 s must still give an approved 7200 s proof its full + /// 7200 s; the fallback only applies when no deadline was resolved. #[test] - fn a_staged_deadline_only_tightens_the_run_timeout() { + fn a_resolved_deadline_is_honored_even_above_the_fallback_run_timeout() { assert_eq!(effective_run_timeout_secs(5400, None), 5400); assert_eq!(effective_run_timeout_secs(5400, Some(0)), 5400); assert_eq!(effective_run_timeout_secs(5400, Some(1800)), 1800); assert_eq!( - effective_run_timeout_secs(5400, Some(9000)), - 5400, - "a deadline never extends the configured run timeout" + effective_run_timeout_secs(5400, Some(7200)), + 7200, + "a 7200s approved deadline must never be clamped to a 5400s host default" ); + let cmd = PROGRAM.run_cmd(effective_run_timeout_secs(5400, Some(7200))); + assert!(cmd.contains("timeout --kill-after=60 7200"), "{cmd}"); let cmd = PROGRAM.run_cmd(effective_run_timeout_secs(5400, Some(1800))); assert!(cmd.contains("timeout --kill-after=60 1800"), "{cmd}"); } #[test] - fn deadline_exit_codes_are_recognised_in_stdout() { + fn deadline_provenance_comes_from_the_wrapper_not_the_exit_code_alone() { assert!(hit_deadline("boot\nexit=124\ntail of run.log\n")); - assert!(hit_deadline("exit=137\n")); + assert!(hit_deadline("exit=137\nEVAL_DEADLINE_HIT\ntail\n")); + assert!( + !hit_deadline("exit=137\ntail\n"), + "a bare 137 is an external SIGKILL (OOM), not the deadline" + ); + assert!(killed_externally("exit=137\nCUDA out of memory\n")); + assert!(!killed_externally("exit=137\nEVAL_DEADLINE_HIT\n")); + assert!(!killed_externally("exit=2\n")); assert!(!hit_deadline("exit=2\n")); assert!(!hit_deadline("DEMO_METRICS={\"a\":1}\nDEMO_EVAL_OK\n")); assert!(!hit_deadline("the log said exit=124 once")); - assert_eq!(DEADLINE_EXIT_CODES, [124, 137]); + assert_eq!(DEADLINE_EXIT_CODE, 124); + assert_eq!(SIGKILL_EXIT_CODE, 137); + let cmd = PROGRAM.run_cmd(600); + assert!(cmd.contains("t0=$(date +%s)"), "{cmd}"); + assert!(cmd.contains("$((t1 - t0)) -ge 600"), "{cmd}"); + assert!(cmd.contains("echo EVAL_DEADLINE_HIT"), "{cmd}"); + } + + /// Run the real command tail in a shell: the wrapper's TERM timeout + /// prints the marker, an external SIGKILL inside the budget does not. + #[test] + fn run_cmd_marks_only_wrapper_caused_kills() { + fn shell(timeout_secs: u64, entrypoint: &str) -> String { + let dir = std::env::temp_dir().join(format!( + "harvest-pod-deadline-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&dir).expect("dir"); + let bin = dir.join("demo-eval"); + std::fs::write(&bin, format!("#!/bin/sh\n{entrypoint}\n")).expect("script"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + } + let program = PodProgram { + workdir: "/tmp/harvest-pod-deadline-workdir", + entrypoint: "demo-eval score", + metrics_marker: "DEMO_METRICS=", + ok_marker: "DEMO_EVAL_OK", + score_binary: "/usr/bin/demo-eval", + }; + let cmd = program + .run_cmd(timeout_secs) + .replace( + "cd /tmp/harvest-pod-deadline-workdir || exit 1;", + &format!("cd {} || exit 1;", dir.display()), + ) + .replace("bin=demo-eval;", &format!("bin={};", bin.display())); + let out = std::process::Command::new("sh") + .arg("-c") + .arg(&cmd) + .output() + .expect("sh"); + let _ = std::fs::remove_dir_all(&dir); + String::from_utf8_lossy(&out.stdout).into_owned() + } + if std::process::Command::new("timeout") + .arg("--version") + .output() + .is_err() + { + eprintln!("skip: GNU timeout not available"); + return; + } + let term = shell(1, "sleep 5"); + assert!(term.contains("exit=124"), "{term}"); + assert!(hit_deadline(&term), "{term}"); + assert!(!killed_externally(&term), "{term}"); + + let oom = shell(30, "kill -9 $$"); + assert!(oom.contains("exit=137"), "{oom}"); + assert!( + !hit_deadline(&oom), + "an early SIGKILL is not the deadline: {oom}" + ); + assert!(killed_externally(&oom), "{oom}"); } #[test] From be2c39f1fbada2bc150eee4ebf989f2f17b5e5f6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:57:13 +0000 Subject: [PATCH 13/16] fix(xtask): proof-executor-offer refuses any tracked or trackable workspace path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 'Protect Tracked Files': the output path is resolved against the workspace root; inside it, git ls-files (tracked) and git check-ignore (would be committed) decide, so README.md, Cargo.toml, or a new offer.json at the root are refused while gitignored deploy/secrets/** and paths outside the repo are allowed. Git unavailable → refuse. Tests cover tracked, trackable, ignored, and outside paths and prove run() leaves README.md untouched. Co-authored-by: Mathis --- xtask/src/main.rs | 1 + xtask/src/proof_executor_offer.rs | 181 +++++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 13 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ac95ebc3d..2ae2f8ef3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -282,6 +282,7 @@ fn dispatch(command: Command, root: &Path) -> Result<(), String> { closed, out, } => proof_executor_offer::run(&proof_executor_offer::ExecutorOfferArgs { + repo_root: root.to_path_buf(), pin: if pin.is_absolute() { pin } else { diff --git a/xtask/src/proof_executor_offer.rs b/xtask/src/proof_executor_offer.rs index 059974e42..ddf75f0dd 100644 --- a/xtask/src/proof_executor_offer.rs +++ b/xtask/src/proof_executor_offer.rs @@ -9,6 +9,7 @@ use std::fs; use std::path::{Path, PathBuf}; +use std::process::Command; use proof_executor::{EvalExecutorOffer, OfferStatus}; use proof_task::ProofPin; @@ -16,6 +17,9 @@ use proof_task::ProofPin; /// Arguments for the executor offer ceremony. #[derive(Debug)] pub struct ExecutorOfferArgs { + /// Workspace root; `--out` is refused when it would touch a tracked (or + /// trackable) path under it. + pub repo_root: PathBuf, /// Pin the offer must validate against (`config/proof-pin.toml`). pub pin: PathBuf, /// Immutable slug for this executor. @@ -50,7 +54,7 @@ pub fn run(args: &ExecutorOfferArgs) -> Result<(), String> { let json = serde_json::to_string_pretty(&offer).map_err(|e| e.to_string())?; match &args.out { Some(out) => { - refuse_repo_path(out)?; + refuse_repo_path(out, &args.repo_root)?; if let Some(parent) = out.parent() { fs::create_dir_all(parent) .map_err(|e| format!("mkdir {}: {e}", parent.display()))?; @@ -109,11 +113,77 @@ fn build(pin: &ProofPin, args: &ExecutorOfferArgs) -> Result Result<(), String> { - let text = out.to_string_lossy(); - if text.contains("/config/") || text.starts_with("config/") || text.contains("/docs/") { +/// Absolute, symlink-resolved form of `out`, resolving through the deepest +/// existing ancestor so a not-yet-created file still normalizes. +fn resolve_out(out: &Path) -> Result { + let absolute = if out.is_absolute() { + out.to_path_buf() + } else { + std::env::current_dir() + .map_err(|e| format!("cwd: {e}"))? + .join(out) + }; + let mut existing = absolute.clone(); + let mut tail = Vec::new(); + while !existing.exists() { + let Some(name) = existing.file_name() else { + break; + }; + tail.push(name.to_owned()); + existing = existing + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| format!("no existing ancestor for {}", absolute.display()))?; + } + let mut resolved = existing + .canonicalize() + .map_err(|e| format!("resolve {}: {e}", existing.display()))?; + for name in tail.iter().rev() { + resolved.push(name); + } + Ok(resolved) +} + +/// `git ` in `repo_root`; `Ok(true)` on exit 0, `Ok(false)` on a +/// non-zero exit, `Err` when git itself cannot be run. +fn git_ok(repo_root: &Path, args: &[&str]) -> Result { + Command::new("git") + .arg("-C") + .arg(repo_root) + .args(args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .map_err(|e| format!("git {}: {e}", args.join(" "))) +} + +/// A live offer is operator state: it must never land on a path git would +/// see. Outside the workspace anything goes. Inside it, the path must be +/// neither tracked (`git ls-files`) nor trackable (not matched by +/// `.gitignore`, e.g. `deploy/secrets/**` is fine, `README.md` or a new +/// `offer.json` at the root is not). When git cannot answer, refuse. +fn refuse_repo_path(out: &Path, repo_root: &Path) -> Result<(), String> { + let resolved = resolve_out(out)?; + let root = repo_root + .canonicalize() + .map_err(|e| format!("resolve workspace root {}: {e}", repo_root.display()))?; + let Ok(rel) = resolved.strip_prefix(&root) else { + return Ok(()); + }; + let rel = rel.to_string_lossy().into_owned(); + if rel.is_empty() { + return Err("refusing to write a live executor offer over the workspace root".into()); + } + if git_ok(&root, &["ls-files", "--error-unmatch", "--", &rel])? { + return Err(format!( + "refusing to write a live executor offer over tracked path {rel}" + )); + } + if !git_ok(&root, &["check-ignore", "-q", "--", &rel])? { return Err(format!( - "refusing to write a live executor offer under a tracked path: {text}" + "refusing to write a live executor offer under the workspace at {rel}: the path is \ + not gitignored and would be committed (use deploy/secrets/ or a path outside the repo)" )); } Ok(()) @@ -127,8 +197,16 @@ mod tests { Path::new(env!("CARGO_MANIFEST_DIR")).join("../config/proof-pin.toml") } + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .canonicalize() + .expect("workspace root") + } + fn args(shape: &str, deadline: u64) -> ExecutorOfferArgs { ExecutorOfferArgs { + repo_root: repo_root(), pin: committed_pin(), offer_id: "lium-1x-v0".into(), template_id: None, @@ -169,23 +247,100 @@ mod tests { assert!(err.contains("machine_shape"), "{err}"); let err = build(&p, &args("1x", 7_201)).expect_err("over ceiling"); assert!(err.contains("max_proof_deadline_s"), "{err}"); + let mut unbound = args("1x", 600); + unbound.template_id = Some("prism-recipe-v10".into()); + let err = build(&p, &unbound).expect_err("not digest-bound"); + assert!(err.contains("pinned eval image digest prefix"), "{err}"); + let digest_hex = p.eval_image_digest.trim_start_matches("sha256:"); let mut off_list = args("1x", 600); - off_list.template_id = Some("prism-recipe-v10".into()); + off_list.template_id = Some(format!("other-{}", &digest_hex[..12])); let err = build(&p, &off_list).expect_err("allowlist"); assert!(err.contains("allowed_lium_template_prefixes"), "{err}"); + let mut raw = args("1x", 600); + raw.template_id = Some("f2f5e84c-3b09-4090-be83-1913eabd009e".into()); + let err = build(&p, &raw).expect_err("raw uuid"); + assert!(err.contains("raw Lium template id"), "{err}"); } #[test] - fn closed_offers_and_tracked_paths() { + fn closed_offers_build() { let p = pin(); let mut closed = args("1x", 600); closed.closed = true; assert!(!build(&p, &closed).expect("closed").is_open()); - assert!(refuse_repo_path(Path::new("config/eval_executor_offer.json")).is_err()); - assert!(refuse_repo_path(Path::new("/x/docs/offer.json")).is_err()); - refuse_repo_path(Path::new( - "/root/.base-secrets/proof/eval_executor_offer.json", - )) - .expect("operator path"); + } + + /// Any git-tracked path — not just names under config/ or docs/ — is + /// refused, and so is an untracked path git would pick up. Only + /// gitignored trees (`deploy/secrets/**`) and paths outside the + /// workspace are writable. + #[test] + fn tracked_or_trackable_workspace_paths_are_refused() { + let root = repo_root(); + for tracked in [ + "README.md", + "Cargo.toml", + "config/proof-pin.toml", + "docs/PROOF.md", + "xtask/src/proof_executor_offer.rs", + "deploy/secrets/README.md", + ] { + let err = refuse_repo_path(&root.join(tracked), &root).expect_err(tracked); + assert!(err.contains("tracked path"), "{tracked}: {err}"); + // Relative form (as typed on the command line) is resolved too. + let cwd = std::env::current_dir().expect("cwd"); + if cwd == root { + assert!( + refuse_repo_path(Path::new(tracked), &root).is_err(), + "{tracked}" + ); + } + } + for trackable in [ + "eval_executor_offer.json", + "docs/executor.json", + "config/new-dir/offer.json", + "crates/proof-executor/offer.json", + ] { + let err = refuse_repo_path(&root.join(trackable), &root).expect_err(trackable); + assert!(err.contains("not gitignored"), "{trackable}: {err}"); + } + refuse_repo_path( + &root.join("deploy/secrets/proof/eval_executor_offer.json"), + &root, + ) + .expect("gitignored operator tree inside the workspace"); + refuse_repo_path(&root.join("deploy/env/proof-challenge.env"), &root) + .expect("gitignored env file"); + refuse_repo_path( + Path::new("/root/.base-secrets/proof/eval_executor_offer.json"), + &root, + ) + .expect("outside the workspace"); + assert!(refuse_repo_path(&root, &root).is_err(), "the root itself"); + } + + /// The end-to-end command refuses to touch a tracked file and leaves its + /// bytes intact. + #[test] + fn run_never_overwrites_a_tracked_file() { + let root = repo_root(); + let readme = root.join("README.md"); + let before = fs::read(&readme).expect("README"); + let mut a = args("1x", 600); + a.out = Some(readme.clone()); + let err = run(&a).expect_err("tracked README"); + assert!(err.contains("tracked path"), "{err}"); + assert_eq!( + fs::read(&readme).expect("README"), + before, + "README must be untouched" + ); + + let mut b = args("1x", 600); + b.out = Some(root.join("brand-new-offer.json")); + let err = run(&b).expect_err("would be committed"); + assert!(err.contains("not gitignored"), "{err}"); + assert!(!root.join("brand-new-offer.json").exists()); } } From d7ae7a009890627202771cdba5584998c2fd03fa Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:57:38 +0000 Subject: [PATCH 14/16] fix(proof-executor): bind executor templates to the pinned digest; refuse raw Lium UUIDs Greptile P1 'Bind UUID Templates': an empty allowlist no longer admits a raw Lium template UUID, which the provider would rent verbatim before the digest-bound resolver could check its image. lium_template_id must now be a digest-scoped template name that carries the pinned digest's 12-hex prefix (the resolver reuses a listed template only when its image is eval_image@digest, else creates one bound to it); a UUID is RawTemplateId under any allowlist, and a pin without a digest binds no executor (UnpinnedDigest). The former test expecting a UUID under an empty allowlist now expects Err. Co-authored-by: Mathis --- config/proof-pin.toml | 10 ++- crates/proof-executor/src/lib.rs | 140 +++++++++++++++++++++++-------- 2 files changed, 113 insertions(+), 37 deletions(-) diff --git a/config/proof-pin.toml b/config/proof-pin.toml index dcd0f9b2d..5e0323c0c 100644 --- a/config/proof-pin.toml +++ b/config/proof-pin.toml @@ -45,10 +45,12 @@ # `max_proof_deadline_s_ceiling`, never longer; a topic may tighten it again # (`eval_executor.max_proof_deadline_s`) and may pin the live offer's # `config_commitment` (`eval_executor.require_offer_commitment`). No per-topic -# machine id. `allowed_lium_template_prefixes` is optional; when set, the -# offer's `lium_template_id` must carry one of the prefixes (the digest-scoped -# harvest template is `proof-eval-<12 hex of the digest>`). Missing / closed / -# shape ≠ 1x → can_score=false → 503. +# machine id. The offer's `lium_template_id` is always the digest-scoped +# template name (`proof-eval-<12 hex of the digest>`, resolved bound to +# eval_image@digest); a raw Lium template UUID is refused under any allowlist. +# `allowed_lium_template_prefixes` is optional; when set, the name must also +# carry one of the prefixes. Missing / closed / shape ≠ 1x → can_score=false +# → 503. # # Holdout size / stratum size are the measurement harness (5 scored splits × 24), # not the problem list. diff --git a/crates/proof-executor/src/lib.rs b/crates/proof-executor/src/lib.rs index c3a80c208..64e7dd7f8 100644 --- a/crates/proof-executor/src/lib.rs +++ b/crates/proof-executor/src/lib.rs @@ -54,10 +54,13 @@ pub const MAX_TEMPLATE_ID_LEN: usize = 128; pub struct EvalExecutorOffer { /// Immutable slug identifying this executor. pub offer_id: String, - /// Lium template the harvest rents. Either the digest-scoped template - /// name (`proof-eval-<12 hex of the digest>`, resolved or created bound - /// to `repo@digest`) or a raw Lium template UUID. The pin's - /// `allowed_lium_template_prefixes` decides which forms are legal. + /// Digest-scoped Lium template **name** the harvest rents + /// (`proof-eval-<12 hex of the pinned digest>` or a longer name that + /// carries that prefix). The harvest resolves it through the digest-bound + /// template resolver, which only reuses a listed template whose image is + /// `eval_image@digest` and otherwise creates one bound to it. A raw Lium + /// template UUID is **refused**: it would be rented verbatim with no way + /// to bind it to the pinned image. pub lium_template_id: String, /// Machine class. Must equal the pin `gpu_class` (`1x`). pub machine_shape: String, @@ -89,6 +92,16 @@ pub enum ExecutorOfferError { /// `lium_template_id` is outside the pin allowlist. #[error("lium_template_id {0:?} is not under the pin allowed_lium_template_prefixes")] TemplateNotAllowed(String), + /// `lium_template_id` is a raw Lium template UUID, which cannot be bound + /// to the pinned image; name the digest-scoped template instead. + #[error( + "lium_template_id {0:?} is a raw Lium template id; use the digest-scoped template name \ + (repo@digest bound) so the rent cannot escape the pinned image" + )] + RawTemplateId(String), + /// The pin has no eval image digest, so no template can be bound to it. + #[error("pin has no eval_image_digest; nothing to bind an executor template to")] + UnpinnedDigest, /// A digest-scoped template name does not name the pinned digest. #[error("lium_template_id {0:?} does not carry the pinned eval image digest prefix {1}")] TemplateDigestMismatch(String, String), @@ -162,8 +175,9 @@ pub fn shape_gpu_count(shape: &str) -> Option { (n >= 1).then_some(n) } -/// `8-4-4-4-12` lowercase/uppercase hex: a raw Lium template id rather than a -/// digest-scoped template name. +/// `8-4-4-4-12` hex: a raw Lium template id rather than a digest-scoped +/// template name. Such ids are refused everywhere an executor template is +/// accepted — nothing can verify what image they run. pub fn is_lium_template_uuid(id: &str) -> bool { let t = id.trim(); t.len() == 36 @@ -173,27 +187,43 @@ pub fn is_lium_template_uuid(id: &str) -> bool { }) } -/// Template identifier shape, pin allowlist, and (for a digest-scoped name) -/// the pinned digest prefix. +/// The 12-hex digest prefix every executor template name must carry. +/// +/// # Errors +/// +/// [`ExecutorOfferError::UnpinnedDigest`] when the pin cannot rent (no +/// `sha256:` digest): there is no image to bind a template to. +pub fn pinned_digest_prefix(pin: &ProofPin) -> Result<&str, ExecutorOfferError> { + if !pin.can_rent() { + return Err(ExecutorOfferError::UnpinnedDigest); + } + let hex = pin.eval_image_digest.trim().trim_start_matches("sha256:"); + hex.get(..12).ok_or(ExecutorOfferError::UnpinnedDigest) +} + +/// Fail-closed template check: printable shape, **not** a raw Lium UUID, +/// carries the pinned digest prefix (so the digest-bound resolver can only +/// reuse or create a template whose image is `eval_image@digest`), and +/// inside the pin allowlist when one is set. pub fn check_template_id(pin: &ProofPin, template_id: &str) -> Result<(), ExecutorOfferError> { let id = template_id.trim(); if id.is_empty() || id.len() > MAX_TEMPLATE_ID_LEN || !id.bytes().all(|b| b.is_ascii_graphic()) { return Err(ExecutorOfferError::BadTemplateId); } + if is_lium_template_uuid(id) { + return Err(ExecutorOfferError::RawTemplateId(id.to_owned())); + } + let prefix = pinned_digest_prefix(pin)?; + if !id.contains(prefix) { + return Err(ExecutorOfferError::TemplateDigestMismatch( + id.to_owned(), + prefix.to_owned(), + )); + } if !pin.allows_template(id) { return Err(ExecutorOfferError::TemplateNotAllowed(id.to_owned())); } - if !is_lium_template_uuid(id) && pin.can_rent() { - let hex = pin.eval_image_digest.trim().trim_start_matches("sha256:"); - let prefix = hex.get(..12).unwrap_or(hex); - if !id.contains(prefix) { - return Err(ExecutorOfferError::TemplateDigestMismatch( - id.to_owned(), - prefix.to_owned(), - )); - } - } Ok(()) } @@ -507,14 +537,18 @@ mod tests { #[test] fn template_id_obeys_the_pin_allowlist_and_the_pinned_digest() { let p = pin(); - assert!(matches!( - offer_for("prism-recipe-v10", 600, &p).validate(&p), - Err(ExecutorOfferError::TemplateNotAllowed(_)) - )); assert!(matches!( offer_for("proof-eval-000000000000", 600, &p).validate(&p), Err(ExecutorOfferError::TemplateDigestMismatch(..)) )); + // Carries the digest prefix but not an allowlisted prefix. + assert!(matches!( + offer_for("other-78b614a1f51c", 600, &p).validate(&p), + Err(ExecutorOfferError::TemplateNotAllowed(_)) + )); + offer_for("proof-eval-78b614a1f51c-b200", 600, &p) + .validate(&p) + .expect("a longer digest-scoped name is fine"); for bad in [ "", "has space", @@ -529,29 +563,69 @@ mod tests { "{bad:?}" ); } - // A raw Lium UUID is legal only when the pin allowlist admits it. + let mut open = p.clone(); + open.allowed_lium_template_prefixes.clear(); + offer_for("any-name-78b614a1f51c", 600, &open) + .validate(&open) + .expect("empty allowlist still requires the digest-scoped name"); + } + + /// A raw Lium template UUID is rented verbatim by the provider, so the + /// digest-bound resolver never sees it: refused under every allowlist, + /// including an empty one. + #[test] + fn raw_lium_uuid_is_refused_under_any_allowlist() { let uuid = "f2f5e84c-3b09-4090-be83-1913eabd009e"; assert!(is_lium_template_uuid(uuid)); + assert!(is_lium_template_uuid(&uuid.to_ascii_uppercase())); assert!(!is_lium_template_uuid("proof-eval-78b614a1f51c")); assert!(!is_lium_template_uuid( "f2f5e84c-3b09-4090-be83-1913eabd009" )); + let p = pin(); assert!(matches!( offer_for(uuid, 600, &p).validate(&p), - Err(ExecutorOfferError::TemplateNotAllowed(_)) + Err(ExecutorOfferError::RawTemplateId(_)) )); let mut open = p.clone(); open.allowed_lium_template_prefixes.clear(); - offer_for(uuid, 600, &open) - .validate(&open) - .expect("uuid under an empty allowlist"); - // Unpinned digest (pre-launch): the name cannot be checked against - // a digest, and the host cannot rent anyway. - let mut unpinned = p.clone(); + assert!( + matches!( + offer_for(uuid, 600, &open).validate(&open), + Err(ExecutorOfferError::RawTemplateId(_)) + ), + "an empty allowlist must not admit an unbindable template" + ); + let mut uuid_prefix = p.clone(); + uuid_prefix.allowed_lium_template_prefixes = vec!["f2f5e84c-".into()]; + assert!(matches!( + offer_for(uuid, 600, &uuid_prefix).validate(&uuid_prefix), + Err(ExecutorOfferError::RawTemplateId(_)) + )); + assert!(matches!( + check_template_id(&open, uuid), + Err(ExecutorOfferError::RawTemplateId(_)) + )); + } + + /// With no pinned digest there is no image to bind a template to, so no + /// executor offer validates (the host cannot rent anyway). + #[test] + fn unpinned_digest_binds_no_executor() { + let mut unpinned = pin(); unpinned.eval_image_digest.clear(); - offer_for("proof-eval-deadbeef0000", 600, &unpinned) - .validate(&unpinned) - .expect("pre-launch pin skips the digest-name check"); + assert!(matches!( + offer_for("proof-eval-deadbeef0000", 600, &unpinned).validate(&unpinned), + Err(ExecutorOfferError::UnpinnedDigest) + )); + assert!(matches!( + pinned_digest_prefix(&unpinned), + Err(ExecutorOfferError::UnpinnedDigest) + )); + assert_eq!( + pinned_digest_prefix(&pin()).expect("pinned"), + "78b614a1f51c" + ); } #[test] From f02cfc9f1ae65a4cf7df72fef54f4926f48105b4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:57:53 +0000 Subject: [PATCH 15/16] fix(proof): refuse config-changing overrides on commitment-pinned topics; commit the resolved plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 'Commit Effective Overrides': a PROOF_HARVEST_* override that changes the template or deadline is refused (OverrideBreaksCommitment) when the topic pins require_offer_commitment — the topic approved the offer's configuration, not the operator's. For every run the plan now carries both the offer's commitment and the commitment of the configuration that actually runs (template, 1x, effective deadline, digest); the harvest request stamps both (executor_offer_commitment / executor_commitment) and the scored row's executor_commitment is the executed one, threaded back through LiveScorer::plan() -> EvalOutcome.executor. Also in this change set: LiveScorer::score takes the resolved ExecutorPlan (single resolution, harvest-side overrides), the harvest always resolves the template by digest-bound name (template_id stays None), and a bare exit=137 is reported as an external SIGKILL rather than the proof deadline. Co-authored-by: Mathis --- crates/proof-eval/src/lib.rs | 54 ++- crates/proof-executor/src/lib.rs | 7 + crates/proof-executor/src/plan.rs | 162 ++++++- crates/proof-harvest/src/lib.rs | 782 +++++++++++++++++++----------- crates/proof-http/src/lib.rs | 13 +- crates/proof-store/src/lib.rs | 5 +- 6 files changed, 712 insertions(+), 311 deletions(-) diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index bd542d972..51a123fd4 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -25,7 +25,10 @@ use std::collections::{BTreeMap, BTreeSet}; use async_trait::async_trait; use prism_lium_types::{EvalReceipt, NoScoreGate}; -use proof_executor::{require_open_executor, EvalExecutorOffer, ExecutorOfferError}; +use proof_executor::{ + executor_plan, require_open_executor, EvalExecutorOffer, ExecutorOfferError, ExecutorPlan, + HarvestOverrides, +}; use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind, SealedBaseline}; use proof_store::ArtifactManifest; use proof_task::{ @@ -258,17 +261,32 @@ impl ProofEvalDocument { /// Handle to the digest-pinned eval image's harvest. #[async_trait] pub trait LiveScorer: Send + Sync { + /// Resolve what one rent is allowed to do for `topic` on `executor`: + /// template, exact width, deadline, and the commitment of that resolved + /// configuration. The harvest applies its `PROOF_HARVEST_*` overrides + /// here; the default applies none. Called before [`Self::score`] so the + /// caller can persist the plan the run was actually held to. + fn plan( + &self, + pin: &ProofPin, + topic: &TopicDocument, + executor: &EvalExecutorOffer, + ) -> Result { + executor_plan(pin, Some(executor), topic, &HarvestOverrides::default()) + .map_err(map_executor_err) + } + /// Score one artifact on one topic's verified holdout. /// - /// `offer` is the RLM judge the image calls; `executor` is the open - /// `1x` machine class the image is rented on. Both are host state. + /// `offer` is the RLM judge the image calls; `plan` is the resolved `1x` + /// rent ([`Self::plan`]) the image is run under. Both are host state. #[allow(clippy::too_many_arguments)] async fn score( &self, pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, - executor: &EvalExecutorOffer, + plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, holdout: &[HoldoutRecord], @@ -481,6 +499,9 @@ pub struct EvalOutcome { pub receipt: EvalReceipt, /// Backend that produced the scores. pub backend: EvalBackend, + /// Executor plan the live run was held to (template, `1x`, deadline, + /// commitment of that resolved configuration). `None` on sim. + pub executor: Option, } /// Declared training metadata plus the holdout fingerprints inside it. @@ -732,6 +753,7 @@ pub async fn eval_after_freeze( OfferError::OriginMismatch.to_string(), )); } + let mut plan = None; let doc = match backend { EvalBackend::Sim => { if let Some(sealed) = sealed { @@ -744,18 +766,23 @@ pub async fn eval_after_freeze( EvalBackend::Lium => { let scorer = live.ok_or(EvalError::LiveHarvestUnavailable)?; let executor = executor.ok_or(EvalError::ExecutorOfferMissing)?; - scorer + // Resolve the rent before anything runs so the row records the + // configuration the run was actually held to. + let resolved = scorer.plan(pin, topic, executor)?; + let doc = scorer .score( pin, topic, offer, - executor, + &resolved, frozen_digest, artifact_digest, holdout, claim, ) - .await? + .await?; + plan = Some(resolved); + doc } }; doc.verify(pin, topic, frozen_digest, artifact_digest)?; @@ -783,6 +810,7 @@ pub async fn eval_after_freeze( harness: doc.harness, receipt, backend, + executor: plan, }) } @@ -884,7 +912,7 @@ mod tests { pin: &ProofPin, topic: &TopicDocument, _offer: &InferenceOffer, - _executor: &EvalExecutorOffer, + _plan: &ExecutorPlan, frozen: &str, artifact: &str, _holdout: &[HoldoutRecord], @@ -979,6 +1007,15 @@ mod tests { assert!(out.agent.reproduced); assert_eq!(out.agent.topic_id, t.id); assert_eq!(out.receipt.provider, "lium"); + let plan = out + .executor + .expect("live outcome carries the resolved plan"); + assert_eq!(plan.offer_id, "lium-1x-v0"); + assert_eq!(plan.topic_id, t.id); + assert_eq!(plan.gpu_count, 1); + assert_eq!(plan.deadline_s, 3_600); + assert_eq!(plan.offer_commitment, executor(&p).config_commitment); + assert_eq!(plan.config_commitment, plan.offer_commitment); } #[test] @@ -1343,6 +1380,7 @@ mod tests { .expect("sim"); assert_eq!(out.backend, EvalBackend::Sim); assert_eq!(out.receipt.provider, "sim"); + assert!(out.executor.is_none(), "sim rents nothing"); assert_eq!(out.agent.rationale, "sim stub win"); assert!(out.harness.holdout_nll <= sealed.holdout_nll + t.metric.quality_floor_nll); assert!( diff --git a/crates/proof-executor/src/lib.rs b/crates/proof-executor/src/lib.rs index 64e7dd7f8..cf8be3b00 100644 --- a/crates/proof-executor/src/lib.rs +++ b/crates/proof-executor/src/lib.rs @@ -135,6 +135,13 @@ pub enum ExecutorOfferError { /// silently fall back to the offer. #[error("{0} is set but not a usable harvest override")] BadOverride(&'static str), + /// The topic pinned the offer commitment and an operator override would + /// run a different template or deadline than the one it approved. + #[error( + "PROOF_HARVEST_* override changes the executor template or deadline, but the topic pins \ + the offer config_commitment; refuse scoring (drop the override or republish the offer)" + )] + OverrideBreaksCommitment, /// The rent would not be exactly the pinned width. #[error("abort: executor would rent {0}x GPUs; the Proof executor is exactly {1}x")] GpuCount(u32, u32), diff --git a/crates/proof-executor/src/plan.rs b/crates/proof-executor/src/plan.rs index 8c376d4f8..63f3c31e0 100644 --- a/crates/proof-executor/src/plan.rs +++ b/crates/proof-executor/src/plan.rs @@ -10,7 +10,7 @@ use proof_task::{ProofPin, TopicDocument, EVAL_EXECUTOR_GPU_COUNT}; use crate::{ - check_template_id, is_lium_template_uuid, require_open_executor, shape_gpu_count, + check_template_id, executor_config_commitment, require_open_executor, shape_gpu_count, EvalExecutorOffer, ExecutorOfferError, }; @@ -104,17 +104,23 @@ pub struct ExecutorPlan { /// per-topic judge VM the pod reports to) can key on the plan without /// changing its shape; this crate does not interpret the id. pub topic_id: String, - /// Lium template id or digest-scoped template name to rent. + /// Digest-scoped template name to rent (resolved / created bound to the + /// pinned `eval_image@digest`; never a raw Lium UUID). pub template_id: String, - /// `true` when `template_id` is a raw Lium UUID (rent verbatim) rather - /// than a template name to resolve / create bound to the pinned digest. - pub template_is_uuid: bool, /// Exact GPUs to rent. Always [`EVAL_EXECUTOR_GPU_COUNT`]. pub gpu_count: u32, /// Proof deadline the pod-side `timeout` and the harvest wait enforce. pub deadline_s: u64, - /// Offer `config_commitment` stamped onto the run request. + /// The live offer's `config_commitment` (what a topic may pin). + pub offer_commitment: String, + /// Commitment of the configuration that actually runs: `template_id`, + /// shape, `deadline_s`, digest. Equals `offer_commitment` only when no + /// override and no topic tighten changed the offer's knobs. This is what + /// the run request and the scored row carry as executor provenance. pub config_commitment: String, + /// `true` when an operator `PROOF_HARVEST_*` override changed the + /// template or the deadline away from the offer. + pub overridden: bool, } /// Resolve the plan for scoring `topic` on `offer` under `pin`. @@ -122,8 +128,9 @@ pub struct ExecutorPlan { /// # Errors /// /// Any [`ExecutorOfferError`]: the offer must be open and valid, the -/// template legal, the width exactly the pin, and the deadline within the -/// ceiling after the topic tighten and the operator override. +/// template legal, the width exactly the pin, the deadline within the +/// ceiling after the topic tighten and the operator override, and — when +/// the topic pins the offer commitment — no override may change what runs. pub fn executor_plan( pin: &ProofPin, offer: Option<&EvalExecutorOffer>, @@ -155,6 +162,14 @@ pub fn executor_plan( let base = overrides .deadline_secs .unwrap_or(offer.max_proof_deadline_s); + let overridden = + template_id != offer.lium_template_id.trim() || base != offer.max_proof_deadline_s; + // A topic that pinned the offer commitment approved *that* template and + // deadline. An operator override that changes either would run a + // configuration the topic never approved: refuse, never re-stamp. + if overridden && topic.eval_executor.require_offer_commitment.is_some() { + return Err(ExecutorOfferError::OverrideBreaksCommitment); + } let deadline_s = topic .eval_executor .max_proof_deadline_s @@ -166,14 +181,21 @@ pub fn executor_plan( )); } + let config_commitment = executor_config_commitment( + &template_id, + &offer.machine_shape, + deadline_s, + &offer.eval_image_digest, + ); Ok(ExecutorPlan { offer_id: offer.offer_id.clone(), topic_id: topic.id.clone(), - template_is_uuid: is_lium_template_uuid(&template_id), template_id, gpu_count, deadline_s, - config_commitment: offer.config_commitment.clone(), + offer_commitment: offer.config_commitment.clone(), + config_commitment, + overridden, }) } @@ -206,10 +228,14 @@ mod tests { "topic scope rides on the plan" ); assert_eq!(plan.template_id, "proof-eval-78b614a1f51c"); - assert!(!plan.template_is_uuid); assert_eq!(plan.gpu_count, 1); assert_eq!(plan.deadline_s, 7_200); - assert_eq!(plan.config_commitment, offer().config_commitment); + assert!(!plan.overridden); + assert_eq!(plan.offer_commitment, offer().config_commitment); + assert_eq!( + plan.config_commitment, plan.offer_commitment, + "untouched offer: the executed configuration is the committed one" + ); } #[test] @@ -244,6 +270,21 @@ mod tests { let plan = executor_plan(&pin(), Some(&offer()), &t, &HarvestOverrides::default()).expect("plan"); assert_eq!(plan.deadline_s, 900); + assert!( + !plan.overridden, + "a topic tighten is signed, not an override" + ); + assert_eq!(plan.offer_commitment, offer().config_commitment); + assert_eq!( + plan.config_commitment, + executor_config_commitment( + "proof-eval-78b614a1f51c", + "1x", + 900, + &pin().eval_image_digest + ), + "provenance commits the executed 900s, not the offer's 7200s" + ); t.eval_executor.require_offer_commitment = Some("ab".repeat(32)); assert!(matches!( executor_plan(&pin(), Some(&offer()), &t, &HarvestOverrides::default()), @@ -262,6 +303,21 @@ mod tests { let plan = executor_plan(&p, Some(&offer()), &topic(), &swapped).expect("plan"); assert_eq!(plan.template_id, "proof-eval-78b614a1f51c-hotfix"); assert_eq!(plan.deadline_s, 3_600); + assert!(plan.overridden); + assert_eq!(plan.offer_commitment, offer().config_commitment); + assert_ne!( + plan.config_commitment, plan.offer_commitment, + "an override must not be stamped as the offer's committed config" + ); + assert_eq!( + plan.config_commitment, + executor_config_commitment( + "proof-eval-78b614a1f51c-hotfix", + "1x", + 3_600, + &p.eval_image_digest + ) + ); // The override replaces the offer deadline, so a longer one is legal // up to the ceiling — never past it. @@ -301,23 +357,97 @@ mod tests { .deadline_s, 300 ); - // The template override obeys the allowlist and the pinned digest. - let off_list = HarvestOverrides { + // The template override obeys the pinned digest and the allowlist. + let unbound = HarvestOverrides { template_id: Some("prism-recipe-v10".into()), ..HarvestOverrides::default() }; + assert!(matches!( + executor_plan(&p, Some(&offer()), &topic(), &unbound), + Err(ExecutorOfferError::TemplateDigestMismatch(..)) + )); + let off_list = HarvestOverrides { + template_id: Some("other-78b614a1f51c".into()), + ..HarvestOverrides::default() + }; assert!(matches!( executor_plan(&p, Some(&offer()), &topic(), &off_list), Err(ExecutorOfferError::TemplateNotAllowed(_)) )); + // A raw UUID override is refused like a raw UUID offer would be. let mut open = p.clone(); open.allowed_lium_template_prefixes.clear(); let uuid = HarvestOverrides { template_id: Some("f2f5e84c-3b09-4090-be83-1913eabd009e".into()), ..HarvestOverrides::default() }; - let plan = executor_plan(&open, Some(&offer()), &topic(), &uuid).expect("plan"); - assert!(plan.template_is_uuid); + assert!(matches!( + executor_plan(&open, Some(&offer()), &topic(), &uuid), + Err(ExecutorOfferError::RawTemplateId(_)) + )); + } + + /// A topic that pinned the offer commitment approved that template and + /// deadline; an override that changes either is refused rather than run + /// under the old stamp. A no-op override (same width) is still fine. + #[test] + fn commitment_pinned_topic_refuses_config_changing_overrides() { + let p = pin(); + let mut t = topic(); + t.eval_executor.require_offer_commitment = Some(offer().config_commitment); + for over in [ + HarvestOverrides { + template_id: Some("proof-eval-78b614a1f51c-hotfix".into()), + ..HarvestOverrides::default() + }, + HarvestOverrides { + deadline_secs: Some(3_600), + ..HarvestOverrides::default() + }, + HarvestOverrides { + deadline_secs: Some(7_200 - 1), + ..HarvestOverrides::default() + }, + ] { + assert!( + matches!( + executor_plan(&p, Some(&offer()), &t, &over), + Err(ExecutorOfferError::OverrideBreaksCommitment) + ), + "{over:?} must not run under a pinned commitment" + ); + } + // Same values as the offer, or only the (no-op) width: not a change. + for over in [ + HarvestOverrides { + template_id: Some("proof-eval-78b614a1f51c".into()), + gpu_count: Some(1), + deadline_secs: Some(7_200), + }, + HarvestOverrides { + gpu_count: Some(1), + ..HarvestOverrides::default() + }, + ] { + let plan = executor_plan(&p, Some(&offer()), &t, &over).expect("no-op override"); + assert!(!plan.overridden); + assert_eq!(plan.config_commitment, plan.offer_commitment); + } + // Without the pin the same override is legal and re-committed. + let mut unpinned = topic(); + unpinned.eval_executor.max_proof_deadline_s = Some(900); + let plan = executor_plan( + &p, + Some(&offer()), + &unpinned, + &HarvestOverrides { + template_id: Some("proof-eval-78b614a1f51c-hotfix".into()), + ..HarvestOverrides::default() + }, + ) + .expect("plan"); + assert!(plan.overridden); + assert_ne!(plan.config_commitment, plan.offer_commitment); } #[test] diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 25c43c73c..55764b6ca 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -38,8 +38,8 @@ use std::time::Duration; use async_trait::async_trait; use harvest_pod::{ - harvest_template_name, hit_deadline, truncate_tail, EvalPod, PodProgram, RunExtras, - HOLDOUT_DIR, PROXY_DIR, + hit_deadline, killed_externally, truncate_tail, EvalPod, PodProgram, RunExtras, HOLDOUT_DIR, + PROXY_DIR, }; use prism_lium_types::InstanceSpec; use proof_eval::{ @@ -118,7 +118,11 @@ pub struct HarvestRequest { pub config_commitment: String, /// Live executor offer the pod was rented on (host stamp). pub executor_offer_id: String, - /// Executor `config_commitment` (template, shape, deadline, digest). + /// The offer's published `config_commitment`. + pub executor_offer_commitment: String, + /// Commitment of the executor configuration this run actually got + /// (template, shape, `max_proof_deadline_s`, digest). Equals the offer + /// commitment unless a topic tighten or an operator override changed it. pub executor_commitment: String, /// Proof deadline this run is held to, seconds. pub max_proof_deadline_s: u64, @@ -138,6 +142,72 @@ pub struct HarvestRequest { pub holdout: Vec, } +impl HarvestRequest { + /// Resolve the judge for `topic` against the committed offer and bind the + /// run to the executor plan. Refuses (no rent) when the offer cannot + /// serve the topic, the resolved judge config is incomplete, or a topic + /// tried to redirect the judge origin. + pub fn build( + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + plan: &ExecutorPlan, + frozen_digest: &str, + artifact_digest: &str, + holdout: &[HoldoutRecord], + claim: &str, + ) -> Result { + offer + .serves_topic(pin, topic) + .map_err(|e| EvalError::InferenceOffer(e.to_string()))?; + let resolved = resolve_inference( + pin, + Some(&topic.inference), + secret_backed_base_url().as_deref(), + Some(offer), + ); + if !resolved.ready_to_score() { + return Err(EvalError::InferenceOffer( + proof_task::OfferError::Incomplete.to_string(), + )); + } + if resolved.base_url.trim() != offer.provider.base_url.trim() { + return Err(EvalError::InferenceOffer( + proof_task::OfferError::OriginMismatch.to_string(), + )); + } + Ok(Self { + schema_version: PROOF_METRICS_SCHEMA, + challenge_id: CHALLENGE_ID.to_owned(), + submission_digest: frozen_digest.to_owned(), + artifact_digest: artifact_digest.to_owned(), + topic_id: topic.id.clone(), + family: topic.metric.family.as_str().to_owned(), + inference_offer_id: offer.offer_id.clone(), + provider_kind: resolved.provider.as_str().to_owned(), + base_url: resolved.base_url.clone(), + mode: resolved.mode.as_str().to_owned(), + model_ref: resolved.model.clone(), + max_input_tokens: resolved.max_input_tokens.min(offer.config.max_input_tokens), + max_output_tokens: resolved + .max_output_tokens + .min(offer.config.max_output_tokens), + config_commitment: offer.config_commitment.clone(), + executor_offer_id: plan.offer_id.clone(), + executor_offer_commitment: plan.offer_commitment.clone(), + executor_commitment: plan.config_commitment.clone(), + max_proof_deadline_s: plan.deadline_s, + eval_image_digest: pin.eval_image_digest.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + constraints: topic.constraints, + flops_budget: topic.flops_budget, + wall_budget_s: topic.metric.wall_budget_s, + claim: claim.to_owned(), + holdout: holdout.to_vec(), + }) + } +} + /// Rent limits for one harvest. Width is not a limit: it is the executor /// plan's exact `1x`, and any other rent aborts. #[derive(Debug, Clone)] @@ -420,22 +490,6 @@ impl LiumProofHarvest { self } - fn plan( - &self, - pin: &ProofPin, - topic: &TopicDocument, - executor: &EvalExecutorOffer, - ) -> Result { - let overrides = match &self.overrides { - Some(o) => o.clone(), - None => HarvestOverrides::from_env().map_err(map_executor_err)?, - }; - if !overrides.is_empty() { - tracing::info!(?overrides, "PROOF_HARVEST_* override in effect"); - } - executor_plan(pin, Some(executor), topic, &overrides).map_err(map_executor_err) - } - /// Inject the judge API key staged into `teacher.env` on the pod. /// /// Empty / whitespace is treated as missing (fail-closed on a live run). @@ -498,15 +552,13 @@ impl LiumProofHarvest { )) } - /// Rent spec for one run: the executor plan's template at exactly its - /// width. A raw Lium UUID is rented verbatim; a digest-scoped template - /// name is resolved (or created) bound to `eval_image@digest`. + /// Rent spec for one run: the executor plan's digest-scoped template at + /// exactly its width. `template_id` stays `None` on purpose — the + /// provider would rent a raw id verbatim — so the digest-bound resolver + /// is the only way a template is chosen: it reuses a listed template only + /// when its image is `eval_image@digest`, and otherwise creates one bound + /// to that pin. fn spec(&self, pin: &ProofPin, frozen_digest: &str, plan: &ExecutorPlan) -> InstanceSpec { - let template_name = if plan.template_is_uuid { - harvest_template_name(&pin.eval_image, &pin.eval_image_digest) - } else { - plan.template_id.clone() - }; InstanceSpec { name: format!("proof-{}", &frozen_digest[..12.min(frozen_digest.len())]), max_lifetime_hours: self.limits.max_lifetime_hours, @@ -518,8 +570,8 @@ impl LiumProofHarvest { ssh_public_keys: self.ssh_public_keys.clone(), ssh_key_name: Some(SSH_KEY_NAME.to_owned()), preferred_offer_id: None, - template_id: plan.template_is_uuid.then(|| plan.template_id.clone()), - template_name: Some(template_name), + template_id: None, + template_name: Some(plan.template_id.clone()), exact_gpu_count: true, } } @@ -582,12 +634,31 @@ impl LiumProofHarvest { #[async_trait] impl LiveScorer for LiumProofHarvest { + /// Pin ceilings + open offer + topic tighten + this host's + /// `PROOF_HARVEST_*` overrides, collapsed into one rent. Refuses before + /// anything is staged or rented: no judge env, no proxy tar, no pod. + fn plan( + &self, + pin: &ProofPin, + topic: &TopicDocument, + executor: &EvalExecutorOffer, + ) -> Result { + let overrides = match &self.overrides { + Some(o) => o.clone(), + None => HarvestOverrides::from_env().map_err(map_executor_err)?, + }; + if !overrides.is_empty() { + tracing::info!(?overrides, "PROOF_HARVEST_* override in effect"); + } + executor_plan(pin, Some(executor), topic, &overrides).map_err(map_executor_err) + } + async fn score( &self, pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, - executor: &EvalExecutorOffer, + plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, holdout: &[HoldoutRecord], @@ -600,61 +671,26 @@ impl LiveScorer for LiumProofHarvest { return Err(EvalError::HoldoutSealed); } self.ready()?; - // Executor first: no judge env, no proxy tar, no rent unless the - // open 1x executor, the topic tighten, and any operator override - // collapse into a legal plan. - let plan = self.plan(pin, topic, executor)?; - offer - .serves_topic(pin, topic) - .map_err(|e| EvalError::InferenceOffer(e.to_string()))?; - let resolved = resolve_inference( - pin, - Some(&topic.inference), - secret_backed_base_url().as_deref(), - Some(offer), - ); - if !resolved.ready_to_score() { - return Err(EvalError::InferenceOffer( - proof_task::OfferError::Incomplete.to_string(), - )); - } - if resolved.base_url.trim() != offer.provider.base_url.trim() { - return Err(EvalError::InferenceOffer( - proof_task::OfferError::OriginMismatch.to_string(), - )); + // The plan is what `Self::plan` resolved for this topic; a plan for + // another topic is a caller bug and must not rent. + if plan.topic_id != topic.id { + return Err(EvalError::ExecutorOffer(format!( + "executor plan is for topic {:?}, scoring {:?}", + plan.topic_id, topic.id + ))); } + let request = HarvestRequest::build( + pin, + topic, + offer, + plan, + frozen_digest, + artifact_digest, + holdout, + claim, + )?; let env = judge_teacher_env(self.judge_api_key.as_deref().unwrap_or(""))?; let (extras, _proxy_guard) = self.live_extras(holdout)?; - let max_in = resolved.max_input_tokens.min(offer.config.max_input_tokens); - let max_out = resolved - .max_output_tokens - .min(offer.config.max_output_tokens); - let request = HarvestRequest { - schema_version: PROOF_METRICS_SCHEMA, - challenge_id: CHALLENGE_ID.to_owned(), - submission_digest: frozen_digest.to_owned(), - artifact_digest: artifact_digest.to_owned(), - topic_id: topic.id.clone(), - family: topic.metric.family.as_str().to_owned(), - inference_offer_id: offer.offer_id.clone(), - provider_kind: resolved.provider.as_str().to_owned(), - base_url: resolved.base_url.clone(), - mode: resolved.mode.as_str().to_owned(), - model_ref: resolved.model.clone(), - max_input_tokens: max_in, - max_output_tokens: max_out, - config_commitment: offer.config_commitment.clone(), - executor_offer_id: plan.offer_id.clone(), - executor_commitment: plan.config_commitment.clone(), - max_proof_deadline_s: plan.deadline_s, - eval_image_digest: pin.eval_image_digest.clone(), - holdout_commitment: topic.holdout_commitment.clone(), - constraints: topic.constraints, - flops_budget: topic.flops_budget, - wall_budget_s: topic.metric.wall_budget_s, - claim: claim.to_owned(), - holdout: holdout.to_vec(), - }; let body = serde_json::to_vec(&request) .map_err(|e| EvalError::Backend(format!("encode request: {e}")))?; @@ -664,11 +700,12 @@ impl LiveScorer for LiumProofHarvest { template_id = %plan.template_id, gpu_count = plan.gpu_count, deadline_s = plan.deadline_s, + overridden = plan.overridden, "proof harvest rent plan" ); let instance = self .pod - .boot(&self.spec(pin, frozen_digest, &plan)) + .boot(&self.spec(pin, frozen_digest, plan)) .await .map_err(EvalError::Backend)?; let stdout = self @@ -676,13 +713,18 @@ impl LiveScorer for LiumProofHarvest { .await?; if !PROGRAM.ran_to_completion(&stdout) { let stdout_tail = truncate_tail(&stdout, STDOUT_TAIL_BYTES); - tracing::warn!( - instance, - stdout_tail = %stdout_tail, - "eval image did not print {OK_MARKER}; refusing" - ); + // An external SIGKILL (GPU OOM, host pressure) is an + // infrastructure failure, not the proof deadline; name it so the + // operator does not chase the wrong budget. + let what = if killed_externally(&stdout) { + "eval image was SIGKILLed before the deadline (exit=137: external kill such as OOM)" + .to_owned() + } else { + format!("eval image did not print {OK_MARKER}") + }; + tracing::warn!(instance, stdout_tail = %stdout_tail, "{what}; refusing"); return Err(EvalError::Backend(format!( - "eval image did not print {OK_MARKER}" + "{what}; stdout_tail: {stdout_tail}" ))); } let body = PROGRAM.extract_document(&stdout).ok_or_else(|| { @@ -717,6 +759,25 @@ mod tests { use super::*; + /// What `eval_after_freeze` does on the Lium path: resolve the rent plan + /// with this harvest's overrides, then score under it. + async fn score_via_plan( + harvest: &LiumProofHarvest, + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + executor: &EvalExecutorOffer, + frozen: &str, + artifact: &str, + holdout: &[HoldoutRecord], + claim: &str, + ) -> Result { + let plan = harvest.plan(pin, topic, executor)?; + harvest + .score(pin, topic, offer, &plan, frozen, artifact, holdout, claim) + .await + } + #[test] fn program_is_proof_not_relearn() { assert_eq!(PROGRAM.metrics_marker, "PROOF_METRICS="); @@ -760,6 +821,7 @@ mod tests { max_output_tokens: 256, config_commitment: "ab".repeat(32), executor_offer_id: "lium-1x-v0".into(), + executor_offer_commitment: "cd".repeat(32), executor_commitment: "cd".repeat(32), max_proof_deadline_s: 3_600, eval_image_digest: String::new(), @@ -999,19 +1061,19 @@ mod tests { let pod = CapturePod::new(); let harvest = harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("capture"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("capture"); assert!(matches!(err, EvalError::Backend(_)), "{err}"); assert!(*pod.booted.lock().expect("booted")); let env = String::from_utf8(pod.env.lock().expect("env").clone()).expect("utf8"); @@ -1058,19 +1120,19 @@ mod tests { let topic = harvest_topic(&recs); let pod = CapturePod::new(); let harvest = harvest_with_assets(pod.clone(), &recs, None); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("no key"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("no key"); assert!(matches!(err, EvalError::InferenceAuthMissing), "{err}"); assert!(!*pod.booted.lock().expect("booted")); } @@ -1086,19 +1148,19 @@ mod tests { vec!["ssh-ed25519 AAAAtest proof".into()], ) .with_judge_api_key(Some("sk-live-not-a-real-secret".into())); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("no proxy"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("no proxy"); assert!(matches!(err, EvalError::ProxyModelMissing), "{err}"); assert!(!*pod.booted.lock().expect("booted")); } @@ -1118,19 +1180,19 @@ mod tests { .with_judge_api_key(Some("sk-live-not-a-real-secret".into())) .with_proxy_model_dir(Some(empty)) .with_holdout_store(Some(store)); - let err = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("empty proxy"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("empty proxy"); assert!(matches!(err, EvalError::ProxyModelMissing), "{err}"); assert!(!*pod.booted.lock().expect("booted")); } @@ -1193,19 +1255,19 @@ mod tests { let mut missing = recs.clone(); missing[0].content_sha256 = "ab".repeat(32); let before = leftover_proxy_tars(&staging); - let err = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &missing, - "claim", - ) - .await - .expect_err("missing shard"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &missing, + "claim", + ) + .await + .expect_err("missing shard"); assert!(matches!(err, EvalError::HoldoutStoreMissing), "{err}"); assert!(!*pod.booted.lock().expect("booted")); let after = leftover_proxy_tars(&staging); @@ -1391,19 +1453,19 @@ mod tests { let pod = CapturePod::new(); let harvest = harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("spoof"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("spoof"); assert!(err.to_string().contains("committed judge origin"), "{err}"); assert!(!*pod.booted.lock().expect("booted")); } @@ -1462,10 +1524,13 @@ mod tests { } } + /// Pod stdout for a run the wrapper ended: `124` on TERM, or `137` after + /// `--kill-after` — the run command marks both with the deadline line. fn deadline_stdout(rc: u32) -> String { format!( - "{}\nexit={rc}\nTraceback: still training step 4200 when the proof deadline hit\n", - "boot ok\n".repeat(3) + "{}\nexit={rc}\n{}\nTraceback: still training step 4200 when the proof deadline hit\n", + "boot ok\n".repeat(3), + harvest_pod::DEADLINE_MARKER ) } @@ -1478,19 +1543,19 @@ mod tests { .with_harvest_overrides(Some(HarvestOverrides::default())); let mut topic = harvest_topic(&recs); topic.eval_executor.max_proof_deadline_s = Some(1_800); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("capture"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("capture"); assert!(matches!(err, EvalError::Backend(_)), "{err}"); let spec = pod.spec(); assert_eq!(spec.gpu_count, 1); @@ -1517,46 +1582,189 @@ mod tests { serde_json::from_slice(&pod.request.lock().expect("req")).expect("json"); assert_eq!(req["executor_offer_id"], "lium-1x-v0"); assert_eq!( - req["executor_commitment"], + req["executor_offer_commitment"], harvest_executor().config_commitment ); + assert_eq!( + req["executor_commitment"], + proof_executor::executor_config_commitment( + "proof-eval-abababababab", + "1x", + 1_800, + &harvest_pin().eval_image_digest + ), + "the request commits the executed configuration (topic-tightened 1800s)" + ); assert_eq!(req["max_proof_deadline_s"], 1_800); } + /// A raw Lium template UUID would be rented verbatim by the provider, so + /// the digest-bound resolver could never check its image: refused before + /// boot even when the pin allowlist is empty (offer or env override). #[tokio::test] - async fn harvest_rents_a_uuid_template_verbatim() { + async fn harvest_refuses_a_raw_uuid_template_from_offer_or_override() { let recs = synthetic_holdout(STRATUM_SIZE, 1); - let pod = CapturePod::new(); let mut pin = harvest_pin(); pin.allowed_lium_template_prefixes.clear(); let mut executor = harvest_executor(); executor.lium_template_id = "f2f5e84c-3b09-4090-be83-1913eabd009e".into(); executor.config_commitment = executor.expected_commitment(); + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())); + let err = score_via_plan( + &harvest, + &pin, + &harvest_topic(&recs), + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("raw uuid offer"); + assert!( + matches!(err, EvalError::ExecutorOffer(ref m) if m.contains("raw Lium template id")), + "{err}" + ); + assert!(!*pod.booted.lock().expect("booted")); + + let pod = CapturePod::new(); let harvest = harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) - .with_harvest_overrides(Some(HarvestOverrides::default())); - let _ = harvest - .score( - &pin, - &harvest_topic(&recs), - &harvest_offer(), - &executor, - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await; - let spec = pod.spec(); + .with_harvest_overrides(Some(HarvestOverrides { + template_id: Some("f2f5e84c-3b09-4090-be83-1913eabd009e".into()), + ..HarvestOverrides::default() + })); + let err = score_via_plan( + &harvest, + &pin, + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("raw uuid override"); + assert!( + matches!(err, EvalError::ExecutorOffer(ref m) if m.contains("raw Lium template id")), + "{err}" + ); + assert!(!*pod.booted.lock().expect("booted")); + } + + /// A topic that pinned the offer commitment never runs under an + /// operator override that changes the template or deadline; the same + /// override on an unpinned topic runs and is re-committed as what ran. + #[tokio::test] + async fn harvest_refuses_config_changing_override_when_the_topic_pins_the_commitment() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let mut pinned = harvest_topic(&recs); + pinned.eval_executor.require_offer_commitment = Some(harvest_executor().config_commitment); + let override_ = HarvestOverrides { + deadline_secs: Some(600), + ..HarvestOverrides::default() + }; + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(override_.clone())); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &pinned, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("pinned topic"); + assert!( + matches!(err, EvalError::ExecutorOffer(ref m) if m.contains("pins the offer config_commitment")), + "{err}" + ); + assert!(!*pod.booted.lock().expect("booted"), "must not rent"); + + let pod = CapturePod::new(); + let harvest = + harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) + .with_harvest_overrides(Some(override_)); + let _ = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await; + assert!(*pod.booted.lock().expect("booted")); + let req: serde_json::Value = + serde_json::from_slice(&pod.request.lock().expect("req")).expect("json"); + assert_eq!(req["max_proof_deadline_s"], 600); assert_eq!( - spec.template_id.as_deref(), - Some("f2f5e84c-3b09-4090-be83-1913eabd009e") + req["executor_offer_commitment"], + harvest_executor().config_commitment + ); + assert_ne!( + req["executor_commitment"], req["executor_offer_commitment"], + "the run is stamped with what actually ran, not the offer's knobs" ); assert_eq!( - spec.template_name.as_deref(), - Some("proof-eval-abababababab") + req["executor_commitment"], + proof_executor::executor_config_commitment( + "proof-eval-abababababab", + "1x", + 600, + &harvest_pin().eval_image_digest + ) + ); + } + + /// `exit=137` without the wrapper's deadline marker is an external + /// SIGKILL (GPU OOM): a backend refusal that names it, not a deadline 503. + #[tokio::test] + async fn an_external_sigkill_is_not_reported_as_the_deadline() { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (proxy, store) = live_asset_dirs(&recs); + let pod = StdoutPod::new("boot ok\nexit=137\nCUDA error: out of memory\n"); + let harvest = LiumProofHarvest::new( + pod.clone(), + HarvestLimits::default(), + vec!["ssh-ed25519 AAAAtest proof".into()], + ) + .with_judge_api_key(Some("sk-live-not-a-real-secret".into())) + .with_proxy_model_dir(Some(proxy)) + .with_holdout_store(Some(store)) + .with_harvest_overrides(Some(HarvestOverrides::default())); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("oom"); + assert!( + matches!(err, EvalError::Backend(ref m) if m.contains("SIGKILLed before the deadline") && m.contains("out of memory")), + "{err}" ); - assert!(spec.exact_gpu_count); } #[tokio::test] @@ -1583,19 +1791,19 @@ mod tests { let harvest = harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) .with_harvest_overrides(Some(HarvestOverrides::default())); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &executor, - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err(label); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err(label); assert!(err.to_string().contains(want), "{label}: {err}"); assert!( !*pod.booted.lock().expect("booted"), @@ -1627,6 +1835,13 @@ mod tests { template_id: Some("prism-recipe-v10".into()), ..HarvestOverrides::default() }, + "does not carry the pinned eval image digest prefix", + ), + ( + HarvestOverrides { + template_id: Some("other-abababababab".into()), + ..HarvestOverrides::default() + }, "allowed_lium_template_prefixes", ), ] { @@ -1634,19 +1849,19 @@ mod tests { let harvest = harvest_with_assets(pod.clone(), &recs, Some("sk-live-not-a-real-secret".into())) .with_harvest_overrides(Some(overrides)); - let err = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("override refused"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("override refused"); assert!( matches!(err, EvalError::ExecutorOffer(ref m) if m.contains(want)), "{err}" @@ -1663,18 +1878,18 @@ mod tests { gpu_count: Some(1), deadline_secs: Some(600), })); - let _ = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await; + let _ = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await; assert!(*pod.booted.lock().expect("booted")); assert_eq!( pod.spec().template_name.as_deref(), @@ -1686,7 +1901,10 @@ mod tests { #[tokio::test] async fn a_run_cut_at_the_deadline_is_a_503_with_the_stdout_tail() { let recs = synthetic_holdout(STRATUM_SIZE, 1); - for rc in harvest_pod::DEADLINE_EXIT_CODES { + for rc in [ + harvest_pod::DEADLINE_EXIT_CODE, + harvest_pod::SIGKILL_EXIT_CODE, + ] { let (proxy, store) = live_asset_dirs(&recs); let pod = StdoutPod::new(deadline_stdout(rc)); let harvest = LiumProofHarvest::new( @@ -1698,19 +1916,19 @@ mod tests { .with_proxy_model_dir(Some(proxy)) .with_holdout_store(Some(store)) .with_harvest_overrides(Some(HarvestOverrides::default())); - let err = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("deadline"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("deadline"); match err { EvalError::ProofDeadlineExceeded { deadline_s, @@ -1781,19 +1999,19 @@ mod tests { .with_holdout_store(Some(store)) .with_harvest_overrides(Some(HarvestOverrides::default())) .with_deadline_wait_grace_secs(0); - let err = harvest - .score( - &harvest_pin(), - &harvest_topic(&recs), - &harvest_offer(), - &executor, - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("wait elapsed"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &harvest_topic(&recs), + &harvest_offer(), + &executor, + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("wait elapsed"); assert!( matches!( err, @@ -1823,19 +2041,19 @@ mod tests { .with_judge_api_key(Some("sk-live-not-a-real-secret".into())) .with_proxy_model_dir(Some(proxy)) .with_holdout_store(Some(store)); - let err = harvest - .score( - &harvest_pin(), - &topic, - &harvest_offer(), - &harvest_executor(), - "digest-abcdef", - "artifact", - &recs, - "claim", - ) - .await - .expect_err("no ok"); + let err = score_via_plan( + &harvest, + &harvest_pin(), + &topic, + &harvest_offer(), + &harvest_executor(), + "digest-abcdef", + "artifact", + &recs, + "claim", + ) + .await + .expect_err("no ok"); assert!( matches!(err, EvalError::Backend(ref m) if m.contains(OK_MARKER)), "{err}" diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 4f6f49109..51297c06f 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -34,7 +34,7 @@ use proof_eval::{ contamination_evidence, eval_after_freeze, force_sim, scoring_readiness, secret_backed_base_url, supported_custom, EvalBackend, EvalError, LiveScorer, }; -use proof_executor::{require_open_executor, EvalExecutorOffer}; +use proof_executor::{require_open_executor, EvalExecutorOffer, ExecutorPlan}; use proof_score::{ judge_topic, primary_from_harness, AgentVerdict, GateFail, HarnessMetrics, MinerTopicRun, ProofKind, ProofVerdict, @@ -406,6 +406,7 @@ async fn submit( persist_scored( &st, executor.as_ref(), + eval.executor.as_ref(), body, hotkey, artifact, @@ -509,6 +510,7 @@ fn persist_pre_eval_reject( fn persist_scored( st: &AppState, executor: Option<&EvalExecutorOffer>, + plan: Option<&ExecutorPlan>, body: SubmitBody, hotkey: String, artifact: String, @@ -553,8 +555,11 @@ fn persist_scored( .map(|o| o.config_commitment.clone()) .unwrap_or_default(), executor_offer_id: executor.map(|x| x.offer_id.clone()).unwrap_or_default(), - executor_commitment: executor - .map(|x| x.config_commitment.clone()) + // A live run stamps the configuration it was actually held to + // (template, 1x, effective deadline); sim has no plan and no rent. + executor_commitment: plan + .map(|p| p.config_commitment.clone()) + .or_else(|| executor.map(|x| x.config_commitment.clone())) .unwrap_or_default(), manifest: body.manifest, nonce, @@ -906,7 +911,7 @@ mod tests { pin: &ProofPin, topic: &TopicDocument, _offer: &InferenceOffer, - _executor: &EvalExecutorOffer, + _plan: &ExecutorPlan, frozen: &str, artifact: &str, _holdout: &[proof_task::HoldoutRecord], diff --git a/crates/proof-store/src/lib.rs b/crates/proof-store/src/lib.rs index 88ca8f4e6..a0dbcb571 100644 --- a/crates/proof-store/src/lib.rs +++ b/crates/proof-store/src/lib.rs @@ -86,7 +86,10 @@ pub struct Submission { /// on sim, which rents nothing). #[serde(default)] pub executor_offer_id: String, - /// Executor `config_commitment` stamped from the host offer. + /// Commitment of the executor configuration a live run was actually held + /// to (template, `1x`, effective deadline, digest) — the offer's + /// `config_commitment` when nothing tightened or overrode it, and the + /// offer's when no run happened (pre-eval reject). #[serde(default)] pub executor_commitment: String, /// Declared training fingerprints. From a7f39762be0554eb9833dcaa2dcef37978e33c65 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 12:57:53 +0000 Subject: [PATCH 16/16] docs(proof): executor binding, commitment, and deadline rules after review Co-authored-by: Mathis --- deploy/env/proof-challenge.env.example | 11 ++++-- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 46 +++++++++++++++++--------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 583859c57..7e8eb0ef4 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -55,8 +55,10 @@ PROOF_SIM_STUB_WIN=false # Operator bearer tokens for POST /v1/admin/proof/topics. # PROOF_ADMIN_TOKENS_FILE=/run/base/proof/admin_tokens -# Seconds the eval image gets to score one artifact on the pod. -# PROOF_EVAL_TIMEOUT_SECS=5400 +# Fallback seconds the eval image gets when no executor deadline was resolved. +# The resolved EvalExecutorOffer max_proof_deadline_s IS the pod timeout and +# is never clamped by this value. Default = pin ceiling (7200). +# PROOF_EVAL_TIMEOUT_SECS=7200 # Live RLM judge InferenceOffer (operator state, never git). The eval image # calls this backend to score miner submissions. Miners do not bind it. @@ -84,7 +86,10 @@ PROOF_SIM_STUB_WIN=false # # Optional hot-swap of the executor plan without a rebuild. Each replaces the # offer's value; the pin ceilings still bind (a value outside them refuses the -# rent, it is never clamped). Unparseable values also refuse. Leave unset. +# rent, it is never clamped). Unparseable values also refuse. A topic that +# pins require_offer_commitment refuses any override that changes the +# template or deadline. The template must be a digest-scoped name (never a +# raw Lium UUID). Leave unset. # PROOF_HARVEST_TEMPLATE_ID= # PROOF_HARVEST_GPU_COUNT=1 # PROOF_HARVEST_DEADLINE_SECS= diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 60d1aae70..cb7b13b1a 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -79,7 +79,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Compose / images | **done** | Default compose + `images.yml` target `proof-challenge`. | | Eval pin | **done** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, digest `sha256:78b614a1…` (publish-proof-eval-image run 33892650063, commit `51f937c7`). No HF bake; `proxy_model` stays empty. Live submits still **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. Do not re-pin a guessed sha256. | | Inference offer | **v0** | Digest-pinned RLM **judge** backend (`proof-eval` / harvest call it). Pin `[inference]` defaults plus schema v1 / ceilings / modes / commitment. `config_commitment` hashes config knobs **and** `provider.base_url`; a topic that spoofs origin is **503** before lattice. Topic `require_judge_offer_commitment` is optional and not a miner bind. Live `InferenceOffer` is operator state. Auth is `PROOF_INFERENCE_API_KEY_FILE` staged as harvest `teacher.env` (never git, never `/v1/status`). Missing/closed/judge down / missing key → `can_score=false` / 503. No baked Qwen; architecture ≠ HF stays retired. | -| Eval executor offer | **v0** | `crates/proof-executor`: live `1x` `EvalExecutorOffer` (Lium template, `machine_shape`, `max_proof_deadline_s`, digest, `config_commitment`, status) — a sibling of the judge offer, not the same document. Pin ceilings `eval_executor_schema_version` / `gpu_class = "1x"` / `max_proof_deadline_s_ceiling = 7200` / optional `allowed_lium_template_prefixes` / `eval_executor_commitment_alg`. Public on `GET /v1/status` + `GET /v1/proof/executor`; rotated via `POST /v1/admin/proof/executor` (in-memory until restart; boot from `PROOF_EVAL_EXECUTOR_OFFER_FILE`). Topic tighten-only `eval_executor.{require_offer_commitment, max_proof_deadline_s}`, no per-topic `machine_id`. Lium path: missing/closed/shape ≠ `1x` → `can_score=false` / 503; harvest rents the offer template at exactly `1x` (`rent_gpu_count ≠ 1` aborts pre-rent) and holds the run to the deadline (pod `timeout` + harvest wait; exceed → 503 + `stdout_tail`). `PROOF_HARVEST_TEMPLATE_ID` / `_GPU_COUNT` / `_DEADLINE_SECS` hot-swap under the pin ceilings. Sim does not consult it. No live Lium rent in CI. | +| Eval executor offer | **v0** | `crates/proof-executor`: live `1x` `EvalExecutorOffer` (Lium template, `machine_shape`, `max_proof_deadline_s`, digest, `config_commitment`, status) — a sibling of the judge offer, not the same document. Pin ceilings `eval_executor_schema_version` / `gpu_class = "1x"` / `max_proof_deadline_s_ceiling = 7200` / optional `allowed_lium_template_prefixes` / `eval_executor_commitment_alg`. Public on `GET /v1/status` + `GET /v1/proof/executor`; rotated via `POST /v1/admin/proof/executor` (in-memory until restart; boot from `PROOF_EVAL_EXECUTOR_OFFER_FILE`). Topic tighten-only `eval_executor.{require_offer_commitment, max_proof_deadline_s}`, no per-topic `machine_id`. Lium path: missing/closed/shape ≠ `1x` → `can_score=false` / 503; harvest rents the offer's digest-scoped template (raw Lium UUIDs refused under any allowlist; the resolver binds the template to `eval_image@digest`) at exactly `1x` (`rent_gpu_count ≠ 1` aborts pre-rent) and holds the run to the deadline (the deadline is the pod `timeout`, never clamped by the host fallback; harvest wait = deadline + grace; wrapper-cut run → 503 + `stdout_tail`, external SIGKILL named separately). `PROOF_HARVEST_TEMPLATE_ID` / `_GPU_COUNT` / `_DEADLINE_SECS` hot-swap under the pin ceilings; refused when the topic pins the offer commitment; the run request and row stamp the commitment of what actually ran. Sim does not consult it. No live Lium rent in CI. | | Topics | **done** | sr25519 under the `proof` trust-root key (`base-proof-topic-v1`). Admin `POST /v1/admin/proof/topics`. A topic must be sealed to `open`. | | Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. | | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | diff --git a/docs/PROOF.md b/docs/PROOF.md index e27f1a4ae..c30d5cc3e 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -77,22 +77,38 @@ baseline + an open topic are on the host. rotate or close) names `offer_id`, `lium_template_id`, `machine_shape`, `max_proof_deadline_s` (≤ ceiling), `eval_image_digest` (must equal the pin when non-empty), `config_commitment` = sha256 of the canonical public knobs, - and `status`. Every field is public: `GET /v1/status` (`eval_executor`, - pin `executor`) and `GET /v1/proof/executor` show it whole. Missing / - closed / `machine_shape ≠ 1x` → `can_score=false` → **503** on the Lium - path (sim rents nothing and does not consult it). Harvest rents exactly - that template at exactly `1x` — a rent that would upsize to a whole host + and `status`. `lium_template_id` is the **digest-scoped template name** + (it must carry the pinned digest's 12-hex prefix); harvest resolves it + through the digest-bound resolver, which reuses a listed template only when + its image is `eval_image@digest` and otherwise creates one bound to it. A + raw Lium template UUID is **refused** (offer or override, under any + allowlist) because the provider would rent it verbatim with no image + check; a pin with no digest binds no executor at all. Every field is + public: `GET /v1/status` (`eval_executor`, pin `executor`) and + `GET /v1/proof/executor` show it whole. Missing / closed / + `machine_shape ≠ 1x` → `can_score=false` → **503** on the Lium path (sim + rents nothing and does not consult it). Harvest rents exactly that + template at exactly `1x` — a rent that would upsize to a whole host (`rent_gpu_count ≠ 1`) aborts before the rent — and holds the run to the - resolved deadline both pod-side (`timeout`) and in its own wait; a run cut - at the deadline is a **503** carrying the pod's `stdout_tail`, never a - zero. A topic may only tighten: `eval_executor.max_proof_deadline_s` - (shorter) and `eval_executor.require_offer_commitment` (64-hex pin of the - live offer, not a miner bind). There is **no per-topic `machine_id`** - (publish reject). Operator hot-swap without a rebuild: - `PROOF_HARVEST_TEMPLATE_ID` / `PROOF_HARVEST_GPU_COUNT` / - `PROOF_HARVEST_DEADLINE_SECS` replace the offer's values; the pin ceilings - still bind, and an unparseable or out-of-ceiling value refuses the rent - rather than clamping. Ceremony: + resolved deadline: the deadline **is** the pod-side `timeout` (never + clamped below it by the host's `PROOF_EVAL_TIMEOUT_SECS` fallback, whose + default equals the ceiling) and the harvest wait is deadline + grace. A run + the wrapper cut (`exit=124`, or `137` after the full budget) is a **503** + carrying the pod's `stdout_tail`, never a zero; a `137` before the deadline + is reported as an external SIGKILL (e.g. OOM), not as the deadline. A + topic may only tighten: `eval_executor.max_proof_deadline_s` (shorter) and + `eval_executor.require_offer_commitment` (64-hex pin of the live offer, + not a miner bind). There is **no per-topic `machine_id`** (publish + reject). Operator hot-swap without a rebuild: `PROOF_HARVEST_TEMPLATE_ID` + / `PROOF_HARVEST_GPU_COUNT` / `PROOF_HARVEST_DEADLINE_SECS` replace the + offer's values; the pin ceilings still bind, an unparseable or + out-of-ceiling value refuses the rent rather than clamping, and a topic + that pins `require_offer_commitment` **refuses** any override that changes + the template or deadline (it approved the offer's configuration, not the + operator's). The run request and the scored row stamp the commitment of + the configuration that actually ran (`executor_commitment`) next to the + offer's (`executor_offer_commitment` on the request); they differ only when + a topic tighten or an override changed the offer's knobs. Ceremony: `cargo run -p xtask -- proof-executor-offer --offer-id --max-proof-deadline-s --out `. Isolation: the control-plane host runs neither the eval image nor the RLM judge; the harvest is the **only** path to the rented GPU, and the executor