From 11e59b3c4a3ecf35ecdd36bd6e63a7a142154116 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:53:30 +0000 Subject: [PATCH 1/3] feat(proof): wire custom-family registry without a lium harvest The runner registry and the FamilyMux custom route were only built inside build_live_scorer().map(with_custom_family), so a host with the topic-VM orchestrator and PROOF_VM_RUNNER_CUSTOM_IDS set but no LIUM_API_KEY kept every custom topic unwired and needed a placeholder Lium harvest to open them. - FamilyMux: the default (harvest) route is optional. FamilyMux::custom_only routes custom to the RLM scorer and refuses every nll / throughput topic at readiness, plan, and score with LiveHarvestUnavailable (503, no row, no rent). ready() stays the default route's readiness; with no harvest there is no host-wide blocker and the refusal lands per topic. - proof-challenge: the topic-VM orchestrator + registry are resolved on their own; with a harvest the mux routes both families (unchanged); with none, a live FirecrackerOrchestrator selected by env plus >= 1 registered id installs the custom-only mux. Env unset or refused (UnwiredVmOrchestrator), no id, or Sim keeps live_scorer None, so the host stays fail-closed exactly as before. - Tests: no Lium + full FC env -> registry non-empty, host-wide gate passes, registered custom topic ready, nll/unlisted ids refuse; env unset / ids over the unwired stub / refused URL / no id -> None -> LiveHarvestUnavailable; HTTP: custom-only host scores the custom topic, a throughput submit is 503 with no row and no scorer call. Co-authored-by: Mathis --- bins/proof-challenge/Cargo.toml | 1 + bins/proof-challenge/src/main.rs | 387 +++++++++++++++++++++++++++---- crates/proof-eval/src/lib.rs | 140 ++++++++++- crates/proof-http/src/lib.rs | 102 +++++++- 4 files changed, 569 insertions(+), 61 deletions(-) diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index dc5c45a18..b66d44c5c 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -35,6 +35,7 @@ tracing = "0.1" [dev-dependencies] crypto = { path = "../../crates/crypto" } hex = "0.4" +proof-rlm = { path = "../../crates/proof-rlm", features = ["test-fixtures"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } sha2 = "0.10" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "time"] } diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index e6141dfff..e36185ad4 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -5,8 +5,10 @@ //! signed documents, not a catalog in git. //! //! Without `PROOF_FORCE_SIM=1` the host needs a `sha256:` eval-image pin, a -//! wired harvest, at least one `open` topic with a verified holdout, and a -//! sealed baseline. Sim is never a fallback. +//! wired scorer for the topic's family (the Lium harvest for `nll` / +//! `throughput`; the topic-VM orchestrator plus a registered custom id for +//! `custom` — each wired on its own), at least one `open` topic with a +//! verified holdout, and a sealed baseline. Sim is never a fallback. #![forbid(unsafe_code)] @@ -174,30 +176,21 @@ fn run(cli: &Cli) -> Result<(), String> { .map_err(|e| e.to_string())?; let rlm_store = rt.block_on(resolve_rlm_store(cli))?; - let live_scorer = build_live_scorer( + let harvest = build_live_scorer( backend, cli.eval_timeout_secs, judge_api_key.clone(), cli.proxy_model_dir.clone(), cli.holdout_store.clone(), - ) - .map(|harvest| with_custom_family(harvest, rlm_store, &cli.artefact_root)); - match backend { - EvalBackend::Lium if live_scorer.is_some() => { - tracing::info!("live harvest wired: digest-pinned proof-eval image on Lium"); - tracing::info!( - registered_custom = ?registered_custom(live_scorer.as_deref()), - artefact_root = %cli.artefact_root.display(), - "custom-family topics route to the rlm scorer; an id with no registered runner \ - answers 503 (no runner is compiled in)" - ); - } - EvalBackend::Lium => tracing::warn!( - "live harvest not wired; every submission will 503. Set the Lium credentials \ - and LIUM_SSH_PUBLIC_KEY_FILE (deploy/env/proof-challenge.env.example)" - ), - EvalBackend::Sim => {} - } + ); + let harvest_wired = harvest.is_some(); + let live_scorer = live_scorer(backend, harvest, rlm_store, &cli.artefact_root); + log_live_wiring( + backend, + harvest_wired, + live_scorer.as_deref(), + &cli.artefact_root, + ); let store = MemoryStore::new(); let registered = registered_custom(live_scorer.as_deref()); @@ -276,7 +269,86 @@ fn build_live_scorer( )) } -/// Route the `custom` metric family to the RLM scorer over the default harvest. +/// Boot log for what [`live_scorer`] wired, and what answers 503 because of +/// what is missing. Nothing here is a boot error: a live host refuses until +/// the operator wires the piece it names. +fn log_live_wiring( + backend: EvalBackend, + harvest_wired: bool, + live: Option<&dyn LiveScorer>, + artefact_root: &Path, +) { + match backend { + EvalBackend::Lium if harvest_wired => { + tracing::info!("live harvest wired: digest-pinned proof-eval image on Lium"); + tracing::info!( + registered_custom = ?registered_custom(live), + artefact_root = %artefact_root.display(), + "custom-family topics route to the rlm scorer; an id with no registered runner \ + answers 503 (no runner is compiled in)" + ); + } + EvalBackend::Lium if live.is_some() => tracing::info!( + registered_custom = ?registered_custom(live), + artefact_root = %artefact_root.display(), + "live harvest not wired: custom-family topics route to the rlm scorer over the \ + topic-vm orchestrator; every nll / throughput topic answers 503 until the Lium \ + credentials and LIUM_SSH_PUBLIC_KEY_FILE are set" + ), + EvalBackend::Lium => tracing::warn!( + "live harvest not wired; every submission will 503. Set the Lium credentials \ + and LIUM_SSH_PUBLIC_KEY_FILE (deploy/env/proof-challenge.env.example), or wire \ + the topic-vm orchestrator and {VM_RUNNER_CUSTOM_IDS_ENV} for custom topics" + ), + EvalBackend::Sim => {} + } +} + +/// The live scorer of this host, by metric family. +/// +/// `nll` / `throughput` score on the digest-pinned Lium harvest; `custom` +/// scores on the RLM scorer over the runner registry [`custom_family`] +/// builds from the topic-VM orchestrator env. The two are wired +/// independently: with a harvest the mux routes both families; with none, +/// the custom family still stands on its own when the env selected the live +/// orchestrator and registered at least one custom id +/// ([`FamilyMux::custom_only`] — every `nll` / `throughput` topic then +/// answers 503 `LiveHarvestUnavailable`, no row, no rent). Neither wired → +/// `None`, and every submission answers 503. Sim scores in-process and +/// wires nothing live. +fn live_scorer( + backend: EvalBackend, + harvest: Option>, + rlm_store: Arc, + artefact_root: &Path, +) -> Option> { + if backend != EvalBackend::Lium { + return None; + } + let custom = custom_family(rlm_store, artefact_root); + match harvest { + Some(harvest) => Some(Arc::new( + FamilyMux::new(harvest).with_custom_family(custom.scorer), + )), + None if custom.standalone => Some(Arc::new(FamilyMux::custom_only(custom.scorer))), + None => None, + } +} + +/// The custom-family scorer of this host. +struct CustomFamily { + /// RLM scorer over the runner registry. + scorer: Arc, + /// The env selected the live topic-VM orchestrator **and** at least one + /// custom id is registered over it: enough to carry the custom family + /// with no Lium harvest. False with the unwired orchestrator (env unset + /// or refused) or an empty registry — a mux with nothing to route to is + /// not wired. + standalone: bool, +} + +/// The RLM scorer over the runner registry, wired from the topic-VM +/// orchestrator env alone — never from the Lium harvest. /// /// No benchmark, model, or repository is compiled in: the registry holds only /// the generic `VmBackedRunner`, under the custom ids the operator lists in @@ -285,14 +357,27 @@ fn build_live_scorer( /// every custom topic answers 503 (`RunnerUnwired`); with ids but an unwired /// or unpinned orchestrator, 503 naming the missing env var. It never falls /// back to the digest-pinned harvest and never spends. -fn with_custom_family( - harvest: Arc, - rlm_store: Arc, - artefact_root: &Path, -) -> Arc { - let scorer = RlmScorer::new(Arc::new(runner_registry()), rlm_store) +fn custom_family(rlm_store: Arc, artefact_root: &Path) -> CustomFamily { + let vm = topic_vm_orchestrator(); + let registry = runner_registry(&vm); + let standalone = vm.live && !registry.is_empty(); + let scorer = RlmScorer::new(Arc::new(registry), rlm_store) .with_artefacts(Some(ArtefactStore::new(artefact_root))); - Arc::new(FamilyMux::new(harvest).with_custom_family(Arc::new(scorer))) + CustomFamily { + scorer: Arc::new(scorer), + standalone, + } +} + +/// What the topic-VM orchestrator env resolved to. +struct TopicVm { + orchestrator: Arc, + /// RLM VM template the runner boots for topics without a VM. + template: VmTemplate, + /// The env selected the live `FirecrackerOrchestrator` (URL + bearer + /// file env, https). False = `UnwiredVmOrchestrator`, which refuses + /// every call. + live: bool, } /// The topic-VM orchestrator this host talks to, plus the RLM VM template. @@ -305,7 +390,7 @@ fn with_custom_family( /// the error logged: a half-configured orchestrator never becomes a host /// fallback. Token / digest are checked at `ready()` so they can be fixed /// without a restart. -fn topic_vm_orchestrator() -> (Arc, VmTemplate) { +fn topic_vm_orchestrator() -> TopicVm { match FirecrackerOrchestrator::from_env() { Ok(Some(fc)) => { let template = fc.template().clone(); @@ -321,7 +406,11 @@ fn topic_vm_orchestrator() -> (Arc, VmTemplate) { topics answer 503 until fixed" ), } - (Arc::new(fc), template) + TopicVm { + orchestrator: Arc::new(fc), + template, + live: true, + } } Ok(None) => { tracing::warn!( @@ -329,21 +418,33 @@ fn topic_vm_orchestrator() -> (Arc, VmTemplate) { {VM_ORCHESTRATOR_TOKEN_FILE_ENV} / {RLM_VM_IMAGE_DIGEST_ENV} unset); every custom \ topic answers 503 and nothing runs on this host" ); - (Arc::new(UnwiredVmOrchestrator), VmTemplate::from_env()) + TopicVm::unwired() } Err(e) => { tracing::warn!( "topic-vm orchestrator refused ({e}); staying unwired, custom topics 503" ); - (Arc::new(UnwiredVmOrchestrator), VmTemplate::from_env()) + TopicVm::unwired() + } + } +} + +impl TopicVm { + fn unwired() -> Self { + Self { + orchestrator: Arc::new(UnwiredVmOrchestrator), + template: VmTemplate::from_env(), + live: false, } } } /// `custom_id → VmBackedRunner` for every id in `PROOF_VM_RUNNER_CUSTOM_IDS`. -fn runner_registry() -> RunnerRegistry { - let (orchestrator, template) = topic_vm_orchestrator(); - let runner = Arc::new(VmBackedRunner::new(orchestrator, template)); +fn runner_registry(vm: &TopicVm) -> RunnerRegistry { + let runner = Arc::new(VmBackedRunner::new( + vm.orchestrator.clone(), + vm.template.clone(), + )); let raw = std::env::var(VM_RUNNER_CUSTOM_IDS_ENV).unwrap_or_default(); registry_for(&raw, &runner) } @@ -682,7 +783,13 @@ mod tests { std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); let root = std::env::temp_dir().join("proof-families-artefacts"); - let mux = with_custom_family(harvest, Arc::new(MemoryRlmStore::new()), &root); + let mux = live_scorer( + EvalBackend::Lium, + Some(harvest), + Arc::new(MemoryRlmStore::new()), + &root, + ) + .expect("a wired harvest is the live scorer"); assert!(registered_custom(Some(mux.as_ref())).is_empty()); for id in ["any_metric", "another_metric"] { let mut custom = TopicDocument::default(); @@ -805,12 +912,13 @@ mod tests { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); clear_vm_env(); - let (unwired, template) = topic_vm_orchestrator(); + let unwired = topic_vm_orchestrator(); assert!(matches!( - unwired.ready(), + unwired.orchestrator.ready(), Err(proof_rlm::VmError::NotWired(_)) )); - assert!(template.image_digest.is_empty()); + assert!(unwired.template.image_digest.is_empty()); + assert!(!unwired.live); let dir = std::env::temp_dir().join(format!("proof-vm-wire-{}", std::process::id())); std::fs::create_dir_all(&dir).expect("dir"); @@ -818,25 +926,32 @@ mod tests { std::fs::write(&token, "vm-bearer-not-a-real-secret\n").expect("token"); std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "https://kvm.example.invalid:8200"); std::env::set_var(VM_ORCHESTRATOR_TOKEN_FILE_ENV, &token); - let (pinned_less, template) = topic_vm_orchestrator(); - let err = pinned_less.ready().expect_err("no image pin"); + let pinned_less = topic_vm_orchestrator(); + let err = pinned_less.orchestrator.ready().expect_err("no image pin"); assert!(matches!(err, proof_rlm::VmError::NotWired(_)), "{err}"); assert!(err.to_string().contains(RLM_VM_IMAGE_DIGEST_ENV), "{err}"); assert_eq!( - (template.vcpus, template.mem_mib), + (pinned_less.template.vcpus, pinned_less.template.mem_mib), (4, 8_192), "locked shape" ); + assert!( + pinned_less.live, + "selected by env; readiness is per request" + ); std::env::set_var( RLM_VM_IMAGE_DIGEST_ENV, format!("sha256:{}", "ab".repeat(32)), ); - let (live, template) = topic_vm_orchestrator(); - live.ready().expect("url + token + digest = wired"); - template.validate().expect("pinned"); + let live = topic_vm_orchestrator(); + live.orchestrator + .ready() + .expect("url + token + digest = wired"); + live.template.validate().expect("pinned"); + assert!(live.live); std::env::set_var(VM_RUNNER_CUSTOM_IDS_ENV, "metric_a"); - let reg = runner_registry(); + let reg = runner_registry(&live); assert_eq!(reg.ids(), vec!["metric_a".to_owned()]); reg.resolve("metric_a") .expect("registered") @@ -844,23 +959,195 @@ mod tests { .expect("runner over the live orchestrator is ready"); std::fs::write(&token, "\n").expect("empty token"); - let (live, _) = topic_vm_orchestrator(); - let err = live.ready().expect_err("empty bearer file"); + let live = topic_vm_orchestrator(); + let err = live.orchestrator.ready().expect_err("empty bearer file"); assert!( err.to_string().contains(VM_ORCHESTRATOR_TOKEN_FILE_ENV), "{err}" ); std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "http://10.0.0.7:8200"); - let (refused, _) = topic_vm_orchestrator(); + let refused = topic_vm_orchestrator(); let err = refused + .orchestrator .ready() .expect_err("plain http off loopback is never wired"); assert!(err.to_string().contains(VM_ORCHESTRATOR_URL_ENV), "{err}"); + assert!(!refused.live, "a refused config is the unwired stub"); + clear_vm_env(); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Full topic-VM env (https URL, bearer file, image pin, one custom id), + /// **no** Lium credentials: the custom family stands on its own. The + /// registry is non-empty, the mux passes the host-wide gate and the + /// registered custom topic is ready over the live orchestrator, while + /// every `nll` / `throughput` topic refuses with + /// `LiveHarvestUnavailable` and an unlisted custom id with + /// `RunnerUnwired` — no placeholder Lium harvest is needed to open + /// custom topics. + #[test] + fn custom_family_stands_without_a_lium_harvest() { + let _guard = LIUM_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + clear_vm_env(); + std::env::remove_var("LIUM_API_KEY"); + std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); + let dir = std::env::temp_dir().join(format!("proof-custom-only-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let token = dir.join("vm_orchestrator_token"); + std::fs::write(&token, "vm-bearer-not-a-real-secret\n").expect("token"); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "https://kvm.example.invalid:8200"); + std::env::set_var(VM_ORCHESTRATOR_TOKEN_FILE_ENV, &token); + std::env::set_var( + RLM_VM_IMAGE_DIGEST_ENV, + format!("sha256:{}", "ab".repeat(32)), + ); + std::env::set_var(VM_RUNNER_CUSTOM_IDS_ENV, "metric_a"); + + let harvest = build_live_scorer(EvalBackend::Lium, 900, None, None, None); + assert!(harvest.is_none(), "no Lium credentials, no harvest"); + let root = dir.join("artefacts"); + let mux = live_scorer( + EvalBackend::Lium, + harvest, + Arc::new(MemoryRlmStore::new()), + &root, + ) + .expect("the custom family is wired from the topic-vm env alone"); + assert_eq!( + registered_custom(Some(mux.as_ref())), + vec!["metric_a".to_owned()] + ); + mux.ready().expect("no host-wide blocker"); + + let mut custom = TopicDocument::default(); + custom.metric.family = proof_task::MetricFamily::Custom; + custom.metric.custom_id = "metric_a".into(); + mux.ready_for_topic(&custom) + .expect("registered runner over the live orchestrator is ready"); + custom.metric.custom_id = "metric_b".into(); + let err = mux.ready_for_topic(&custom).expect_err("unlisted id"); + assert!( + matches!(err, proof_eval::EvalError::RunnerUnwired { .. }), + "{err}" + ); + let nll = TopicDocument::default(); + let err = mux.ready_for_topic(&nll).expect_err("no harvest"); + assert!( + matches!(err, proof_eval::EvalError::LiveHarvestUnavailable), + "{err}" + ); + + // Exactly the gate `/v1/status` and `POST /v1/submissions` apply + // host-wide: it passes, so custom topics can score here. + let pin = proof_rlm::fixtures::pin(); + let exec = executor_for(&pin); + proof_eval::scoring_readiness( + &pin, + EvalBackend::Lium, + Some(mux.as_ref()), + true, + Some(&proof_rlm::fixtures::offer()), + Some(&exec), + Some("test-judge-key"), + ) + .expect("ready host-wide without Lium"); + + // Sim scores in-process and never wires anything live. + assert!(live_scorer( + EvalBackend::Sim, + None, + Arc::new(MemoryRlmStore::new()), + &root + ) + .is_none()); clear_vm_env(); let _ = std::fs::remove_dir_all(&dir); } + /// Without Lium the custom family needs the live orchestrator **and** a + /// listed id, or nothing is wired and the host stays fail-closed: + /// orchestrator env unset (ids listed or not), env refused, or ids + /// unset → `None` → `LiveHarvestUnavailable` host-wide (503). The + /// `UnwiredVmOrchestrator` never carries a mux. + #[test] + fn without_lium_the_custom_family_needs_the_live_orchestrator_and_an_id() { + let _guard = LIUM_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + clear_vm_env(); + std::env::remove_var("LIUM_API_KEY"); + std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); + let dir = std::env::temp_dir().join(format!("proof-unwired-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let root = dir.join("artefacts"); + let store: Arc = Arc::new(MemoryRlmStore::new()); + let none = |label: &str| { + assert!( + live_scorer(EvalBackend::Lium, None, store.clone(), &root).is_none(), + "{label}: nothing may be wired" + ); + }; + + none("no env at all"); + std::env::set_var(VM_RUNNER_CUSTOM_IDS_ENV, "metric_a"); + none("ids listed over the unwired orchestrator"); + + let token = dir.join("vm_orchestrator_token"); + std::fs::write(&token, "vm-bearer-not-a-real-secret\n").expect("token"); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "http://10.0.0.7:8200"); + std::env::set_var(VM_ORCHESTRATOR_TOKEN_FILE_ENV, &token); + std::env::set_var( + RLM_VM_IMAGE_DIGEST_ENV, + format!("sha256:{}", "ab".repeat(32)), + ); + none("refused orchestrator config (plain http off loopback)"); + + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, "https://kvm.example.invalid:8200"); + std::env::remove_var(VM_RUNNER_CUSTOM_IDS_ENV); + none("live orchestrator but no custom id registered"); + std::env::set_var(VM_RUNNER_CUSTOM_IDS_ENV, " , Bad Id "); + none("live orchestrator but no valid custom id"); + + // What that `None` is on the wire: the host-wide gate refuses. + let pin = proof_rlm::fixtures::pin(); + let err = proof_eval::scoring_readiness( + &pin, + EvalBackend::Lium, + None, + true, + Some(&proof_rlm::fixtures::offer()), + Some(&executor_for(&pin)), + Some("test-judge-key"), + ) + .expect_err("unwired host"); + assert!( + matches!(err, proof_eval::EvalError::LiveHarvestUnavailable), + "{err}" + ); + clear_vm_env(); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Open `1x` executor on `pin`'s digest-scoped template (host state the + /// live gate requires; nothing here rents). + fn executor_for(pin: &ProofPin) -> EvalExecutorOffer { + let hex = pin.eval_image_digest.trim_start_matches("sha256:"); + let mut offer = EvalExecutorOffer { + offer_id: "executor-placeholder".into(), + lium_template_id: format!("proof-eval-{}", hex.get(..12).unwrap_or("unpinned")), + machine_shape: "1x".into(), + max_proof_deadline_s: 3_600, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: proof_challenge::OfferStatus::Open, + }; + offer.config_commitment = offer.expected_commitment(); + offer + } + static LIUM_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Compose always points `PROOF_PIN_FILE` at the committed pin. Empty diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 461880cab..0726be70c 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -367,9 +367,13 @@ pub trait LiveScorer: Send + Sync { /// custom-family scorer (which resolves the runner by `custom_id`, fail-closed); /// `nll` / `throughput` go to the default digest-pinned harvest. Planning, /// readiness, promotion, and artefact hooks follow the same route, so an -/// unregistered custom id can never fall back to the harvest. +/// unregistered custom id can never fall back to the harvest, and — on a +/// host with no harvest ([`Self::custom_only`]) — no `nll` / `throughput` +/// topic can ever reach the custom scorer or an in-process sim. pub struct FamilyMux { - default: Arc, + /// Digest-pinned harvest for `nll` / `throughput`. `None` on a host with + /// no Lium harvest: those families refuse per topic. + default: Option>, custom: Option>, } @@ -379,11 +383,24 @@ impl FamilyMux { #[must_use] pub fn new(default: Arc) -> Self { Self { - default, + default: Some(default), custom: None, } } + /// Mux with **no** default harvest: the `custom` family routes to + /// `scorer`; every `nll` / `throughput` topic refuses with + /// [`EvalError::LiveHarvestUnavailable`] at readiness, plan, and score + /// (503, no row, no rent). For a host whose topic-VM orchestrator and + /// custom runners are wired but whose Lium harvest is not. + #[must_use] + pub fn custom_only(scorer: Arc) -> Self { + Self { + default: None, + custom: Some(scorer), + } + } + /// Route the whole `custom` family to `scorer`. #[must_use] pub fn with_custom_family(mut self, scorer: Arc) -> Self { @@ -393,7 +410,10 @@ impl FamilyMux { fn route(&self, topic: &TopicDocument) -> Result<&dyn LiveScorer, EvalError> { if topic.metric.family != MetricFamily::Custom { - return Ok(self.default.as_ref()); + return self + .default + .as_deref() + .ok_or(EvalError::LiveHarvestUnavailable); } self.custom .as_deref() @@ -444,8 +464,12 @@ impl LiveScorer for FamilyMux { .await } + /// Host-wide gate: the default harvest's readiness. With no harvest + /// there is no host-wide blocker — the host still scores its `custom` + /// family — and the `nll` / `throughput` refusal lands per topic in the + /// route, where `ready_for_topic`, `plan`, and `score` look. fn ready(&self) -> Result<(), EvalError> { - self.default.ready() + self.default.as_deref().map_or(Ok(()), LiveScorer::ready) } fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { @@ -484,9 +508,10 @@ impl LiveScorer for FamilyMux { submission_id: &str, promoted: bool, ) { - self.default - .on_persisted(topic_id, submission_digest, submission_id, promoted) - .await; + if let Some(d) = &self.default { + d.on_persisted(topic_id, submission_digest, submission_id, promoted) + .await; + } if let Some(c) = &self.custom { c.on_persisted(topic_id, submission_digest, submission_id, promoted) .await; @@ -1617,6 +1642,105 @@ mod tests { mux.on_persisted("t", "d", "pf_0", false).await; } + /// A host with a registered custom runner but no Lium harvest: the mux + /// passes the host-wide gate, `custom` routes to the registry, and every + /// `nll` / `throughput` topic refuses at readiness, plan, and score with + /// `LiveHarvestUnavailable` — never the custom scorer, never a sim, no + /// plan to rent under. + #[tokio::test] + async fn custom_only_mux_scores_custom_and_refuses_the_harvest_families() { + let mux = FamilyMux::custom_only(Arc::new(OneRunner)); + mux.ready().expect("no host-wide blocker without a harvest"); + assert_eq!(mux.custom_ids(), vec!["registered_metric".to_owned()]); + mux.ready_for_topic(&custom_topic("registered_metric")) + .expect("registered id"); + assert!(matches!( + mux.ready_for_topic(&custom_topic("unknown_metric")), + Err(EvalError::RunnerUnwired { .. }) + )); + + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let exec = executor(&p); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + for t in [topic(), throughput_topic()] { + assert!( + matches!( + mux.ready_for_topic(&t), + Err(EvalError::LiveHarvestUnavailable) + ), + "{}", + t.id + ); + assert!(matches!( + mux.plan(&p, &t, &exec), + Err(EvalError::LiveHarvestUnavailable) + )); + let plan = executor_plan(&p, Some(&exec), &t, &HarvestOverrides::default()) + .expect("plan resolved outside the mux"); + let err = mux + .score(&p, &t, &offer(), &plan, "d", "a", None, 1, &recs, "c") + .await + .expect_err("no harvest to score on"); + assert!(matches!(err, EvalError::LiveHarvestUnavailable), "{err}"); + assert!(!mux.auto_promote(&t, "d", true, Some(1.0), Some(0.5)).await); + } + mux.on_persisted("t", "d", "pf_0", false).await; + + // The whole live path: the host-wide gate passes, an `nll` run + // refuses before any plan, a registered custom run reaches its runner. + scoring_readiness( + &p, + EvalBackend::Lium, + Some(&mux), + true, + Some(&offer()), + Some(&exec), + Some("test-judge-key"), + ) + .expect("custom-only host is ready host-wide"); + let err = eval_after_freeze( + &p, + &topic(), + &offer(), + Some(&exec), + "digest-a", + "art", + None, + 1, + &recs, + "claim", + EvalBackend::Lium, + Some(&mux), + Some("test-judge-key"), + None, + ) + .await + .expect_err("nll needs the harvest"); + assert!(matches!(err, EvalError::LiveHarvestUnavailable), "{err}"); + let err = eval_after_freeze( + &p, + &custom_topic("registered_metric"), + &offer(), + Some(&exec), + "digest-a", + "art", + Some("https://example.invalid/a.zip"), + 1, + &recs, + "claim", + EvalBackend::Lium, + Some(&mux), + Some("test-judge-key"), + None, + ) + .await + .expect_err("the stub runner refuses after routing"); + assert!( + matches!(err, EvalError::Backend(ref m) if m.contains("registered runner")), + "{err}" + ); + } + fn tight_sealed() -> SealedBaseline { let mut split = BTreeMap::new(); for s in HoldoutSplit::SCORED { diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index d6fb428e8..fbb2c4e97 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -799,7 +799,7 @@ mod tests { use axum::body::Body; use axum::http::Request; use http_body_util::BodyExt; - use proof_eval::{sim_document, BaselineMeasurement, BASELINE_SKILL}; + use proof_eval::{sim_document, BaselineMeasurement, FamilyMux, BASELINE_SKILL}; use proof_task::{ default_adamw, holdout_commitment, inference_config_commitment, synthetic_holdout, Constraints, HoldoutSplit, InferenceConfig, InferenceMode, InferenceOffer, @@ -1986,9 +1986,16 @@ mod tests { /// Live host: the throughput topic scores through the harvest stub, the /// custom topic goes through `scorer` (registered id `topic_minted_metric`). fn app_with_custom(scorer: Arc) -> Router { + app_with_live_scorer(scorer) + } + + /// Live host holding the open, sealed throughput topic `dt-no-ib-v0` and + /// custom topic `custom-topic-v0` (id `topic_minted_metric`, which + /// `live.custom_ids()` must list), scored by `live`. + fn app_with_live_scorer(live: Arc) -> Router { let p = pin(&format!("sha256:{}", "ab".repeat(32))); let store = MemoryStore::new(); - let registered = scorer.custom_ids(); + let registered = live.custom_ids(); for draft in [ unsigned_topic(&[]), unsigned_custom_topic(&[], "topic_minted_metric"), @@ -2008,7 +2015,7 @@ mod tests { store, pin: p, backend: EvalBackend::Lium, - live_scorer: Some(scorer), + live_scorer: Some(live), offer: Some(offer()), executor: executor_slot(Some(executor)), judge_api_key: Some("test-judge-key".into()), @@ -2017,6 +2024,95 @@ mod tests { }) } + /// A host whose topic-VM runner is registered but whose Lium harvest is + /// not wired (`FamilyMux::custom_only`): the host is ready, the custom + /// topic is scorable and scores, and every `nll` / `throughput` topic is + /// open but not scorable — a submit there is a 503 with no row and no + /// scorer call, never an in-process sim. + #[tokio::test] + async fn a_custom_only_host_scores_custom_and_refuses_the_harvest_families() { + let scorer = Arc::new(FamilyStub::win("topic_minted_metric")); + let app = app_with_live_scorer(Arc::new(FamilyMux::custom_only(scorer.clone()))); + let (st, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + let ids = |key: &str| -> Vec { + status[key] + .as_array() + .expect(key) + .iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }; + let mut open = ids("open_topics"); + open.sort(); + assert_eq!(open, ["custom-topic-v0", "dt-no-ib-v0"], "{status}"); + assert_eq!(ids("scorable_topics"), ["custom-topic-v0"], "{status}"); + assert_eq!( + ids("registered_custom"), + ["topic_minted_metric"], + "{status}" + ); + assert_eq!(status["live_harvest_wired"], true, "{status}"); + assert_eq!(status["can_score"], true, "{status}"); + + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("harvest-topic", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!( + body["error"], + EvalError::LiveHarvestUnavailable.to_string(), + "{body}" + ); + assert_eq!(scorer.inner.hits.load(Ordering::SeqCst), 0, "no run"); + let (st, list) = json_req( + app.clone(), + "GET", + "/v1/submissions", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert!( + list["items"].as_array().is_some_and(Vec::is_empty), + "a harvest-family refusal banked a row: {list}" + ); + + let (st, created) = json_req( + app, + "POST", + "/v1/submissions", + submit_body( + "custom-without-lium", + &serde_json::json!({ + "topic_id": "custom-topic-v0", + "artifact_uri": "https://example.invalid/custom-without-lium.zip", + }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "champion", "{created}"); + assert_eq!(scorer.inner.hits.load(Ordering::SeqCst), 1); + let persisted = scorer.persisted.lock().expect("p").clone(); + assert_eq!(persisted.len(), 1, "{persisted:?}"); + assert_eq!(persisted[0].0, "custom-topic-v0"); + } + /// A registered runner whose topic VM is not wired: the topic is open /// but not scorable, and a submit is a 503 with no row. #[tokio::test] From 78c6540ed319c872c6706a2769757220b95a296e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:53:43 +0000 Subject: [PATCH 2/3] docs(proof): custom family is wired from the topic-vm env, not lium State in the operator spec, miner status table, KVM-host runbook, env example, completeness sheet, and both AGENTS contracts that the custom runner registry stands without a Lium harvest (custom scores, nll / throughput 503), what live_harvest_wired now means, and that a placeholder Lium key must not be staged to open custom topics. Co-authored-by: Mathis --- AGENTS.md | 2 +- deploy/AGENTS.md | 5 ++++- deploy/env/proof-challenge.env.example | 17 +++++++++++----- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 27 ++++++++++++++++++++++---- docs/external-miner/proof.md | 4 ++-- docs/runbooks/proof-vm-orchestrator.md | 10 ++++++++++ 7 files changed, 53 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7236cfa94..6c6facdae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). 8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 2800c9745..66b2bf0c9 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -81,7 +81,10 @@ client: set `PROOF_VM_ORCHESTRATOR_URL`, `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` `/etc/proof-vm/token`, mode 0400), `PROOF_RLM_VM_IMAGE_DIGEST`, and `PROOF_VM_RUNNER_CUSTOM_IDS` in `deploy/env/proof-challenge.env`. Unset → unwired (503); token missing, digest unpinned, or agent down → 503, never a -host-local fallback. Kernel / RLM / sister image digests are computed from the +host-local fallback. Those four are the whole custom-family prerequisite: with +them set and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, custom topics +score and `nll` / `throughput` answer 503 — do not stage a placeholder Lium +key to open custom topics. Kernel / RLM / sister image digests are computed from the files the operator stages (`sha256sum`) — never invented, never in git. Procedure and the mandatory submission verification: [`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index ff055f8f5..6a96182d8 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -11,10 +11,13 @@ BASE_NETUID=541 # Sim is CI/local only and is never a fallback. On a live host the service # needs all of: a sha256 eval_image_digest in config/proof-pin.toml, a wired -# harvest (LIUM_API_KEY + LIUM_SSH_PUBLIC_KEY_FILE below), at least one open -# signed topic with a verified holdout, and a sealed baseline. Until it has -# them every submission answers 503. Check GET /v1/status → can_score, -# live_harvest_wired, baseline_sealed. +# scorer for the topic's family — the Lium harvest (LIUM_API_KEY + +# LIUM_SSH_PUBLIC_KEY_FILE below) for nll / throughput, the topic-VM +# orchestrator plus PROOF_VM_RUNNER_CUSTOM_IDS (bottom of this file) for +# custom; each is wired on its own — at least one open signed topic with a +# verified holdout, and a sealed baseline. Until it has them every submission +# answers 503. Check GET /v1/status → can_score, live_harvest_wired, +# scorable_topics, baseline_sealed. PROOF_FORCE_SIM=false # Leftover no-op. Under PROOF_FORCE_SIM a sealed topic already emits @@ -27,7 +30,8 @@ PROOF_SIM_STUB_WIN=false # LIUM_API_KEY= # LIUM_API_BASE_URL= # Without the master public key the pod boots unreachable, so the harvest is -# not wired and submissions refuse. +# not wired and nll / throughput submissions refuse (custom topics still +# score over the topic-VM orchestrator below; Lium is not their prerequisite). # LIUM_SSH_PUBLIC_KEY_FILE=/run/base/lium/lium_ssh_ed25519.pub # Signed topic documents (JSON array). Operator-published; never a holdout @@ -128,6 +132,9 @@ PROOF_SIM_STUB_WIN=false # Custom metric ids the generic VmBackedRunner serves over the orchestrator # (comma-separated). Registration is an operator action: unset = empty # registry = every custom topic 503. Ids are topic data, never a code list. +# With the orchestrator URL + token file + digest above and at least one id +# here, custom topics score with NO Lium harvest (nll / throughput then 503 +# until LIUM_* are set); never stage a placeholder Lium key for this. # PROOF_VM_RUNNER_CUSTOM_IDS= # Owner paid-inference key file probed (presence only) at awaiting_owner_keys # before the baseline run. The MATERIAL is staged into the topic VM by the diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 97f8b705f..e7b39a5e7 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -85,7 +85,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | -| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. | +| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. | | Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | diff --git a/docs/PROOF.md b/docs/PROOF.md index a92683a83..1401a79ed 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -212,9 +212,11 @@ Trust-root keygen is the throwaway owner path in ## HTTP - `GET /health`, `GET /v1/status` — `can_score`, `eval_backend`, `force_sim`, - `live_harvest_wired`, `baseline_sealed`, `open_topics`, `scorable_topics` - (open topics whose scorer is wired on this host; `can_score` is true when - it is non-empty), `registered_custom` (custom ids with a runner), public + `live_harvest_wired` (a live scorer is wired: the Lium harvest for `nll` / + `throughput` and/or the custom-family RLM scorer over the topic-VM + orchestrator), `baseline_sealed`, `open_topics`, `scorable_topics` + (open topics whose family's scorer is wired on this host; `can_score` is + true when it is non-empty), `registered_custom` (custom ids with a runner), public pin `inference` judge defaults (no origin), public `inference_offer` (RLM judge backend), public `eval_executor` (live `1x` executor offer) and pin `executor` ceilings. Never leak origins, keys, or holdout records. @@ -231,7 +233,8 @@ Trust-root keygen is the throwaway owner path in open / unsealed baseline / empty digest / missing or closed RLM judge backend / missing, closed, or non-`1x` executor / agent down / run cut at the proof deadline / no registered or wired runner for the topic's - `custom_id` → **503**. Refusals must **not** persist rows. Scored rows + `custom_id` / `nll` or `throughput` topic on a host with no Lium harvest + → **503**. Refusals must **not** persist rows. Scored rows stamp `executor_offer_id` + `executor_commitment` next to the judge `inference_offer_id` + `config_commitment`. - A pass that the family scorer crowns (custom: green checklist and @@ -478,6 +481,22 @@ whose runner reports its backend unwired) is open but not in Publishing an `open` custom topic without a registered runner is **400**; the same document drafts fine. +The registry is wired from the topic-VM orchestrator env alone, **not** +from the Lium harvest. `proof-challenge` builds the RLM scorer over the +registry whenever the env selects the live `FirecrackerOrchestrator` +(`PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`, https) +and `PROOF_VM_RUNNER_CUSTOM_IDS` registers at least one id; with no Lium +credentials the mux is `FamilyMux::custom_only` — custom topics score over +the topic VMs while every `nll` / `throughput` topic is open but not in +`scorable_topics` and a submit there is **503** (`LiveHarvestUnavailable`, +no row, no rent, never an in-process sim). No placeholder harvest is needed +to open a custom topic. Token file and image digest are still checked per +request (**503** naming the variable). URL unset or refused (plain `http://` +off loopback) keeps `UnwiredVmOrchestrator`, and with no Lium harvest +either the host has no live scorer at all (`live_harvest_wired: false`, +every submission **503**); the unwired stub never carries a mux, and ids +listed over it register nothing. + ### Artefacts and promotion Every scored row leaves `$PROOF_ARTEFACT_ROOT/{topic_id}/{submission_id}.zip` diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 49370023a..2f503e3ce 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -88,10 +88,10 @@ rented. | `inference_offer` | Public RLM **judge** backend (id, kind, mode, model_ref, token caps, commitment, status). Missing/closed/misconfigured → **503**. You do not pass an offer id | | `eval_executor` | Public `1x` **executor**: the Lium machine class your recipe is re-run on (`lium_template_id`, `machine_shape`, `max_proof_deadline_s`, commitment, status). Your recipe must finish inside `max_proof_deadline_s` (≤ pin ceiling 7200 s; a topic may name a shorter one) on **one** GPU — the host never rents more. Missing/closed/any shape but `1x` → **503**. You do not pass or rent it | | `open_topics` empty | No currently `open` signed topic with a sealed baseline → **503** | -| `scorable_topics` | Open topics whose scorer is wired on this host. An open topic **not** listed here (a `custom` topic whose runner is not registered or not wired) answers **503** | +| `scorable_topics` | Open topics whose family's scorer is wired on this host. An open topic **not** listed here (a `custom` topic whose runner is not registered or not wired; an `nll` / `throughput` topic on a host whose Lium harvest is not wired) answers **503** | | `registered_custom` | Custom metric ids with a registered runner. Nothing is compiled in; ids come from signed topics | | `baseline_sealed: false` | An open topic without `script_sha256` + `metrics_commitment` → **503** | -| `live_harvest_wired: false` | Live RLM harvest is not connected → **503** | +| `live_harvest_wired: false` | No live scorer is connected — neither the Lium harvest (`nll` / `throughput`) nor a topic-VM runner (`custom`) → **503** for everything. `true` does not by itself make every open topic scorable: check `scorable_topics` | ## 1. List open topics diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index c48edc8c9..3e88f0bd0 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -113,6 +113,16 @@ mode 0400, uid 65532). Restart `proof-challenge`; its boot log must show registered` line per id. `GET /v1/status` → `registered_custom` lists the ids; an open custom topic with a listed id appears in `scorable_topics`. +The Lium harvest is **not** a prerequisite. With these four variables set +and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, the boot log shows +`live harvest not wired: custom-family topics route to the rlm scorer over +the topic-vm orchestrator`, custom topics open and score, and every `nll` / +`throughput` topic stays out of `scorable_topics` (submit **503**, no row). +Do not stage a placeholder Lium key to open custom topics. If the log shows +`live harvest not wired; every submission will 503` instead, the orchestrator +URL is unset or refused (plain `http://` off loopback) or no id registered — +fix that, not Lium. + The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / `PROOF_RLM_VM_MEM_MIB` exist for a deliberate change only. From 2ce929fbf61ee88dd63f615d2077089b02e61121 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:29:50 +0000 Subject: [PATCH 3/3] fix(proof): keep live_harvest_wired lium-only; report custom family apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 on #246: on a custom-only host the status endpoint treated any live_scorer as a wired Lium harvest and reported live_harvest_wired: true while nll / throughput submissions were refused with 503. - LiveScorer gains two status-only introspection methods (never gates): harvest_wired() — default true (the handle is the harvest), RlmScorer false, FamilyMux answers for its default route only, so a custom_only mux is never harvest-wired; ready_custom_ids() — registered custom ids whose runner reports ready (RlmScorer: local orchestrator bearer file + image pin checks, no request leaves the host). - /v1/status: live_harvest_wired = Lium harvest only. New custom_family_wired (a custom-family scorer with >= 1 registered runner) and custom_ready (subset of registered_custom that can run now). registered_custom / scorable_topics unchanged. The proof-challenge boot log derives its harvest flag from the same method. - ctx status prints custom_family_wired next to live_harvest_wired. - Tests: custom-only host -> live_harvest_wired false, custom_family_wired true, custom_ready = the id (proof-http stub, proof-challenge bin with the real FirecrackerOrchestrator + RlmScorer, rlm_e2e stack now boots FamilyMux::custom_only like a Lium-less host); Lium harvest present -> true as before, with and without a custom route; no scorer -> both false; registered-but-unwired runner -> wired true, custom_ready empty; emptied bearer file drops the id from ready but not from registered. - Docs: PROOF.md, external-miner status table, KVM-host runbook, env example, operator script hint, COMPLETENESS, both AGENTS contracts. Co-authored-by: Mathis --- AGENTS.md | 2 +- bins/ctx/src/catalog.rs | 1 + bins/proof-challenge/src/main.rs | 40 ++++++++++------ crates/proof-eval/src/lib.rs | 53 ++++++++++++++++++++ crates/proof-http/src/lib.rs | 61 +++++++++++++++++++++--- crates/proof-rlm-scorer/src/scorer.rs | 20 ++++++++ crates/proof-rlm-scorer/tests/rlm_e2e.rs | 51 ++++++++------------ deploy/AGENTS.md | 4 +- deploy/env/proof-challenge.env.example | 5 +- deploy/scripts/proof-operator-path.sh | 2 +- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 26 ++++++---- docs/external-miner/proof.md | 11 +++-- docs/runbooks/proof-vm-orchestrator.md | 12 +++-- 14 files changed, 218 insertions(+), 72 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6c6facdae..b7aa67e94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). 8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/bins/ctx/src/catalog.rs b/bins/ctx/src/catalog.rs index 2e4d89221..7c5a17d12 100644 --- a/bins/ctx/src/catalog.rs +++ b/bins/ctx/src/catalog.rs @@ -140,6 +140,7 @@ fn print_challenge_status(c: &Challenge, body: &Value) { "scoring_backend", "force_sim", "live_harvest_wired", + "custom_family_wired", "baseline_sealed", "eval_image_digest", ] { diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index e36185ad4..82e58c3eb 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -183,14 +183,8 @@ fn run(cli: &Cli) -> Result<(), String> { cli.proxy_model_dir.clone(), cli.holdout_store.clone(), ); - let harvest_wired = harvest.is_some(); let live_scorer = live_scorer(backend, harvest, rlm_store, &cli.artefact_root); - log_live_wiring( - backend, - harvest_wired, - live_scorer.as_deref(), - &cli.artefact_root, - ); + log_live_wiring(backend, live_scorer.as_deref(), &cli.artefact_root); let store = MemoryStore::new(); let registered = registered_custom(live_scorer.as_deref()); @@ -271,13 +265,10 @@ fn build_live_scorer( /// Boot log for what [`live_scorer`] wired, and what answers 503 because of /// what is missing. Nothing here is a boot error: a live host refuses until -/// the operator wires the piece it names. -fn log_live_wiring( - backend: EvalBackend, - harvest_wired: bool, - live: Option<&dyn LiveScorer>, - artefact_root: &Path, -) { +/// the operator wires the piece it names. The harvest flag is the same +/// Lium-only answer `/v1/status` gives as `live_harvest_wired`. +fn log_live_wiring(backend: EvalBackend, live: Option<&dyn LiveScorer>, artefact_root: &Path) { + let harvest_wired = live.is_some_and(LiveScorer::harvest_wired); match backend { EvalBackend::Lium if harvest_wired => { tracing::info!("live harvest wired: digest-pinned proof-eval image on Lium"); @@ -790,7 +781,9 @@ mod tests { &root, ) .expect("a wired harvest is the live scorer"); + assert!(mux.harvest_wired(), "the Lium harvest is the default route"); assert!(registered_custom(Some(mux.as_ref())).is_empty()); + assert!(mux.ready_custom_ids().is_empty()); for id in ["any_metric", "another_metric"] { let mut custom = TopicDocument::default(); custom.metric.family = proof_task::MetricFamily::Custom; @@ -1016,10 +1009,19 @@ mod tests { &root, ) .expect("the custom family is wired from the topic-vm env alone"); + assert!( + !mux.harvest_wired(), + "live_harvest_wired is Lium-only; a custom-only host reads false" + ); assert_eq!( registered_custom(Some(mux.as_ref())), vec!["metric_a".to_owned()] ); + assert_eq!( + mux.ready_custom_ids(), + vec!["metric_a".to_owned()], + "custom readiness is reported on its own" + ); mux.ready().expect("no host-wide blocker"); let mut custom = TopicDocument::default(); @@ -1055,6 +1057,16 @@ mod tests { ) .expect("ready host-wide without Lium"); + // Registration and readiness are separate: an emptied bearer file + // keeps the id registered but no longer ready (per-request 503). + std::fs::write(&token, "\n").expect("empty token"); + assert_eq!( + registered_custom(Some(mux.as_ref())), + vec!["metric_a".to_owned()] + ); + assert!(mux.ready_custom_ids().is_empty()); + assert!(!mux.harvest_wired()); + // Sim scores in-process and never wires anything live. assert!(live_scorer( EvalBackend::Sim, diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 0726be70c..204eeb2f2 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -331,6 +331,25 @@ pub trait LiveScorer: Send + Sync { Vec::new() } + /// Custom metric ids whose registered runner could run right now (its + /// topic-VM orchestrator wired, image pinned): a subset of + /// [`Self::custom_ids`]. Default: none. Status reporting only, never a + /// gate — the submit path asks [`Self::ready_for_topic`]. + fn ready_custom_ids(&self) -> Vec { + Vec::new() + } + + /// Whether the digest-pinned Lium harvest — the scorer of the `nll` / + /// `throughput` families — is wired behind this handle. Default: this + /// scorer *is* that harvest. A custom-family scorer answers `false`; a + /// mux answers for its default route. Status reporting only, never a + /// gate: `live_harvest_wired` on `/v1/status` is this and nothing else, + /// so a host that scores only its custom family never reads as + /// harvest-ready. + fn harvest_wired(&self) -> bool { + true + } + /// Whether the run for `submission_digest` on `topic` should be crowned /// champion automatically. /// @@ -482,6 +501,19 @@ impl LiveScorer for FamilyMux { .map_or_else(Vec::new, LiveScorer::custom_ids) } + fn ready_custom_ids(&self) -> Vec { + self.custom + .as_deref() + .map_or_else(Vec::new, LiveScorer::ready_custom_ids) + } + + /// Only the default route is a harvest: a custom-only mux is not one. + fn harvest_wired(&self) -> bool { + self.default + .as_deref() + .is_some_and(LiveScorer::harvest_wired) + } + async fn auto_promote( &self, topic: &TopicDocument, @@ -1544,6 +1576,15 @@ mod tests { vec!["registered_metric".into()] } + fn ready_custom_ids(&self) -> Vec { + self.custom_ids() + } + + /// A custom-family scorer is never the harvest. + fn harvest_wired(&self) -> bool { + false + } + async fn auto_promote( &self, _t: &TopicDocument, @@ -1569,6 +1610,8 @@ mod tests { bare.ready_for_topic(&topic()) .expect("nll routes to the harvest"); assert!(bare.custom_ids().is_empty()); + assert!(bare.ready_custom_ids().is_empty()); + assert!(bare.harvest_wired(), "the default route is the harvest"); assert!(matches!( bare.ready_for_topic(&custom_topic("anything")), Err(EvalError::RunnerUnwired { .. }) @@ -1577,6 +1620,11 @@ mod tests { let mux = FamilyMux::new(Arc::new(Harvest { reproduced: true })) .with_custom_family(Arc::new(OneRunner)); assert_eq!(mux.custom_ids(), vec!["registered_metric".to_owned()]); + assert_eq!(mux.ready_custom_ids(), vec!["registered_metric".to_owned()]); + assert!( + mux.harvest_wired(), + "adding a custom route keeps the harvest" + ); assert_eq!( registered_custom(Some(&mux)), vec!["registered_metric".to_owned()] @@ -1651,7 +1699,12 @@ mod tests { async fn custom_only_mux_scores_custom_and_refuses_the_harvest_families() { let mux = FamilyMux::custom_only(Arc::new(OneRunner)); mux.ready().expect("no host-wide blocker without a harvest"); + assert!( + !mux.harvest_wired(), + "a custom-only host must never read as harvest-wired" + ); assert_eq!(mux.custom_ids(), vec!["registered_metric".to_owned()]); + assert_eq!(mux.ready_custom_ids(), vec!["registered_metric".to_owned()]); mux.ready_for_topic(&custom_topic("registered_metric")) .expect("registered id"); assert!(matches!( diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index fbb2c4e97..24242d2d9 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -68,8 +68,10 @@ pub struct AppState { pub pin: ProofPin, /// Backend that is allowed to produce scores on this host. pub backend: EvalBackend, - /// Harvest handle for the digest-pinned eval image. `None` on a live host - /// means nothing can score, so submissions refuse. + /// Live scorer by metric family: the digest-pinned Lium harvest for + /// `nll` / `throughput` and/or the custom-family RLM scorer (each wired + /// on its own; a family whose route is missing refuses per topic). `None` + /// on a live host means nothing can score, so submissions refuse. pub live_scorer: Option>, /// Live RLM judge backend (operator state). Missing/closed → can_score false. pub offer: Option, @@ -121,6 +123,20 @@ impl AppState { registered_custom(self.live()) } + /// Whether the digest-pinned Lium harvest — the `nll` / `throughput` + /// scorer — is wired. Lium only: a custom-only host answers `false`. + fn live_harvest_wired(&self) -> bool { + self.live().is_some_and(LiveScorer::harvest_wired) + } + + /// Registered custom ids whose runner could run right now (topic-VM + /// orchestrator wired, image pinned). Independent of the harvest and of + /// which topics are open. + fn custom_ready(&self) -> Vec { + self.live() + .map_or_else(Vec::new, LiveScorer::ready_custom_ids) + } + /// Whether the host-wide gates (digest, harvest, judge offer, executor, /// key, any open sealed topic) pass. fn host_ready(&self) -> bool { @@ -195,6 +211,9 @@ async fn health() -> impl IntoResponse { async fn status(State(st): State) -> impl IntoResponse { let open = st.store.open_ids(st.epoch).unwrap_or_default(); let baseline_sealed = st.store.any_open_scorable(st.epoch).unwrap_or(false); + // Family wiring is reported per family, never conflated: the harvest + // flag is Lium-only, the custom family has its own fields. + let registered_custom = st.registered_custom(); Json(serde_json::json!({ "challenge_id": CHALLENGE_ID, "scoring_version": SCORING_VERSION, @@ -215,11 +234,13 @@ async fn status(State(st): State) -> impl IntoResponse { "force_sim": force_sim(), "sim_stub_win": st.backend == EvalBackend::Sim, "can_score": st.can_score(), - "live_harvest_wired": st.live_scorer.is_some(), + "live_harvest_wired": st.live_harvest_wired(), + "custom_family_wired": !registered_custom.is_empty(), "baseline_sealed": baseline_sealed, "open_topics": open, "scorable_topics": st.scorable_topics(), - "registered_custom": st.registered_custom(), + "registered_custom": registered_custom, + "custom_ready": st.custom_ready(), "epoch": st.epoch, })) } @@ -1684,6 +1705,11 @@ mod tests { assert_eq!(st, StatusCode::OK); assert_eq!(body["can_score"], false, "{body}"); assert_eq!(body["baseline_sealed"], false, "{body}"); + // No live scorer at all: neither family is wired. + assert_eq!(body["live_harvest_wired"], false, "{body}"); + assert_eq!(body["custom_family_wired"], false, "{body}"); + assert_eq!(body["registered_custom"], serde_json::json!([]), "{body}"); + assert_eq!(body["custom_ready"], serde_json::json!([]), "{body}"); let (st, body) = json_req( app_full( @@ -1702,7 +1728,10 @@ mod tests { ) .await; assert_eq!(st, StatusCode::OK); + // A harvest with no custom route: the harvest flag alone is true. assert_eq!(body["live_harvest_wired"], true); + assert_eq!(body["custom_family_wired"], false, "{body}"); + assert_eq!(body["custom_ready"], serde_json::json!([]), "{body}"); assert_eq!(body["can_score"], false, "{body}"); } @@ -1934,6 +1963,14 @@ mod tests { vec![self.custom_id.clone()] } + fn ready_custom_ids(&self) -> Vec { + if self.wired { + self.custom_ids() + } else { + Vec::new() + } + } + async fn auto_promote( &self, _topic: &TopicDocument, @@ -2028,7 +2065,9 @@ mod tests { /// not wired (`FamilyMux::custom_only`): the host is ready, the custom /// topic is scorable and scores, and every `nll` / `throughput` topic is /// open but not scorable — a submit there is a 503 with no row and no - /// scorer call, never an in-process sim. + /// scorer call, never an in-process sim. `/v1/status` says so per + /// family: `live_harvest_wired` stays **false** (Lium only) while the + /// custom family reports wired and ready on its own fields. #[tokio::test] async fn a_custom_only_host_scores_custom_and_refuses_the_harvest_families() { let scorer = Arc::new(FamilyStub::win("topic_minted_metric")); @@ -2059,7 +2098,12 @@ mod tests { ["topic_minted_metric"], "{status}" ); - assert_eq!(status["live_harvest_wired"], true, "{status}"); + assert_eq!( + status["live_harvest_wired"], false, + "no Lium harvest: the harvest flag must not follow the custom mux: {status}" + ); + assert_eq!(status["custom_family_wired"], true, "{status}"); + assert_eq!(ids("custom_ready"), ["topic_minted_metric"], "{status}"); assert_eq!(status["can_score"], true, "{status}"); let (st, body) = json_req( @@ -2146,6 +2190,11 @@ mod tests { ["topic_minted_metric"], "{status}" ); + // Harvest wired, custom registered but its VM backend not: each + // family reports its own state. + assert_eq!(status["live_harvest_wired"], true, "{status}"); + assert_eq!(status["custom_family_wired"], true, "{status}"); + assert!(ids("custom_ready").is_empty(), "{status}"); assert_eq!(status["can_score"], true, "{status}"); let (st, body) = json_req( diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index ec0a6ece7..fc9442ad7 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -759,6 +759,26 @@ impl LiveScorer for RlmScorer { self.registry.ids() } + /// Registered ids whose runner reports ready (topic-VM orchestrator + /// bearer file present, image pinned). Local checks only; no request + /// leaves the host. + fn ready_custom_ids(&self) -> Vec { + self.registry + .ids() + .into_iter() + .filter(|id| { + self.registry + .resolve(id) + .is_ok_and(|runner| runner.ready().is_ok()) + }) + .collect() + } + + /// The RLM scorer is the custom family, never the Lium harvest. + fn harvest_wired(&self) -> bool { + false + } + /// Decided under the topic lease this run has held since `score` /// returned, against the harder of the caller's bar and the store's /// current best: no other run of this topic can be between score and diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 8c3472cc5..df653c85d 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -1,10 +1,12 @@ //! Full control-plane path for the generic RLM engine, without any VM, //! Lium, or paid inference: `POST /v1/submissions` on an open custom-family -//! topic through `FamilyMux` → `RlmScorer` → registry → generic -//! `VmBackedRunner` → fake orchestrator, with the memory RLM store and a -//! temp artefact root. +//! topic through `FamilyMux::custom_only` (the shape a host with no Lium +//! harvest boots) → `RlmScorer` → registry → generic `VmBackedRunner` → +//! fake orchestrator, with the memory RLM store and a temp artefact root. //! -//! Covers: an unregistered `custom_id` is a 503 with no row; a custom +//! Covers: `/v1/status` reports the custom family wired and ready while +//! `live_harvest_wired` stays false; an unregistered `custom_id` is a 503 +//! with no row; a custom //! submission without an artefact locator is a 400 with no row; a green //! checklist scores and is crowned against the sealed value, with the //! miner's artefact locator and declaration reaching the runner and the @@ -53,34 +55,12 @@ use proof_rlm_store::{MemoryRlmStore, PromotionRow, RlmStore}; use proof_score::SealedBaseline; use proof_store::MemoryStore; use proof_task::{ - holdout_commitment, synthetic_holdout, HoldoutRecord, HoldoutSplit, InferenceOffer, ProofPin, - TopicDocument, TopicError, TopicStatus, STRATUM_SIZE, + holdout_commitment, synthetic_holdout, HoldoutSplit, ProofPin, TopicDocument, TopicError, + TopicStatus, STRATUM_SIZE, }; use sha2::{Digest, Sha256}; use tower::ServiceExt; -/// Default route: a harvest that is ready and never asked to score here. -struct IdleHarvest; - -#[async_trait::async_trait] -impl LiveScorer for IdleHarvest { - async fn score( - &self, - _pin: &ProofPin, - _topic: &TopicDocument, - _offer: &InferenceOffer, - _plan: &ExecutorPlan, - _frozen: &str, - _artifact: &str, - _artifact_uri: Option<&str>, - _declared_flops: u64, - _holdout: &[HoldoutRecord], - _claim: &str, - ) -> Result { - Err(EvalError::Backend("idle harvest".into())) - } -} - /// Open `1x` executor on the digest-scoped template of `pin` (host state the /// live path requires; the RLM path only records its plan commitment). fn test_executor(pin: &ProofPin) -> EvalExecutorOffer { @@ -166,7 +146,8 @@ fn stack(register: bool) -> Stack { let root = tmp_root("stack"); let scorer = RlmScorer::new(Arc::new(registry), rlm_store.clone()) .with_artefacts(Some(ArtefactStore::new(&root))); - let mux = FamilyMux::new(Arc::new(IdleHarvest)).with_custom_family(Arc::new(scorer)); + // No Lium harvest on this host: the custom family stands on its own. + let mux = FamilyMux::custom_only(Arc::new(scorer)); let executor = test_executor(&pin); let app = proof_router(AppState { store, @@ -271,6 +252,9 @@ async fn an_unregistered_custom_id_is_503_with_no_row() { status["registered_custom"].as_array().unwrap().is_empty(), "{status}" ); + assert_eq!(status["live_harvest_wired"], false, "{status}"); + assert_eq!(status["custom_family_wired"], false, "{status}"); + assert!(status["custom_ready"].as_array().unwrap().is_empty()); assert_eq!(status["can_score"], false, "{status}"); let (st, body) = json_req( @@ -299,7 +283,8 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { } = stack(true); let tid = topic.id.clone(); - // 0. Open, registered, scorable. + // 0. Open, registered, scorable — and reported per family: the custom + // family is wired and ready, the Lium harvest is not. let (st, status) = json_req(app.clone(), "GET", "/v1/status", serde_json::json!({})).await; assert_eq!(st, StatusCode::OK); assert_eq!(status["can_score"], true, "{status}"); @@ -308,6 +293,12 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { status["registered_custom"][0], topic.metric.custom_id, "{status}" ); + assert_eq!(status["live_harvest_wired"], false, "{status}"); + assert_eq!(status["custom_family_wired"], true, "{status}"); + assert_eq!( + status["custom_ready"][0], topic.metric.custom_id, + "{status}" + ); let (_, topics) = json_req( app.clone(), "GET", diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 66b2bf0c9..46efdf564 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -84,7 +84,9 @@ unwired (503); token missing, digest unpinned, or agent down → 503, never a host-local fallback. Those four are the whole custom-family prerequisite: with them set and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, custom topics score and `nll` / `throughput` answer 503 — do not stage a placeholder Lium -key to open custom topics. Kernel / RLM / sister image digests are computed from the +key to open custom topics. On `/v1/status` that host reads +`live_harvest_wired: false` (Lium only) with `custom_family_wired: true` and +its ids in `registered_custom` / `custom_ready`. Kernel / RLM / sister image digests are computed from the files the operator stages (`sha256sum`) — never invented, never in git. Procedure and the mandatory submission verification: [`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 6a96182d8..77c893298 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -16,8 +16,9 @@ BASE_NETUID=541 # orchestrator plus PROOF_VM_RUNNER_CUSTOM_IDS (bottom of this file) for # custom; each is wired on its own — at least one open signed topic with a # verified holdout, and a sealed baseline. Until it has them every submission -# answers 503. Check GET /v1/status → can_score, live_harvest_wired, -# scorable_topics, baseline_sealed. +# answers 503. Check GET /v1/status → can_score, live_harvest_wired (Lium +# harvest only), custom_family_wired / registered_custom / custom_ready (the +# custom family, reported apart), scorable_topics, baseline_sealed. PROOF_FORCE_SIM=false # Leftover no-op. Under PROOF_FORCE_SIM a sealed topic already emits diff --git a/deploy/scripts/proof-operator-path.sh b/deploy/scripts/proof-operator-path.sh index 4f3cc5c4c..5ae88ce4a 100755 --- a/deploy/scripts/proof-operator-path.sh +++ b/deploy/scripts/proof-operator-path.sh @@ -99,7 +99,7 @@ cargo run -p xtask -- proof-executor-offer \\ # PROOF_EVAL_EXECUTOR_OFFER_FILE=${SECRETS}/eval_executor_offer.json # LIUM_API_KEY=… LIUM_SSH_PUBLIC_KEY_FILE=… # Restart proof-challenge, then: -# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,baseline_sealed,eval_executor}' +# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,custom_family_wired,custom_ready,baseline_sealed,eval_executor}' # curl -sS "\$PROOF_BASE/v1/proof/executor" | jq '{ready,reason}' # can_score is true only with: real digest + harvest wired + open topic + diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index e7b39a5e7..c13eb7657 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -85,7 +85,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | -| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. | +| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | | Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | diff --git a/docs/PROOF.md b/docs/PROOF.md index 1401a79ed..c87ab33a4 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -212,11 +212,15 @@ Trust-root keygen is the throwaway owner path in ## HTTP - `GET /health`, `GET /v1/status` — `can_score`, `eval_backend`, `force_sim`, - `live_harvest_wired` (a live scorer is wired: the Lium harvest for `nll` / - `throughput` and/or the custom-family RLM scorer over the topic-VM - orchestrator), `baseline_sealed`, `open_topics`, `scorable_topics` - (open topics whose family's scorer is wired on this host; `can_score` is - true when it is non-empty), `registered_custom` (custom ids with a runner), public + `live_harvest_wired` (**Lium harvest only** — the `nll` / `throughput` + scorer; never true because a custom-family scorer is present), + `custom_family_wired` (a custom-family scorer with ≥1 registered runner is + on this host, independent of Lium), `baseline_sealed`, `open_topics`, + `scorable_topics` (open topics whose family's scorer is wired on this + host; `can_score` is true when it is non-empty), `registered_custom` + (custom ids with a runner), `custom_ready` (registered ids whose runner + could run right now: topic-VM orchestrator bearer file present, image + pinned — independent of which topics are open), public pin `inference` judge defaults (no origin), public `inference_offer` (RLM judge backend), public `eval_executor` (live `1x` executor offer) and pin `executor` ceilings. Never leak origins, keys, or holdout records. @@ -490,10 +494,14 @@ credentials the mux is `FamilyMux::custom_only` — custom topics score over the topic VMs while every `nll` / `throughput` topic is open but not in `scorable_topics` and a submit there is **503** (`LiveHarvestUnavailable`, no row, no rent, never an in-process sim). No placeholder harvest is needed -to open a custom topic. Token file and image digest are still checked per -request (**503** naming the variable). URL unset or refused (plain `http://` -off loopback) keeps `UnwiredVmOrchestrator`, and with no Lium harvest -either the host has no live scorer at all (`live_harvest_wired: false`, +to open a custom topic. `GET /v1/status` reports the two families apart: +such a host shows `live_harvest_wired: false` (that flag is the Lium harvest +and nothing else) next to `custom_family_wired: true`, `registered_custom`, +and `custom_ready`. Token file and image digest are still checked per +request (**503** naming the variable; the id then drops out of +`custom_ready` while staying in `registered_custom`). URL unset or refused +(plain `http://` off loopback) keeps `UnwiredVmOrchestrator`, and with no +Lium harvest either the host has no live scorer at all (both flags false, every submission **503**); the unwired stub never carries a mux, and ids listed over it register nothing. diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 2f503e3ce..8bd2d1ca7 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -47,7 +47,8 @@ its static checks and model measurements are only part of that design. Holdout records are not included in public topic responses. `GET /challenge/proof/v1/status` shows `can_score`, `eval_backend`, -`force_sim`, `live_harvest_wired`, `baseline_sealed`, public pin `inference` +`force_sim`, `live_harvest_wired`, `custom_family_wired`, `baseline_sealed`, +public pin `inference` judge defaults (provider, model, mode, token caps — never the origin), the public RLM judge `inference_offer` (id, kind, mode, model_ref, token caps, commitment, status), and the public `eval_executor` — the `1x` Lium machine @@ -72,7 +73,9 @@ curl -sS https://network.cortex.foundation/challenge/proof/v1/status ``` `GET /challenge/proof/v1/status` shows `can_score`, `eval_backend`, -`force_sim`, `live_harvest_wired`, `baseline_sealed`, `eval_image_digest`, +`force_sim`, `live_harvest_wired` (Lium harvest, `nll` / `throughput`), +`custom_family_wired` + `registered_custom` + `custom_ready` (the `custom` +family, reported apart), `baseline_sealed`, `eval_image_digest`, public pin `inference` (no origin), public RLM judge `inference_offer`, public `eval_executor` (plus the pin `executor` ceilings), and `open_topics`. It never leaks holdout records, teacher hosts, origins, or @@ -90,8 +93,10 @@ rented. | `open_topics` empty | No currently `open` signed topic with a sealed baseline → **503** | | `scorable_topics` | Open topics whose family's scorer is wired on this host. An open topic **not** listed here (a `custom` topic whose runner is not registered or not wired; an `nll` / `throughput` topic on a host whose Lium harvest is not wired) answers **503** | | `registered_custom` | Custom metric ids with a registered runner. Nothing is compiled in; ids come from signed topics | +| `custom_ready` | The subset of `registered_custom` whose runner can run right now (topic-VM orchestrator reachable by config, image pinned). A registered id missing here → its topics answer **503** | | `baseline_sealed: false` | An open topic without `script_sha256` + `metrics_commitment` → **503** | -| `live_harvest_wired: false` | No live scorer is connected — neither the Lium harvest (`nll` / `throughput`) nor a topic-VM runner (`custom`) → **503** for everything. `true` does not by itself make every open topic scorable: check `scorable_topics` | +| `live_harvest_wired: false` | The Lium harvest — the `nll` / `throughput` scorer — is not connected → **503** on those families. **Lium only**: it says nothing about `custom` topics | +| `custom_family_wired: false` | No custom-family runner is registered on this host → **503** on `custom` topics. Independent of `live_harvest_wired`; both `false` → **503** for everything | ## 1. List open topics diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 3e88f0bd0..5f7892241 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -118,10 +118,14 @@ and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, the boot log shows `live harvest not wired: custom-family topics route to the rlm scorer over the topic-vm orchestrator`, custom topics open and score, and every `nll` / `throughput` topic stays out of `scorable_topics` (submit **503**, no row). -Do not stage a placeholder Lium key to open custom topics. If the log shows -`live harvest not wired; every submission will 503` instead, the orchestrator -URL is unset or refused (plain `http://` off loopback) or no id registered — -fix that, not Lium. +`GET /v1/status` then reads `live_harvest_wired: false` (that flag is Lium +only — expected here, not a fault), `custom_family_wired: true`, +`registered_custom` = your ids, and `custom_ready` = the ids whose runner can +run now; an id in `registered_custom` but not in `custom_ready` means the +bearer file or image pin is missing on this host. Do not stage a placeholder +Lium key to open custom topics. If the log shows `live harvest not wired; +every submission will 503` instead, the orchestrator URL is unset or refused +(plain `http://` off loopback) or no id registered — fix that, not Lium. The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / `PROOF_RLM_VM_MEM_MIB` exist for a deliberate change only.