From 2a62f30f4afa5b195ad33aff437bfcf90de4ea41 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:09:03 +0000 Subject: [PATCH 01/14] feat(proof): generic topic bindings for dynamic custom metrics Proof topics are the challenge; the code is not. The signed topic schema gains only generic, shape-checked bindings whose values come from the document: - constraints.{firecracker_required, model_pin (vendor/model[:tag]), task_slice (opaque), params (<=32 opaque slug -> value)} - checklist: [{id, text}] anti-cheat rule vector (<=64, unique slug ids) - eval_executor.{require_offer_commitment (64 hex), max_proof_deadline_s} mirroring the inference judge pattern; the deadline is tighten-only against a new pin knob max_proof_deadline_s (crate lock 21600 s) - metric.custom_id is topic data: any [a-z0-9][a-z0-9_-]{1,63} drafts, and a custom topic may open only when a runner is registered under its id on the host (validate takes the registered set). The compiled-in custom id list is gone. proof-canon is a new tiny crate holding canonical JSON and the id shapes shared by every Proof crate (moved out of proof-task, which sits near the LOC cap). proof-score exposes relative_win / novelty_bar for reuse. Signed topics need re-signing (the signing payload carries the new default fields), as with the inference addition. --- Cargo.lock | 9 + crates/proof-canon/Cargo.toml | 16 ++ .../src/canonical.rs | 0 crates/proof-canon/src/lib.rs | 266 ++++++++++++++++++ crates/proof-harvest/src/lib.rs | 5 +- crates/proof-score/src/lib.rs | 27 +- crates/proof-score/src/payout.rs | 6 +- crates/proof-task/Cargo.toml | 1 + crates/proof-task/src/lib.rs | 41 +-- crates/proof-task/src/topic.rs | 246 ++++++++++++---- 10 files changed, 520 insertions(+), 97 deletions(-) create mode 100644 crates/proof-canon/Cargo.toml rename crates/{proof-task => proof-canon}/src/canonical.rs (100%) create mode 100644 crates/proof-canon/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 802848b56..54a66e386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3417,6 +3417,14 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proof-canon" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "proof-challenge" version = "0.1.0" @@ -3563,6 +3571,7 @@ version = "0.1.0" dependencies = [ "crypto", "hex", + "proof-canon", "proof-holdout", "serde", "serde_json", diff --git a/crates/proof-canon/Cargo.toml b/crates/proof-canon/Cargo.toml new file mode 100644 index 000000000..5af457675 --- /dev/null +++ b/crates/proof-canon/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "proof-canon" +description = "Canonical JSON and identifier shapes shared by the Proof crates (no challenge content)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/proof-task/src/canonical.rs b/crates/proof-canon/src/canonical.rs similarity index 100% rename from crates/proof-task/src/canonical.rs rename to crates/proof-canon/src/canonical.rs diff --git a/crates/proof-canon/src/lib.rs b/crates/proof-canon/src/lib.rs new file mode 100644 index 000000000..e8ed86976 --- /dev/null +++ b/crates/proof-canon/src/lib.rs @@ -0,0 +1,266 @@ +//! Canonical JSON, identifier shapes, and the generic document shapes shared +//! by the Proof crates: the `{id, text}` anti-cheat rule and the topic +//! `constraints` block. +//! +//! Nothing here names a challenge, a metric, a model, or a repository. These +//! are the byte-level rules every Proof document is checked against — +//! canonical signing form, hex digests, http origins, the slug shapes a +//! signed topic may mint for its own ids, and the shape (never the values) +//! of the bindings a topic carries — so the signer, the verifier, the store, +//! the harvest, and the RLM engine all agree on them. + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown, clippy::must_use_candidate)] + +mod canonical; + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +pub use canonical::canonical_json; + +/// Most anti-cheat rules one topic may carry. +pub const MAX_CHECKLIST_RULES: usize = 64; + +/// Longest rule text. +pub const MAX_RULE_TEXT_LEN: usize = 2_048; + +/// Most opaque constraint params one topic may carry. +pub const MAX_CONSTRAINT_PARAMS: usize = 32; + +/// Why a shared shape is malformed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShapeError { + /// Offending field (`constraints.model_pin`, `checklist[id]`, …). + pub field: String, + /// What is wrong. + pub why: &'static str, +} + +/// Machine-checkable constraints the eval image / topic runner enforces. +/// +/// `deny_unknown_fields` is the point: a constraint this control plane does +/// not understand is a constraint nothing can be trusted to enforce, so an +/// unknown key rejects the topic at publish instead of being ignored. The +/// knobs are generic policy; their **values** come from the signed document, +/// never from a catalog. Knobs added after the first signed topics are +/// omitted from the signed payload when unset, so those documents keep +/// verifying (same rule as `eval_executor`). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +#[allow(clippy::struct_excessive_bools)] +pub struct Constraints { + /// No `InfiniBand` fabric. + pub no_infiniband: bool, + /// No NVLink between ranks. + pub no_nvlink: bool, + /// No NCCL all-reduce over a fast fabric. + pub no_nccl_fast_fabric: bool, + /// Inter-node (or emulated inter-rank) bandwidth cap in Gbit/s. + pub max_inter_node_gbps: Option, + /// Sandbox policy: miner code runs only inside a Firecracker guest under + /// the topic's isolated VM, never on the control-plane host. + #[serde(skip_serializing_if = "<&bool as std::ops::Not>::not")] + pub firecracker_required: bool, + /// Provider model id (`vendor/model[:tag]`) every paid inference call + /// made by the runner or the miner harness must name. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pin: Option, + /// Opaque task-slice label the runner interprets (the control plane does not). + #[serde(skip_serializing_if = "Option::is_none")] + pub task_slice: Option, + /// Opaque runner params (bounded). Keys and values are topic data. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub params: BTreeMap, +} + +impl Constraints { + /// Shape check of the generic knobs (values are never interpreted). + /// + /// # Errors + /// + /// [`ShapeError`] naming the first malformed knob. + pub fn validate_shape(&self) -> Result<(), ShapeError> { + let bad = |field: &str, why| { + Err(ShapeError { + field: format!("constraints.{field}"), + why, + }) + }; + if !self.model_pin.as_deref().is_none_or(is_model_pin) { + return bad("model_pin", "vendor/model[:tag]"); + } + if !self.task_slice.as_deref().is_none_or(is_opaque_param) { + return bad("task_slice", "single printable line, <=256 chars"); + } + if self.params.len() > MAX_CONSTRAINT_PARAMS + || self + .params + .iter() + .any(|(k, v)| !is_custom_id(k) || !is_opaque_param(v)) + { + return bad("params", "<=32 slug keys with printable values"); + } + Ok(()) + } +} + +/// One anti-cheat rule the RLM ticks before any paid inference spend. +/// +/// Rules are a **vector carried by the signed topic** (and re-versioned in +/// the store when the topic's RLM rewrites them). This crate knows the shape +/// only; it never ships a rule list. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChecklistRule { + /// Rule id (`[a-z0-9][a-z0-9_-]{1,63}`), unique within the vector. + pub id: String, + /// What an inspector must show for the rule to pass (English). + pub text: String, +} + +/// Shape check for a rule vector: bounded, unique slug ids, bounded text. +/// +/// # Errors +/// +/// [`ShapeError`] naming the first offending rule (`checklist[id]`). +pub fn validate_rules(rules: &[ChecklistRule]) -> Result<(), ShapeError> { + let bad = |id: &str, why| ShapeError { + field: format!("checklist[{id}]"), + why, + }; + if rules.len() > MAX_CHECKLIST_RULES { + return Err(bad("", "too many rules")); + } + for (i, r) in rules.iter().enumerate() { + let text = r.text.trim(); + if !is_custom_id(&r.id) { + return Err(bad(&r.id, "id must match [a-z0-9][a-z0-9_-]{1,63}")); + } + if rules[..i].iter().any(|prev| prev.id == r.id) { + return Err(bad(&r.id, "duplicate id")); + } + if text.is_empty() || text.chars().count() > MAX_RULE_TEXT_LEN { + return Err(bad(&r.id, "text must be 1..=2048 chars")); + } + } + Ok(()) +} + +/// 64 hex chars (a sha256 digest or a 32-byte key), surrounding whitespace ignored. +pub fn is_hex64(s: &str) -> bool { + let t = s.trim(); + t.len() == 64 && t.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// `http://` / `https://` origin with no whitespace. +pub fn is_http_origin(url: &str) -> bool { + let u = url.trim(); + (u.starts_with("http://") || u.starts_with("https://")) + && u.len() >= 8 + && !u.contains(['\n', ' ']) +} + +fn ident(id: &str, max: usize, underscore: bool) -> bool { + let b = id.as_bytes(); + (2..=max).contains(&b.len()) + && (b[0].is_ascii_lowercase() || b[0].is_ascii_digit()) + && b.iter().all(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-' || (underscore && *c == b'_') + }) +} + +/// Topic / offer id: `[a-z0-9][a-z0-9-]{1,62}`. +pub fn is_slug(id: &str) -> bool { + ident(id, 63, false) +} + +/// Identifier a topic may mint for a custom metric or a checklist rule: +/// `[a-z0-9][a-z0-9_-]{1,63}`. Values come from the signed document, never +/// from a list compiled into any crate. +pub fn is_custom_id(id: &str) -> bool { + ident(id, 64, true) +} + +fn is_segment(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'-' | b'.')) +} + +/// `vendor/model` provider model id, optional `:tag`. Shape only — the model +/// itself is topic data, never a default anywhere in this repository. +pub fn is_model_pin(s: &str) -> bool { + let (name, tag) = s.split_once(':').unwrap_or((s, "x")); + matches!(name.trim().split_once('/'), Some((v, m)) if is_segment(v) && is_segment(m)) + && is_segment(tag) +} + +/// Opaque runner-facing text (task slice label, constraint param): printable, +/// single line, bounded. The control plane never interprets it. +pub fn is_opaque_param(s: &str) -> bool { + !s.trim().is_empty() && s.len() <= 256 && !s.chars().any(char::is_control) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identifier_shapes_are_data_not_catalog() { + for good in ["placeholder_metric", "bench-x_primary-value", "a1"] { + assert!(is_custom_id(good), "{good}"); + } + for bad in [ + "", + "a", + "_lead", + "Upper", + "has space", + "dot.id", + &"x".repeat(65), + ] { + assert!(!is_custom_id(bad), "{bad:?}"); + } + assert!(is_slug("dt-no-ib-v0") && !is_slug("has_underscore")); + for good in ["vendor/model", "vendor/model-2.5:thinking", "a/b"] { + assert!(is_model_pin(good), "{good}"); + } + for bad in ["", "model", "/model", "vendor/", "a/b/c", "a b/c", "a/b:"] { + assert!(!is_model_pin(bad), "{bad:?}"); + } + assert!(is_opaque_param("0..20")); + assert!(is_opaque_param("split:public")); + assert!(!is_opaque_param(" ")); + assert!(!is_opaque_param("two\nlines")); + assert!(!is_opaque_param(&"x".repeat(257))); + assert!(is_hex64(&"ab".repeat(32)) && !is_hex64("abc")); + assert!(is_http_origin("https://example.invalid/v1") && !is_http_origin("ftp://x")); + } + + #[test] + fn rule_vectors_are_bounded_unique_and_non_empty_text() { + let rule = |id: &str, text: &str| ChecklistRule { + id: id.into(), + text: text.into(), + }; + validate_rules(&[]).expect("no rules is a legal (empty) vector"); + validate_rules(&[rule("a_1", "x"), rule("b-2", "y")]).expect("two rules"); + assert_eq!( + validate_rules(&[rule("a_1", "x"), rule("a_1", "y")]).map_err(|e| e.why), + Err("duplicate id") + ); + assert!(validate_rules(&[rule("Bad Id", "x")]).is_err()); + assert!(validate_rules(&[rule("ok", " ")]).is_err()); + assert!(validate_rules(&[rule("ok", &"x".repeat(MAX_RULE_TEXT_LEN + 1))]).is_err()); + let many: Vec = (0..=MAX_CHECKLIST_RULES) + .map(|i| rule(&format!("r_{i}"), "x")) + .collect(); + assert_eq!( + validate_rules(&many).map_err(|e| e.why), + Err("too many rules") + ); + } +} diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 55764b6ca..06bfc28bb 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -199,7 +199,7 @@ impl HarvestRequest { max_proof_deadline_s: plan.deadline_s, eval_image_digest: pin.eval_image_digest.clone(), holdout_commitment: topic.holdout_commitment.clone(), - constraints: topic.constraints, + constraints: topic.constraints.clone(), flops_budget: topic.flops_budget, wall_budget_s: topic.metric.wall_budget_s, claim: claim.to_owned(), @@ -799,6 +799,7 @@ mod tests { no_nvlink: true, no_nccl_fast_fabric: true, max_inter_node_gbps: Some(12.5), + ..proof_task::Constraints::default() }, baseline: b, holdout_commitment: holdout_commitment(&recs), @@ -826,7 +827,7 @@ mod tests { max_proof_deadline_s: 3_600, eval_image_digest: String::new(), holdout_commitment: topic.holdout_commitment.clone(), - constraints: topic.constraints, + constraints: topic.constraints.clone(), flops_budget: topic.flops_budget, wall_budget_s: topic.metric.wall_budget_s, claim: String::new(), diff --git a/crates/proof-score/src/lib.rs b/crates/proof-score/src/lib.rs index 18f6cc95b..6c0fee16b 100644 --- a/crates/proof-score/src/lib.rs +++ b/crates/proof-score/src/lib.rs @@ -19,8 +19,8 @@ mod payout; pub use payout::{ - payout_lattices, primary_from_harness, primary_metric, sealed_primary, topic_masses, - topic_share_bps, MinerTopicRun, PrimaryExtras, PROOF_SHARE_BPS, + novelty_bar, payout_lattices, primary_from_harness, primary_metric, sealed_primary, + topic_masses, topic_share_bps, MinerTopicRun, PrimaryExtras, PROOF_SHARE_BPS, }; use std::collections::BTreeMap; @@ -247,7 +247,17 @@ fn finite(x: f64) -> bool { x.is_finite() } -fn rel_win(challenger: f64, baseline: f64, direction: MetricDirection, epsilon: f64) -> bool { +/// Relative win rule shared by the throughput and custom families (and by +/// automatic promotion): `challenger` beats `baseline` by at least `epsilon` +/// relative, direction-aware. A zero or non-finite baseline can never be +/// beaten — there is no number to be relative to. +#[must_use] +pub fn relative_win( + challenger: f64, + baseline: f64, + direction: MetricDirection, + epsilon: f64, +) -> bool { if !finite(challenger) || !finite(baseline) || baseline.abs() < 1e-12 { return false; } @@ -257,6 +267,10 @@ fn rel_win(challenger: f64, baseline: f64, direction: MetricDirection, epsilon: } } +fn rel_win(challenger: f64, baseline: f64, direction: MetricDirection, epsilon: f64) -> bool { + relative_win(challenger, baseline, direction, epsilon) +} + fn nll_gates( topic: &TopicDocument, harness: &HarnessMetrics, @@ -520,6 +534,7 @@ mod tests { no_nvlink: true, no_nccl_fast_fabric: true, max_inter_node_gbps: Some(12.5), + ..Constraints::default() }, metric: MetricSpec { family: MetricFamily::Throughput, @@ -693,10 +708,10 @@ mod tests { } #[test] - fn harness_success_rate_is_listed_and_fail_closes_without_a_harness_value() { + fn a_registered_custom_metric_still_fail_closes_without_a_harness_value() { let mut topic = nll_topic(); topic.metric.family = MetricFamily::Custom; - topic.metric.custom_id = proof_task::CUSTOM_HARNESS_SUCCESS_RATE.into(); + topic.metric.custom_id = "agent_success_rate".into(); topic.metric.primary = "success_rate".into(); topic.metric.direction = MetricDirection::Max; topic.metric.epsilon_rel = 0.05; @@ -708,7 +723,7 @@ mod tests { &harness, &flat_nll(3.0), &[], - &[proof_task::CUSTOM_HARNESS_SUCCESS_RATE], + &["agent_success_rate"], ); assert!(v .failed diff --git a/crates/proof-score/src/payout.rs b/crates/proof-score/src/payout.rs index 7703ff4ef..da2d7cebf 100644 --- a/crates/proof-score/src/payout.rs +++ b/crates/proof-score/src/payout.rs @@ -267,7 +267,11 @@ fn discovery( out } -fn novelty_bar( +/// The bar a discovery run must clear for novelty weight (and a run for +/// automatic promotion): the sealed primary, the reigning champion, or the +/// better of the two, direction-aware. `None` when nothing has been measured. +#[must_use] +pub fn novelty_bar( topic: &TopicDocument, sealed: Option<&SealedBaseline>, champion: Option, diff --git a/crates/proof-task/Cargo.toml b/crates/proof-task/Cargo.toml index 95b780c59..13ead043d 100644 --- a/crates/proof-task/Cargo.toml +++ b/crates/proof-task/Cargo.toml @@ -11,6 +11,7 @@ publish = false [dependencies] crypto = { path = "../crypto" } hex = "0.4" +proof-canon = { path = "../proof-canon" } proof-holdout = { path = "../proof-holdout" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/proof-task/src/lib.rs b/crates/proof-task/src/lib.rs index 4a0ff18e5..d0fdea25c 100644 --- a/crates/proof-task/src/lib.rs +++ b/crates/proof-task/src/lib.rs @@ -31,13 +31,11 @@ clippy::must_use_candidate )] -mod canonical; mod executor; mod inference; mod pin; mod topic; -pub use canonical::canonical_json; pub use executor::{ TopicEvalExecutor, EVAL_EXECUTOR_COMMITMENT_ALG, EVAL_EXECUTOR_GPU_CLASS, EVAL_EXECUTOR_GPU_COUNT, EVAL_EXECUTOR_SCHEMA_VERSION, MAX_PROOF_DEADLINE_S_CEILING, @@ -49,15 +47,20 @@ pub use inference::{ INFERENCE_OFFER_COMMITMENT_ALG, MAX_INPUT_TOKENS_CEILING, MAX_OUTPUT_TOKENS_CEILING, }; pub use pin::{PinError, ProofPin}; +pub(crate) use proof_canon::is_http_origin; +pub use proof_canon::{ + canonical_json, is_custom_id, is_hex64, is_model_pin, is_opaque_param, is_slug, ChecklistRule, + Constraints, MAX_CHECKLIST_RULES, MAX_CONSTRAINT_PARAMS, MAX_RULE_TEXT_LEN, +}; pub use proof_holdout::{ contamination, holdout_commitment, synthetic_holdout, verify_holdout, HoldoutError, HoldoutRecord, HoldoutSplit, HOLDOUT_DOMAIN, HOLDOUT_SIZE, LONGCTX_MAX_TOKENS, LONGCTX_MIN_TOKENS, STRATUM_SIZE, }; pub use topic::{ - default_adamw, topic_signing_payload, Baseline, Constraints, DiscoverySpec, MetricDirection, - MetricFamily, MetricSpec, PayoutMode, TopicDocument, TopicError, TopicStatus, ValidationSpec, - BPS_DENOM, DISCOVERY_NOVELTY_POOL_SHARE_BPS, DISCOVERY_PASS_FLOOR_SHARE_BPS, MAX_STATEMENT_LEN, + default_adamw, topic_signing_payload, Baseline, DiscoverySpec, MetricDirection, MetricFamily, + MetricSpec, PayoutMode, TopicDocument, TopicError, TopicStatus, ValidationSpec, BPS_DENOM, + DISCOVERY_NOVELTY_POOL_SHARE_BPS, DISCOVERY_PASS_FLOOR_SHARE_BPS, MAX_STATEMENT_LEN, MAX_TOPIC_ID_LEN, MAX_VALIDATION_LEN, METRIC_STEP_LATENCY_MS, METRIC_TOKENS_PER_SEC, MIN_TOPIC_ID_LEN, PRIMARY_HOLDOUT_NLL, TOPIC_SCHEMA_VERSION, }; @@ -103,11 +106,6 @@ pub const EVAL_IMAGE: &str = "ghcr.io/cortexlm/proof-eval"; /// Public docs pointer (this control-plane repo). pub const PROOF_GIT_URL: &str = "https://github.com/CortexLM/cortex"; -/// Custom metric id for the agent-harness success-rate topic. Listed so an -/// operator can publish the document; the eval image fail-closes until a -/// real harness fills `custom_value`. -pub const CUSTOM_HARNESS_SUCCESS_RATE: &str = "harness_success_rate"; - /// Proof challenge emission share (basis points of the subnet). pub const PROOF_EMISSION_BPS: u16 = 8_000; @@ -132,28 +130,6 @@ pub const QUALITY_FLOOR_NLL_MAX: f64 = 0.02; /// Slice id prefix bound into per-topic measurements. pub const HOLDOUT_SLICE_PREFIX: &str = "proof-holdout"; -/// Whether `s` (trimmed) is exactly 64 hex characters. -pub fn is_hex64(s: &str) -> bool { - let t = s.trim(); - t.len() == 64 && t.chars().all(|c| c.is_ascii_hexdigit()) -} - -pub(crate) fn is_http_origin(url: &str) -> bool { - let u = url.trim(); - (u.starts_with("http://") || u.starts_with("https://")) - && u.len() >= 8 - && !u.contains(['\n', ' ']) -} - -/// Whether `id` matches `[a-z0-9][a-z0-9-]{1,62}` (offer and topic ids). -pub fn is_slug(id: &str) -> bool { - let b = id.as_bytes(); - (2..=63).contains(&b.len()) - && (b[0].is_ascii_lowercase() || b[0].is_ascii_digit()) - && b.iter() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') -} - #[cfg(test)] mod tests { use super::*; @@ -215,7 +191,6 @@ mod tests { u32::from(PROOF_EMISSION_BPS) + u32::from(BOUNTY_EMISSION_BPS), 10_000 ); - assert_eq!(CUSTOM_HARNESS_SUCCESS_RATE, "harness_success_rate"); assert_eq!(INFERENCE_CONFIG_SCHEMA_VERSION, 1); assert_eq!(INFERENCE_OFFER_COMMITMENT_ALG, "sha256"); assert_eq!(MAX_INPUT_TOKENS_CEILING, LONGCTX_MAX_TOKENS); diff --git a/crates/proof-task/src/topic.rs b/crates/proof-task/src/topic.rs index 6d0e7dfaa..cf9391cba 100644 --- a/crates/proof-task/src/topic.rs +++ b/crates/proof-task/src/topic.rs @@ -27,7 +27,10 @@ use serde::{Deserialize, Serialize}; -use crate::{canonical_json, is_hex64, ProofPin, TopicEvalExecutor, TopicInference, TOPIC_DOMAIN}; +use crate::{ + canonical_json, is_hex64, ChecklistRule, Constraints, ProofPin, TopicEvalExecutor, + TopicInference, TOPIC_DOMAIN, +}; /// Only accepted `schema_version`. pub const TOPIC_SCHEMA_VERSION: u32 = 1; @@ -321,24 +324,6 @@ impl Baseline { } } -/// Machine-checkable constraints the eval image enforces. -/// -/// `deny_unknown_fields` is the point: a constraint this control plane does -/// not understand is a constraint the image cannot be trusted to enforce, so -/// an unknown key rejects the topic at publish instead of being ignored. -#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, default)] -pub struct Constraints { - /// No `InfiniBand` fabric. - pub no_infiniband: bool, - /// No NVLink between ranks. - pub no_nvlink: bool, - /// No NCCL all-reduce over a fast fabric. - pub no_nccl_fast_fabric: bool, - /// Inter-node (or emulated inter-rank) bandwidth cap in Gbit/s. - pub max_inter_node_gbps: Option, -} - /// One signed research problem. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] @@ -373,6 +358,10 @@ pub struct TopicDocument { /// Omitted from the signed payload when empty, so older signatures hold. #[serde(skip_serializing_if = "TopicEvalExecutor::is_empty")] pub eval_executor: TopicEvalExecutor, + /// Anti-cheat rules ticked before any paid inference spend (custom family). + /// Omitted from the signed payload when empty, so older signatures hold. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub checklist: Vec, /// Sealed baseline recipe plus its two seal hashes. pub baseline: Baseline, /// Commitment over this topic's holdout records. @@ -411,6 +400,7 @@ impl Default for TopicDocument { proxy_model: None, inference: TopicInference::default(), eval_executor: TopicEvalExecutor::default(), + checklist: Vec::new(), baseline: default_adamw(crate::FLOPS_BUDGET_MAX), holdout_commitment: String::new(), holdout_size: crate::HOLDOUT_SIZE, @@ -464,9 +454,19 @@ pub enum TopicError { /// Family it declared. family: &'static str, }, - /// A custom metric this build cannot compute. - #[error("custom metric {0:?} is not implemented in proof-eval")] + /// An open custom topic names a metric no registered runner can compute. + #[error( + "custom metric {0:?} has no registered runner on this host; a topic may draft but not open" + )] UnknownCustomMetric(String), + /// A generic binding (constraint, checklist rule) is malformed. + #[error("topic binding {field}: {why}")] + BadBinding { + /// Which field (`constraints.*`, `checklist[id]`). + field: String, + /// What is wrong. + why: &'static str, + }, /// A family knob is missing. #[error("family {family:?} requires {field}")] MissingFamilyField { @@ -601,7 +601,17 @@ pub fn topic_signing_payload(doc: &TopicDocument) -> Result, TopicError> impl MetricSpec { /// Enforce the family's allowlist and knobs. - fn validate(&self, pin: &ProofPin, supported_custom: &[&str]) -> Result<(), TopicError> { + /// + /// `registered_custom` is the set of custom ids with a runner on this + /// host. A custom topic may **draft** with any well-formed id (the RLM + /// mints ids); it may **open** only when a runner is registered, because + /// nobody is paid for a metric nobody can compute. + fn validate( + &self, + pin: &ProofPin, + registered_custom: &[&str], + status: TopicStatus, + ) -> Result<(), TopicError> { let family = self.family.as_str(); let bad_metric = || TopicError::BadMetric { name: self.primary.clone(), @@ -681,10 +691,16 @@ impl MetricSpec { field: "custom_id", }); } - if !supported_custom.contains(&id) { + if !crate::is_custom_id(id) { + return Err(TopicError::BadBinding { + field: "metric.custom_id".into(), + why: "must match [a-z0-9][a-z0-9_-]{1,63}", + }); + } + if status == TopicStatus::Open && !registered_custom.contains(&id) { return Err(TopicError::UnknownCustomMetric(id.to_owned())); } - if self.primary.trim().is_empty() { + if self.primary.trim().is_empty() || !crate::is_custom_id(self.primary.trim()) { return Err(bad_metric()); } if self.epsilon_rel.is_nan() || self.epsilon_rel <= 0.0 { @@ -789,11 +805,14 @@ impl TopicDocument { /// Structural + floor validation. Does **not** check the signature. /// + /// `registered_custom` is the set of custom metric ids with a runner on + /// this host (empty when none is registered). + /// /// # Errors /// - /// See [`TopicError`]. A `draft` topic with an unsealed baseline is legal; - /// an `open` one is not. - pub fn validate(&self, pin: &ProofPin, supported_custom: &[&str]) -> Result<(), TopicError> { + /// See [`TopicError`]. A `draft` topic with an unsealed baseline or an + /// unregistered custom id is legal; an `open` one is not. + pub fn validate(&self, pin: &ProofPin, registered_custom: &[&str]) -> Result<(), TopicError> { if self.schema_version != TOPIC_SCHEMA_VERSION { return Err(TopicError::WrongSchema { got: self.schema_version, @@ -814,7 +833,14 @@ impl TopicDocument { + u32::from(self.discovery.novelty_pool_share_bps), }); } - self.metric.validate(pin, supported_custom)?; + self.metric.validate(pin, registered_custom, self.status)?; + self.constraints + .validate_shape() + .and_then(|()| proof_canon::validate_rules(&self.checklist)) + .map_err(|e| TopicError::BadBinding { + field: e.field, + why: e.why, + })?; if self.flops_budget == 0 || self.flops_budget > pin.flops_budget_max { return Err(TopicError::BadFlopsBudget { got: self.flops_budget, @@ -971,6 +997,7 @@ mod tests { no_nvlink: true, no_nccl_fast_fabric: true, max_inter_node_gbps: Some(12.5), + ..Constraints::default() }, metric: MetricSpec { family: MetricFamily::Throughput, @@ -1285,43 +1312,152 @@ mod tests { )); } - /// A custom metric nothing implements is a 400 at publish, never a - /// silently-skipped gate. - #[test] - fn custom_metrics_must_be_implemented() { - let p = pin(); + fn custom_topic(custom_id: &str) -> TopicDocument { let mut doc = nll_topic(); + doc.id = "custom-topic-v0".into(); doc.metric = MetricSpec { family: MetricFamily::Custom, - primary: "bits_per_joule".into(), - direction: MetricDirection::Min, - unit: "bits/J".into(), - epsilon_rel: 0.10, + primary: "primary_value".into(), + direction: MetricDirection::Max, + unit: "rate".into(), + epsilon_rel: 0.05, quality_floor_nll: 0.0, wall_budget_s: 0, - custom_id: "bits_per_joule".into(), + custom_id: custom_id.into(), }; + doc + } + + /// A custom metric id is topic data: any well-formed id drafts. Opening + /// needs a registered runner on the host, because nobody is paid for a + /// metric nobody can compute. There is no id list in this crate. + #[test] + fn custom_metrics_draft_freely_and_open_only_with_a_registered_runner() { + let p = pin(); + let mut open = custom_topic("bits_per_joule"); assert!(matches!( - doc.validate(&p, &[]), + open.validate(&p, &[]), Err(TopicError::UnknownCustomMetric(_)) )); - doc.validate(&p, &["bits_per_joule"]) - .expect("implemented custom metric"); + open.validate(&p, &["bits_per_joule"]) + .expect("registered runner may open"); + open.status = TopicStatus::Draft; + open.validate(&p, &[]).expect("a draft may name any id"); + + for bad in ["", "Upper", "has space", "x", "dotted.id"] { + let mut doc = custom_topic(bad); + doc.status = TopicStatus::Draft; + assert!( + matches!( + doc.validate(&p, &[]), + Err(TopicError::BadBinding { .. } | TopicError::MissingFamilyField { .. }) + ), + "{bad:?}" + ); + } + let mut bad_primary = custom_topic("bits_per_joule"); + bad_primary.metric.primary = "bits per joule".into(); + assert!(matches!( + bad_primary.validate(&p, &["bits_per_joule"]), + Err(TopicError::BadMetric { .. }) + )); + } - let mut harness = nll_topic(); - harness.metric = MetricSpec { - family: MetricFamily::Custom, - primary: "success_rate".into(), - direction: MetricDirection::Max, - unit: "rate".into(), - epsilon_rel: 0.05, - quality_floor_nll: 0.0, - wall_budget_s: 0, - custom_id: crate::CUSTOM_HARNESS_SUCCESS_RATE.into(), - }; - harness - .validate(&p, &[crate::CUSTOM_HARNESS_SUCCESS_RATE]) - .expect("listed custom stub is publishable"); + /// Sandbox / model / slice knobs are generic policy; values are checked + /// for shape only and the document is what carries them. + #[test] + fn generic_constraints_and_rules_are_shape_checked_topic_data() { + let p = pin(); + let mut doc = custom_topic("agent_pass_rate"); + doc.status = TopicStatus::Draft; + doc.constraints.firecracker_required = true; + doc.constraints.model_pin = Some("vendor/model:tag".into()); + doc.constraints.task_slice = Some("0..20".into()); + doc.constraints + .params + .insert("target_repo".into(), "owner/name".into()); + doc.checklist = vec![ + ChecklistRule { + id: "same_seed".into(), + text: "every paid call uses the topic seed".into(), + }, + ChecklistRule { + id: "no_hardcode".into(), + text: "no task answers in the artefact".into(), + }, + ]; + doc.validate(&p, &[]).expect("generic bindings validate"); + let payload = + String::from_utf8(topic_signing_payload(&doc).expect("payload")).expect("utf8"); + for field in [ + "\"firecracker_required\":true", + "\"model_pin\":\"vendor/model:tag\"", + "\"task_slice\":\"0..20\"", + "\"params\":{\"target_repo\":\"owner/name\"}", + "\"checklist\":[{\"id\":\"same_seed\"", + ] { + assert!(payload.contains(field), "missing {field} in {payload}"); + } + + let mut bad_model = doc.clone(); + bad_model.constraints.model_pin = Some("model".into()); + assert!(matches!( + bad_model.validate(&p, &[]), + Err(TopicError::BadBinding { ref field, .. }) if field == "constraints.model_pin" + )); + let mut bad_slice = doc.clone(); + bad_slice.constraints.task_slice = Some("two\nlines".into()); + assert!(matches!( + bad_slice.validate(&p, &[]), + Err(TopicError::BadBinding { ref field, .. }) if field == "constraints.task_slice" + )); + let mut bad_param = doc.clone(); + bad_param + .constraints + .params + .insert("Bad Key".into(), "v".into()); + assert!(matches!( + bad_param.validate(&p, &[]), + Err(TopicError::BadBinding { ref field, .. }) if field == "constraints.params" + )); + let mut dup = doc.clone(); + dup.checklist.push(ChecklistRule { + id: "same_seed".into(), + text: "again".into(), + }); + assert!(matches!( + dup.validate(&p, &[]), + Err(TopicError::BadBinding { + why: "duplicate id", + .. + }) + )); + let mut blank = doc.clone(); + blank.checklist[0].text = " ".into(); + assert!(matches!( + blank.validate(&p, &[]), + Err(TopicError::BadBinding { .. }) + )); + let mut bad_id = doc.clone(); + bad_id.checklist[0].id = "Same Seed".into(); + assert!(matches!( + bad_id.validate(&p, &[]), + Err(TopicError::BadBinding { .. }) + )); + let mut many = doc; + many.checklist = (0..=crate::MAX_CHECKLIST_RULES) + .map(|i| ChecklistRule { + id: format!("rule_{i}"), + text: "x".into(), + }) + .collect(); + assert!(matches!( + many.validate(&p, &[]), + Err(TopicError::BadBinding { + why: "too many rules", + .. + }) + )); } #[test] From 047e5df1d257516ad5f210721be45f3f8689b6a5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:09:28 +0000 Subject: [PATCH 02/14] feat(proof-eval): registry-driven custom family with scorer hooks Drop the static supported_custom() list. Custom metric ids now come from the host's live scorer (LiveScorer::custom_ids, registered_custom): a topic mints its id, a runner registered under it makes the topic scorable. FamilyMux routes the whole custom family to one pluggable scorer and everything else to the digest-pinned harvest; with no custom scorer, every custom topic is RunnerUnwired (503), never a harvest fallback. LiveScorer gains ready_for_topic (per-topic refusal before any row or rent), custom_ids, and async auto_promote / on_persisted hooks so a family scorer can crown a pass and write artefacts once the row has its id. EvalError::RunnerUnwired names the missing runner as the root cause. --- crates/proof-eval/src/lib.rs | 338 +++++++++++++++++++++++++++++++++-- 1 file changed, 328 insertions(+), 10 deletions(-) diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 51a123fd4..3cc40c2df 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -22,6 +22,7 @@ )] use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; use async_trait::async_trait; use prism_lium_types::{EvalReceipt, NoScoreGate}; @@ -146,6 +147,14 @@ pub enum EvalError { /// Last bytes of pod stdout (run log tail), for the operator. stdout_tail: String, }, + /// A custom metric family has no registered runner on this host. + #[error("custom metric {custom_id:?} has no registered runner ({detail}); refuse scoring")] + RunnerUnwired { + /// Custom metric id the topic names. + custom_id: String, + /// Which piece is missing (never a secret). + detail: String, + }, } /// Map an executor refusal onto the eval error the HTTP layer answers 503 with. @@ -160,15 +169,19 @@ pub fn map_executor_err(e: ExecutorOfferError) -> EvalError { /// Schema version of the metrics+verdict document the eval image emits. pub const PROOF_METRICS_SCHEMA: u32 = 1; -/// Custom metric ids this control-plane build can score. -/// -/// `harness_success_rate` is listed so an operator can publish the agent-harness -/// topic. The real GPU harness is not in this image yet: scoring fail-closes -/// (`EvidenceMissing` on `custom_value`) until that sidecar exists. Do not -/// invent a success rate. +/// Custom metric ids with a registered runner, as reported by the host's +/// live scorer. There is **no** compiled-in list: a topic mints its +/// `custom_id`, a runner registered under that id makes it scorable, and an +/// open topic whose id is absent here answers 503. +#[must_use] +pub fn registered_custom(live: Option<&dyn LiveScorer>) -> Vec { + live.map(LiveScorer::custom_ids).unwrap_or_default() +} + +/// Borrow a registered-id list as the `&[&str]` the topic validator takes. #[must_use] -pub fn supported_custom() -> Vec<&'static str> { - vec![proof_task::CUSTOM_HARNESS_SUCCESS_RATE] +pub fn custom_ids_ref(ids: &[String]) -> Vec<&str> { + ids.iter().map(String::as_str).collect() } /// The document `proof-eval` must print for one scored artifact. @@ -297,6 +310,177 @@ pub trait LiveScorer: Send + Sync { fn ready(&self) -> Result<(), EvalError> { Ok(()) } + + /// Whether this scorer could score `topic` right now. A custom family + /// whose runner is not registered refuses here, before any row or rent. + fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + let _ = topic; + self.ready() + } + + /// Custom metric ids this scorer has a registered runner for. Default: + /// none — the digest-pinned harvest scores `nll` / `throughput` only. + fn custom_ids(&self) -> Vec { + Vec::new() + } + + /// Whether the run for `submission_digest` on `topic` should be crowned + /// champion automatically. + /// + /// `pass` is the harness verdict, `primary` its primary metric, `bar` the + /// current novelty bar (sealed baseline vs reigning champion, + /// direction-aware). Default: never — promotion stays an operator action. + async fn auto_promote( + &self, + topic: &TopicDocument, + submission_digest: &str, + pass: bool, + primary: Option, + bar: Option, + ) -> bool { + let _ = (topic, submission_digest, pass, primary, bar); + false + } + + /// Called once the scored row is persisted and has its `pf_…` id, so a + /// scorer can write per-submission artefacts and promotion events. + /// Failures are the scorer's to log; the row is already final. + async fn on_persisted( + &self, + topic_id: &str, + submission_digest: &str, + submission_id: &str, + promoted: bool, + ) { + let _ = (topic_id, submission_digest, submission_id, promoted); + } +} + +/// Route scoring by metric family: `custom` topics go to the registered +/// custom-family scorer (which resolves the runner by `custom_id`, fail-closed); +/// `nll` / `throughput` go to the default digest-pinned harvest. Planning, +/// readiness, promotion, and artefact hooks follow the same route, so an +/// unregistered custom id can never fall back to the harvest. +pub struct FamilyMux { + default: Arc, + custom: Option>, +} + +impl FamilyMux { + /// Mux over the default harvest with no custom-family scorer: every + /// custom topic is unscorable (503). + #[must_use] + pub fn new(default: Arc) -> Self { + Self { + default, + custom: None, + } + } + + /// Route the whole `custom` family to `scorer`. + #[must_use] + pub fn with_custom_family(mut self, scorer: Arc) -> Self { + self.custom = Some(scorer); + self + } + + fn route(&self, topic: &TopicDocument) -> Result<&dyn LiveScorer, EvalError> { + if topic.metric.family != MetricFamily::Custom { + return Ok(self.default.as_ref()); + } + self.custom + .as_deref() + .ok_or_else(|| EvalError::RunnerUnwired { + custom_id: topic.metric.custom_id.trim().to_owned(), + detail: "no custom-family scorer on this host".into(), + }) + } +} + +#[async_trait] +impl LiveScorer for FamilyMux { + fn plan( + &self, + pin: &ProofPin, + topic: &TopicDocument, + executor: &EvalExecutorOffer, + ) -> Result { + self.route(topic)?.plan(pin, topic, executor) + } + + async fn score( + &self, + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + plan: &ExecutorPlan, + frozen_digest: &str, + artifact_digest: &str, + holdout: &[HoldoutRecord], + claim: &str, + ) -> Result { + self.route(topic)? + .score( + pin, + topic, + offer, + plan, + frozen_digest, + artifact_digest, + holdout, + claim, + ) + .await + } + + fn ready(&self) -> Result<(), EvalError> { + self.default.ready() + } + + fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + self.route(topic)?.ready_for_topic(topic) + } + + fn custom_ids(&self) -> Vec { + self.custom + .as_deref() + .map_or_else(Vec::new, LiveScorer::custom_ids) + } + + async fn auto_promote( + &self, + topic: &TopicDocument, + submission_digest: &str, + pass: bool, + primary: Option, + bar: Option, + ) -> bool { + match self.route(topic) { + Ok(s) => { + s.auto_promote(topic, submission_digest, pass, primary, bar) + .await + } + Err(_) => false, + } + } + + /// The persist hook arrives without the document, so both routes are + /// told; only the scorer holding a pending bundle for this digest acts. + async fn on_persisted( + &self, + topic_id: &str, + submission_digest: &str, + submission_id: &str, + promoted: bool, + ) { + self.default + .on_persisted(topic_id, submission_digest, submission_id, promoted) + .await; + if let Some(c) = &self.custom { + c.on_persisted(topic_id, submission_digest, submission_id, promoted) + .await; + } + } } /// Operator-recorded sealed baseline for one topic. @@ -1256,14 +1440,148 @@ mod tests { } #[test] - fn harness_success_rate_is_listed_and_sim_does_not_invent_a_value() { - assert!(supported_custom().contains(&proof_task::CUSTOM_HARNESS_SUCCESS_RATE)); + fn custom_ids_come_from_the_live_scorer_and_sim_never_invents_a_value() { + assert!(registered_custom(None).is_empty()); + assert!(registered_custom(Some(&Harvest { reproduced: true })).is_empty()); let t = topic(); let pin = pin(""); let doc = sim_document(&pin, &t, "f", "art", 1.0, true); assert!(doc.harness.custom_value.is_none()); } + /// A custom-family scorer with a registry of one id: that id is + /// scorable, another id refuses, and no custom topic ever reaches the + /// default harvest. + struct OneRunner; + + #[async_trait] + impl LiveScorer for OneRunner { + async fn score( + &self, + _pin: &ProofPin, + topic: &TopicDocument, + _offer: &InferenceOffer, + _plan: &ExecutorPlan, + _frozen: &str, + _artifact: &str, + _holdout: &[HoldoutRecord], + _claim: &str, + ) -> Result { + self.ready_for_topic(topic)?; + Err(EvalError::Backend("would run the registered runner".into())) + } + + fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + if topic.metric.custom_id == "registered_metric" { + Ok(()) + } else { + Err(EvalError::RunnerUnwired { + custom_id: topic.metric.custom_id.clone(), + detail: "not in registry".into(), + }) + } + } + + fn custom_ids(&self) -> Vec { + vec!["registered_metric".into()] + } + + async fn auto_promote( + &self, + _t: &TopicDocument, + _digest: &str, + pass: bool, + p: Option, + b: Option, + ) -> bool { + pass && p > b + } + } + + fn custom_topic(id: &str) -> TopicDocument { + let mut t = topic(); + t.metric.family = MetricFamily::Custom; + t.metric.custom_id = id.into(); + t + } + + #[tokio::test] + async fn family_mux_routes_custom_to_the_registry_and_never_to_the_harvest() { + let bare = FamilyMux::new(Arc::new(Harvest { reproduced: true })); + bare.ready_for_topic(&topic()) + .expect("nll routes to the harvest"); + assert!(bare.custom_ids().is_empty()); + assert!(matches!( + bare.ready_for_topic(&custom_topic("anything")), + Err(EvalError::RunnerUnwired { .. }) + )); + + let mux = FamilyMux::new(Arc::new(Harvest { reproduced: true })) + .with_custom_family(Arc::new(OneRunner)); + assert_eq!(mux.custom_ids(), vec!["registered_metric".to_owned()]); + assert_eq!( + registered_custom(Some(&mux)), + vec!["registered_metric".to_owned()] + ); + assert_eq!(custom_ids_ref(&mux.custom_ids()), vec!["registered_metric"]); + mux.ready_for_topic(&custom_topic("registered_metric")) + .expect("registered id"); + let err = mux + .ready_for_topic(&custom_topic("unknown_metric")) + .expect_err("unregistered id"); + assert!(matches!(err, EvalError::RunnerUnwired { .. }), "{err}"); + assert!( + !mux.auto_promote(&topic(), "d", true, Some(1.0), Some(0.5)) + .await + ); + assert!( + mux.auto_promote( + &custom_topic("registered_metric"), + "d", + true, + Some(1.0), + Some(0.5) + ) + .await + ); + assert!( + !mux.auto_promote( + &custom_topic("registered_metric"), + "d", + false, + Some(1.0), + Some(0.5) + ) + .await + ); + + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let exec = executor(&p); + let plan = mux.plan(&p, &topic(), &exec).expect("harvest plan"); + let err = mux + .score( + &p, + &custom_topic("unknown_metric"), + &offer(), + &plan, + "d", + "a", + &recs, + "c", + ) + .await + .expect_err("unregistered family must not sim or harvest"); + assert!(matches!(err, EvalError::RunnerUnwired { .. }), "{err}"); + // With no custom-family scorer at all, even planning a custom topic + // refuses: nothing may rent for a family nobody can score. + assert!(matches!( + bare.plan(&p, &custom_topic("unknown_metric"), &exec), + Err(EvalError::RunnerUnwired { .. }) + )); + mux.on_persisted("t", "d", "pf_0", false).await; + } + fn tight_sealed() -> SealedBaseline { let mut split = BTreeMap::new(); for s in HoldoutSplit::SCORED { From f528097fa585c166890a74d861fd22495e1611bc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:09:28 +0000 Subject: [PATCH 03/14] feat(proof-rlm): generic rlm engine core Core types for Proof as a dynamic agentic challenge system. Nothing here names a benchmark, metric, model, rule list, or repository (a self-check test scans the non-test source for such literals). - RuleSet / Checklist / SpendToken: rules are a versioned vector (v1 = the signed topic's checklist, later versions written by the RLM). A checklist binds to a rule version and digest and is green only when every rule is ticked with evidence and passes; the token has no public constructor, so no paid run can be reached from a red, incomplete, or stale checklist. - Lifecycle: draft -> owner_presend -> awaiting_owner_keys -> provisioning -> baselining -> open <-> evaluating -> promoting -> closed as an explicit table, with an askUser-style OwnerHook (refuses without a hook) and a presence-only owner key probe. - CustomRunner + RunnerRegistry: custom_id -> runner, empty by default; an unregistered id is RunnerError::Unregistered (the 503 root cause). - TopicVmOrchestrator: create / attach / run / teardown for per-topic VMs; VmJobs carry public data only (never a host path, key, or origin). UnwiredVmOrchestrator refuses and names the env vars; the generic VmBackedRunner turns inspect / evaluate into VM jobs and is only ever registered by an operator. - decide_promote: pass + green checklist + relative win over the bar, direction from the topic. Fixtures (fake orchestrator, placeholder topic) ship behind the test-fixtures feature for sibling crates' tests. --- crates/proof-rlm/Cargo.toml | 30 + crates/proof-rlm/src/fixtures_tests.rs | 308 ++++++++++ crates/proof-rlm/src/gate.rs | 150 +++++ crates/proof-rlm/src/lib.rs | 114 ++++ crates/proof-rlm/src/promote.rs | 186 ++++++ crates/proof-rlm/src/rules.rs | 501 ++++++++++++++++ crates/proof-rlm/src/runner.rs | 504 +++++++++++++++++ crates/proof-rlm/src/state.rs | 753 +++++++++++++++++++++++++ crates/proof-rlm/src/vm.rs | 510 +++++++++++++++++ 9 files changed, 3056 insertions(+) create mode 100644 crates/proof-rlm/Cargo.toml create mode 100644 crates/proof-rlm/src/fixtures_tests.rs create mode 100644 crates/proof-rlm/src/gate.rs create mode 100644 crates/proof-rlm/src/lib.rs create mode 100644 crates/proof-rlm/src/promote.rs create mode 100644 crates/proof-rlm/src/rules.rs create mode 100644 crates/proof-rlm/src/runner.rs create mode 100644 crates/proof-rlm/src/state.rs create mode 100644 crates/proof-rlm/src/vm.rs diff --git a/crates/proof-rlm/Cargo.toml b/crates/proof-rlm/Cargo.toml new file mode 100644 index 000000000..4c3dd9ea2 --- /dev/null +++ b/crates/proof-rlm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "proof-rlm" +description = "Generic Proof RLM engine: topic-carried checklist rules + spend gate, per-topic lifecycle with owner hooks, custom-runner registry (fail-closed), topic-VM orchestrator boundary (unwired stub), promotion rule. No challenge content." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[features] +# Exposes `proof_rlm::fixtures` (fake orchestrator, canned report) to sibling crates' tests. +test-fixtures = [] + +[dependencies] +async-trait = "0.1" +hex = "0.4" +proof-canon = { path = "../proof-canon" } +proof-score = { path = "../proof-score" } +proof-task = { path = "../proof-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/proof-rlm/src/fixtures_tests.rs b/crates/proof-rlm/src/fixtures_tests.rs new file mode 100644 index 000000000..581541e44 --- /dev/null +++ b/crates/proof-rlm/src/fixtures_tests.rs @@ -0,0 +1,308 @@ +//! Shared test fixtures: a custom-family topic with placeholder ids, a judge +//! offer, a rule set, a run request, a canned report, a green checklist / +//! spend token, a pinned VM template, and a recording fake orchestrator. +//! No network, no secrets, no challenge content — every id here is a +//! placeholder a test invents, never something a runner would recognise. +//! Compiled for tests and the `test-fixtures` feature only (the file name +//! keeps it out of the LOC cap's non-test count, like every other +//! `*_tests.rs`). + +// Test-only code: never compiled into a host binary (see the cfg in lib.rs). +#![allow( + clippy::missing_panics_doc, + clippy::must_use_candidate, + clippy::expect_used, + clippy::unwrap_used +)] + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use proof_task::{ + inference_config_commitment, ChecklistRule, InferenceConfig, InferenceMode, InferenceOffer, + InferenceProvider, InferenceProviderKind, MetricDirection, MetricFamily, OfferStatus, ProofPin, + TopicDocument, TopicStatus, +}; + +use crate::gate::{authorize_spend, SpendToken}; +use crate::rules::{Checklist, RuleSet}; +use crate::runner::{ + ArtifactFile, CustomRunReport, CustomRunRequest, InspectOutcome, LogFile, RunOutcome, + RUN_REPORT_SCHEMA, +}; +use crate::vm::{ + RetainPolicy, TopicVmOrchestrator, TopicVmSpec, VmError, VmHandle, VmJob, VmJobOutput, + VmTemplate, +}; + +/// Pin with a topic key and a judge model (no digest unless asked). +pub fn pin() -> ProofPin { + let mut p = ProofPin { + eval_image_digest: format!("sha256:{}", "ab".repeat(32)), + topic_pubkey: "ab".repeat(32), + ..ProofPin::default() + }; + p.inference.model = "judge-model-placeholder".into(); + p +} + +pub fn offer() -> InferenceOffer { + let config = InferenceConfig { + mode: InferenceMode::Chat, + model_ref: "judge-model-placeholder".into(), + max_input_tokens: 32_768, + max_output_tokens: 8_192, + temperature: Some(0.0), + top_p: None, + timeout_ms: None, + }; + InferenceOffer { + offer_id: "judge-offer-placeholder".into(), + provider: InferenceProvider { + kind: InferenceProviderKind::OpenaiCompatible, + base_url: "http://127.0.0.1:8000/v1".into(), + }, + config_commitment: inference_config_commitment(&config, "http://127.0.0.1:8000/v1"), + config, + status: OfferStatus::Open, + } +} + +/// A sealed, open custom-family topic with placeholder bindings. +pub fn topic() -> TopicDocument { + let mut doc = TopicDocument { + id: "topic-a".into(), + statement: "Placeholder research problem scored by a topic-minted custom metric.".into(), + ..TopicDocument::default() + }; + doc.metric.family = MetricFamily::Custom; + doc.metric.custom_id = "placeholder_metric".into(); + doc.metric.primary = "primary_value".into(); + doc.metric.direction = MetricDirection::Max; + doc.metric.epsilon_rel = 0.02; + doc.constraints.firecracker_required = true; + doc.constraints.model_pin = Some("vendor/model-placeholder".into()); + doc.constraints.task_slice = Some("slice-placeholder".into()); + doc.constraints + .params + .insert("param_a".into(), "value-a".into()); + doc.checklist = vec![ + ChecklistRule { + id: "rule_a".into(), + text: "placeholder rule a".into(), + }, + ChecklistRule { + id: "rule_b".into(), + text: "placeholder rule b".into(), + }, + ChecklistRule { + id: "rule_c".into(), + text: "placeholder rule c".into(), + }, + ]; + doc.baseline.optimizer = "reference-placeholder".into(); + doc.baseline.lr = 1.0; + doc.baseline.schedule = "n/a".into(); + doc.baseline.dtype = "n/a".into(); + doc.baseline.script_sha256 = "11".repeat(32); + doc.baseline.metrics_commitment = "22".repeat(32); + doc.status = TopicStatus::Open; + doc +} + +pub fn rules() -> RuleSet { + RuleSet::from_topic(&topic()).expect("rules") +} + +pub fn request() -> CustomRunRequest { + CustomRunRequest::from_topic( + &topic(), + &pin(), + &offer(), + &rules(), + "digest-a", + &"ab".repeat(32), + Some("https://example.invalid/artifact.zip"), + "placeholder claim", + ) + .expect("request") +} + +/// A report that echoes `req` with `primary_value`. +pub fn report_for(req: &CustomRunRequest, primary_value: f64) -> CustomRunReport { + let mut evidence = BTreeMap::new(); + evidence.insert( + "rows".into(), + serde_json::json!([{"index": 0, "passed": true}]), + ); + CustomRunReport { + schema_version: RUN_REPORT_SCHEMA, + topic_id: req.topic_id.clone(), + custom_id: req.custom_id.clone(), + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.clone(), + rules_version: req.rules_version, + primary_value, + claim_holds: true, + sandboxed: true, + evidence, + } +} + +pub fn green(rules: &RuleSet, submission_digest: &str) -> Checklist { + let mut c = Checklist::new(rules, submission_digest, "art"); + for r in &rules.rules { + c.record(&r.id, true, &r.text); + } + c +} + +pub fn token_for(req: &CustomRunRequest) -> SpendToken { + let set = rules(); + let mut c = Checklist::new(&set, &req.submission_digest, &req.artifact_digest); + for r in &set.rules { + c.record(&r.id, true, &r.text); + } + authorize_spend(&c, &set, &req.topic_id, &req.submission_digest).expect("token") +} + +pub fn pinned_template() -> VmTemplate { + VmTemplate { + image_digest: format!("sha256:{}", "cc".repeat(32)), + vcpus: 2, + mem_mib: 4_096, + } +} + +/// Records jobs and answers with canned documents. Never touches the network. +pub struct FakeOrchestrator { + primary: Mutex, + red: Mutex>, + sandboxed: AtomicBool, + created: AtomicUsize, + vms: Mutex>, + jobs: Mutex>, + proposed: Mutex>, +} + +impl FakeOrchestrator { + pub fn new(primary: f64) -> Arc { + Arc::new(Self { + primary: Mutex::new(primary), + red: Mutex::new(None), + sandboxed: AtomicBool::new(true), + created: AtomicUsize::new(0), + vms: Mutex::new(Vec::new()), + jobs: Mutex::new(Vec::new()), + proposed: Mutex::new(vec![ChecklistRule { + id: "rlm_rule".into(), + text: "a rule the fake rlm wrote".into(), + }]), + }) + } + + pub fn set_primary(&self, v: f64) { + *self.primary.lock().unwrap() = v; + } + + /// Make inspections fail this rule id (None = green). + pub fn set_red(&self, id: Option<&str>) { + *self.red.lock().unwrap() = id.map(str::to_owned); + } + + pub fn set_sandboxed(&self, v: bool) { + self.sandboxed.store(v, Ordering::SeqCst); + } + + pub fn set_proposed(&self, rules: Vec) { + *self.proposed.lock().unwrap() = rules; + } + + pub fn created(&self) -> usize { + self.created.load(Ordering::SeqCst) + } + + pub fn jobs(&self) -> Vec { + self.jobs.lock().unwrap().clone() + } + + pub fn vms(&self) -> Vec { + self.vms.lock().unwrap().clone() + } + + fn report(&self, req: &CustomRunRequest) -> CustomRunReport { + let mut r = report_for(req, *self.primary.lock().unwrap()); + r.sandboxed = self.sandboxed.load(Ordering::SeqCst); + r + } +} + +#[async_trait] +impl TopicVmOrchestrator for FakeOrchestrator { + fn ready(&self) -> Result<(), VmError> { + Ok(()) + } + + async fn create(&self, spec: &TopicVmSpec) -> Result { + spec.validate()?; + let n = self.created.fetch_add(1, Ordering::SeqCst); + let h = VmHandle { + topic_id: spec.topic_id.clone(), + vm_id: format!("vm-{n}"), + }; + self.vms.lock().unwrap().push(h.clone()); + Ok(h) + } + + async fn attach(&self, topic_id: &str) -> Result, VmError> { + Ok(self + .vms + .lock() + .unwrap() + .iter() + .find(|h| h.topic_id == topic_id) + .cloned()) + } + + async fn run(&self, handle: &VmHandle, job: VmJob) -> Result { + assert!( + self.vms.lock().unwrap().contains(handle), + "job on an unknown vm" + ); + self.jobs.lock().unwrap().push(job.clone()); + Ok(match job { + VmJob::ProposeRules { .. } => VmJobOutput::Rules(self.proposed.lock().unwrap().clone()), + VmJob::Baseline { request } => VmJobOutput::Baseline(self.report(&request)), + VmJob::Inspect { request, rules } => { + let red = self.red.lock().unwrap().clone(); + let mut checklist = + Checklist::new(&rules, &request.submission_digest, &request.artifact_digest); + for r in &rules.rules { + checklist.record(&r.id, red.as_deref() != Some(r.id.as_str()), &r.text); + } + VmJobOutput::Inspected(InspectOutcome { + checklist, + artifact: vec![ArtifactFile { + path: "src/main.rs".into(), + bytes: b"fn main() {}\n".to_vec(), + }], + }) + } + VmJob::Evaluate { request, .. } => VmJobOutput::Evaluated(RunOutcome { + report: self.report(&request), + logs: vec![LogFile { + name: "run.log".into(), + bytes: b"ok\n".to_vec(), + }], + }), + VmJob::Archive { .. } => VmJobOutput::Archived, + }) + } + + async fn teardown(&self, handle: &VmHandle, _policy: RetainPolicy) -> Result { + self.vms.lock().unwrap().retain(|h| h != handle); + Ok(true) + } +} diff --git a/crates/proof-rlm/src/gate.rs b/crates/proof-rlm/src/gate.rs new file mode 100644 index 000000000..33220e791 --- /dev/null +++ b/crates/proof-rlm/src/gate.rs @@ -0,0 +1,150 @@ +//! Spend gate: nothing makes a paid inference call without a [`SpendToken`], +//! and the only way to mint one is a checklist that is green **for the +//! topic's current rule version**. +//! +//! The token has no public constructor. A runner that takes `&SpendToken` +//! therefore cannot be reached from a red, incomplete, or stale checklist, +//! and the token is bound to the exact rules, checklist bytes, and +//! submission it covers, so a token minted for one run cannot authorise +//! another. + +use crate::rules::{Checklist, ChecklistError, RuleSet}; + +/// Proof that the anti-cheat checklist for one submission was green. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpendToken { + topic_id: String, + submission_digest: String, + rules_version: u32, + checklist_digest: String, +} + +/// Why spend was refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum GateError { + /// The checklist is incomplete, red, or for another rule version. + #[error("paid inference refused: {0}")] + Checklist(#[from] ChecklistError), + /// The checklist names a different submission than the one being run. + #[error("paid inference refused: checklist covers another submission")] + WrongSubmission, +} + +impl SpendToken { + /// Topic the token covers. + #[must_use] + pub fn topic_id(&self) -> &str { + &self.topic_id + } + + /// Frozen submission digest the token covers. + #[must_use] + pub fn submission_digest(&self) -> &str { + &self.submission_digest + } + + /// Rule version the checklist was green for. + #[must_use] + pub fn rules_version(&self) -> u32 { + self.rules_version + } + + /// Digest of the checklist that minted this token. + #[must_use] + pub fn checklist_digest(&self) -> &str { + &self.checklist_digest + } + + /// Whether this token authorises spend for `topic_id` / `submission_digest`. + #[must_use] + pub fn covers(&self, topic_id: &str, submission_digest: &str) -> bool { + self.topic_id == topic_id.trim() && self.submission_digest == submission_digest.trim() + } +} + +/// Mint a spend token from a checklist that is green under `rules` for the +/// submission it names. +/// +/// # Errors +/// +/// [`GateError::Checklist`] when any rule is missing, unknown, duplicated, +/// evidence-less, failed, or the checklist is for another rule version; +/// [`GateError::WrongSubmission`] when it was produced for another digest. +pub fn authorize_spend( + checklist: &Checklist, + rules: &RuleSet, + topic_id: &str, + submission_digest: &str, +) -> Result { + checklist.verify(rules)?; + if checklist.topic_id.trim() != topic_id.trim() + || checklist.submission_digest.trim() != submission_digest.trim() + { + return Err(GateError::WrongSubmission); + } + Ok(SpendToken { + topic_id: topic_id.trim().to_owned(), + submission_digest: submission_digest.trim().to_owned(), + rules_version: rules.version, + checklist_digest: checklist.digest(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::{green, rules}; + use crate::rules::RuleSource; + + #[test] + fn a_green_checklist_mints_a_bound_token() { + let set = rules(); + let c = green(&set, "digest-a"); + let t = authorize_spend(&c, &set, &set.topic_id, "digest-a").expect("token"); + assert_eq!(t.topic_id(), set.topic_id); + assert_eq!(t.submission_digest(), "digest-a"); + assert_eq!(t.rules_version(), 1); + assert_eq!(t.checklist_digest(), c.digest()); + assert!(t.covers(&set.topic_id, "digest-a")); + assert!(!t.covers(&set.topic_id, "digest-b")); + assert!(!t.covers("other", "digest-a")); + } + + /// The headline rule: any red item, no token, no paid inference. + #[test] + fn a_red_incomplete_or_stale_checklist_never_mints() { + let set = rules(); + let mut red = green(&set, "digest-a"); + red.items[1].pass = false; + assert!(matches!( + authorize_spend(&red, &set, &set.topic_id, "digest-a"), + Err(GateError::Checklist(ChecklistError::Failed(_))) + )); + let mut partial = green(&set, "digest-a"); + partial.items.pop(); + assert!(matches!( + authorize_spend(&partial, &set, &set.topic_id, "digest-a"), + Err(GateError::Checklist(ChecklistError::Missing(_))) + )); + let stale = green(&set, "digest-a"); + let v2 = set.next(RuleSource::Rlm, set.rules.clone()).expect("v2"); + assert_eq!( + authorize_spend(&stale, &v2, &set.topic_id, "digest-a"), + Err(GateError::Checklist(ChecklistError::WrongRuleSet)) + ); + } + + #[test] + fn a_token_cannot_be_borrowed_across_submissions() { + let set = rules(); + let c = green(&set, "digest-a"); + assert_eq!( + authorize_spend(&c, &set, &set.topic_id, "digest-b"), + Err(GateError::WrongSubmission) + ); + assert_eq!( + authorize_spend(&c, &set, "other-topic", "digest-a"), + Err(GateError::WrongSubmission) + ); + } +} diff --git a/crates/proof-rlm/src/lib.rs b/crates/proof-rlm/src/lib.rs new file mode 100644 index 000000000..adbce4b26 --- /dev/null +++ b/crates/proof-rlm/src/lib.rs @@ -0,0 +1,114 @@ +//! Generic Proof RLM engine — core types. No challenge content. +//! +//! Proof is a **dynamic agentic challenge system**: every research problem is +//! an operator-published signed topic, and each topic's RLM (research +//! lifecycle manager) runs **inside a VM attributed to that topic** where it +//! writes the anti-cheat rules, runs the baseline, inspects and runs miner +//! submissions, and promotes the best artefact. This crate is the control +//! plane's side of that boundary. It knows shapes, not challenges: nothing +//! here names a benchmark, a metric, a model, or a repository. +//! +//! 1. [`RuleSet`] + [`Checklist`] + [`SpendToken`] — the anti-cheat gate as +//! data. Rules are a versioned vector (v1 = the signed topic's +//! `checklist`, later versions written by the RLM and persisted by the +//! store). A checklist is green only when every rule of its version is +//! ticked with evidence and passes; the token type is the only way to +//! reach a paid run. +//! 2. [`Lifecycle`] — `draft → owner_presend → awaiting_owner_keys → +//! provisioning → baselining → open ⇄ evaluating → promoting → closed`, +//! with `owner_presend` (`askUser`-style) and owner-key presence hooks. +//! 3. [`CustomRunner`] + [`RunnerRegistry`] — `custom_id → runner`, empty by +//! default. An unregistered id is [`RunnerError::Unregistered`], which the +//! host turns into a 503 before any row or rent. +//! 4. [`TopicVmOrchestrator`] — create / attach / run / teardown for topic +//! VMs, with [`VmJob`]s that carry public data only. The shipped +//! implementation is [`UnwiredVmOrchestrator`] (refuses); the generic +//! [`VmBackedRunner`] turns inspect / evaluate into VM jobs and is only +//! ever registered by an operator. +//! 5. [`decide_promote`] — pass + green checklist + relative win over the +//! bar, direction from the topic. +//! +//! Persistence (`proof-rlm-store`), the artefact store, and the +//! `LiveScorer` glue (`proof-rlm-scorer`) live next door. The topic schema +//! (constraints, `eval_executor`, checklist vector, custom id shape) lives +//! in `proof-task`; the live `EvalExecutorOffer` is owned elsewhere. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +mod gate; +mod promote; +mod rules; +mod runner; +mod state; +mod vm; + +/// Shared test fixtures (fake orchestrator, canned report, placeholder topic). +/// Test builds and the `test-fixtures` feature only; never part of a host binary. +#[cfg(any(test, feature = "test-fixtures"))] +#[path = "fixtures_tests.rs"] +pub mod fixtures; + +pub use gate::{authorize_spend, GateError, SpendToken}; +pub use promote::{decide_promote, KeepReason, PromoteDecision}; +pub use rules::{ + CheckItem, Checklist, ChecklistError, RuleSet, RuleSource, CHECKLIST_SCHEMA, MAX_EVIDENCE_LEN, +}; +pub use runner::{ + ArtifactFile, CustomRunReport, CustomRunRequest, CustomRunner, InspectOutcome, JudgeRef, + LogFile, ReportError, RunOutcome, RunnerError, RunnerRegistry, SandboxPolicy, + RUN_REPORT_SCHEMA, RUN_REQUEST_SCHEMA, +}; +pub use state::{ + await_owner_keys, owner_presend, transition, FileKeysProbe, HookError, Lifecycle, NoOwnerHook, + OwnerDecision, OwnerHook, OwnerKeysProbe, OwnerPrompt, RlmEvent, RlmState, StateError, + StaticOwnerHook, Transition, OWNER_INFERENCE_KEY_FILE_ENV, +}; +pub use vm::{ + RetainPolicy, TopicVmOrchestrator, TopicVmSpec, UnwiredVmOrchestrator, VmBackedRunner, VmError, + VmHandle, VmJob, VmJobOutput, VmTemplate, RLM_VM_IMAGE_DIGEST_ENV, + VM_ORCHESTRATOR_TOKEN_FILE_ENV, VM_ORCHESTRATOR_URL_ENV, +}; + +#[cfg(test)] +mod tests { + use super::*; + + /// The crate compiles no challenge: no benchmark, model, repository, or + /// topic id appears in its non-test source. + #[test] + fn no_challenge_content_is_compiled_in() { + let sources = [ + include_str!("gate.rs"), + include_str!("promote.rs"), + include_str!("rules.rs"), + include_str!("runner.rs"), + include_str!("state.rs"), + include_str!("vm.rs"), + include_str!("lib.rs"), + ]; + for src in sources { + let non_test = src.split("#[cfg(test)]").next().unwrap_or(""); + let lower = non_test.to_ascii_lowercase(); + for forbidden in [ + "terminal-bench", + "terminal bench", + "tb4", + "kimi", + "openrouter", + "cortexlm/", + "pass_rate", + "harness_success", + ] { + assert!(!lower.contains(forbidden), "{forbidden:?} is compiled in"); + } + } + assert!(RunnerRegistry::new().is_empty()); + assert_eq!(RlmState::ORDER.len(), 9); + } +} diff --git a/crates/proof-rlm/src/promote.rs b/crates/proof-rlm/src/promote.rs new file mode 100644 index 000000000..bb1346af6 --- /dev/null +++ b/crates/proof-rlm/src/promote.rs @@ -0,0 +1,186 @@ +//! Promotion rule: the best artefact is promoted when a **passing** run's +//! primary beats the current bar by the topic's relative epsilon **and** the +//! anti-cheat checklist was green. The bar is the sealed baseline or the +//! reigning best, whichever is better (`proof_score::novelty_bar`). Direction +//! comes from the topic, so a `min` metric promotes on a lower primary. + +use proof_score::relative_win; +use proof_task::MetricDirection; +use serde::{Deserialize, Serialize}; + +/// Why a run was not promoted. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KeepReason { + /// The harness gates did not pass. + NotPassed, + /// The checklist was red or incomplete (ids listed). + ChecklistRed(Vec), + /// The report did not yield a primary. + NoPrimary, + /// Nothing sealed and no best: no bar to beat (fail-closed). + NoBar, + /// Passed, but not by `epsilon_rel` over the bar. + BelowBar { + /// Measured primary. + primary: f64, + /// Bar it had to clear. + bar: f64, + /// Required relative win. + epsilon_rel: f64, + }, +} + +/// Promote or keep. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PromoteDecision { + /// Crown this run; its artefact becomes the topic's best. + Promote { + /// Measured primary. + primary: f64, + /// Bar it cleared. + bar: f64, + }, + /// Keep the current best. + Keep(KeepReason), +} + +impl PromoteDecision { + /// Whether this is a promotion. + #[must_use] + pub fn is_promote(&self) -> bool { + matches!(self, Self::Promote { .. }) + } +} + +/// Decide promotion for one scored run. +/// +/// `pass` is the harness verdict, `checklist_red` the red rule ids (empty and +/// `checklist_complete` for green), `primary` the report's primary, `bar` +/// the current novelty bar, `direction` / `epsilon_rel` the topic's. +#[must_use] +pub fn decide_promote( + pass: bool, + checklist_complete: bool, + checklist_red: &[String], + primary: Option, + bar: Option, + direction: MetricDirection, + epsilon_rel: f64, +) -> PromoteDecision { + if !pass { + return PromoteDecision::Keep(KeepReason::NotPassed); + } + if !checklist_complete || !checklist_red.is_empty() { + return PromoteDecision::Keep(KeepReason::ChecklistRed(checklist_red.to_vec())); + } + let Some(primary) = primary.filter(|p| p.is_finite()) else { + return PromoteDecision::Keep(KeepReason::NoPrimary); + }; + let Some(bar) = bar.filter(|b| b.is_finite()) else { + return PromoteDecision::Keep(KeepReason::NoBar); + }; + if relative_win(primary, bar, direction, epsilon_rel) { + PromoteDecision::Promote { primary, bar } + } else { + PromoteDecision::Keep(KeepReason::BelowBar { + primary, + bar, + epsilon_rel, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MAX: MetricDirection = MetricDirection::Max; + + #[test] + fn promote_needs_pass_green_and_a_relative_win_over_the_bar() { + assert_eq!( + decide_promote(true, true, &[], Some(0.60), Some(0.50), MAX, 0.02), + PromoteDecision::Promote { + primary: 0.60, + bar: 0.50 + } + ); + assert!(decide_promote(true, true, &[], Some(0.55), Some(0.50), MAX, 0.02).is_promote()); + assert_eq!( + decide_promote(true, true, &[], Some(0.50), Some(0.50), MAX, 0.02), + PromoteDecision::Keep(KeepReason::BelowBar { + primary: 0.50, + bar: 0.50, + epsilon_rel: 0.02 + }) + ); + assert_eq!( + decide_promote(false, true, &[], Some(0.9), Some(0.5), MAX, 0.02), + PromoteDecision::Keep(KeepReason::NotPassed) + ); + assert_eq!( + decide_promote(true, true, &[], None, Some(0.5), MAX, 0.02), + PromoteDecision::Keep(KeepReason::NoPrimary) + ); + assert_eq!( + decide_promote(true, true, &[], Some(0.9), None, MAX, 0.02), + PromoteDecision::Keep(KeepReason::NoBar) + ); + } + + /// Checklist green is a hard condition even when the numbers win. + #[test] + fn a_red_or_incomplete_checklist_never_promotes() { + let red = vec!["rule_b".to_owned()]; + assert_eq!( + decide_promote(true, true, &red, Some(1.0), Some(0.1), MAX, 0.02), + PromoteDecision::Keep(KeepReason::ChecklistRed(red.clone())) + ); + assert_eq!( + decide_promote(true, false, &[], Some(1.0), Some(0.1), MAX, 0.02), + PromoteDecision::Keep(KeepReason::ChecklistRed(Vec::new())) + ); + } + + /// Direction comes from the topic; a zero bar is unbeatable relatively. + #[test] + fn direction_is_topic_data_and_a_zero_bar_is_not_beatable() { + assert!(decide_promote( + true, + true, + &[], + Some(2.0), + Some(3.0), + MetricDirection::Min, + 0.05 + ) + .is_promote()); + assert!(!decide_promote( + true, + true, + &[], + Some(3.0), + Some(2.0), + MetricDirection::Min, + 0.05 + ) + .is_promote()); + assert!(matches!( + decide_promote(true, true, &[], Some(0.5), Some(0.0), MAX, 0.02), + PromoteDecision::Keep(KeepReason::BelowBar { .. }) + )); + let json = serde_json::to_string(&decide_promote( + true, + true, + &[], + Some(0.6), + Some(0.5), + MAX, + 0.02, + )) + .expect("json"); + assert!(json.contains("\"promote\""), "{json}"); + } +} diff --git a/crates/proof-rlm/src/rules.rs b/crates/proof-rlm/src/rules.rs new file mode 100644 index 000000000..9fd3f79b8 --- /dev/null +++ b/crates/proof-rlm/src/rules.rs @@ -0,0 +1,501 @@ +//! Rule sets and checklists: the anti-cheat gate as **data**. +//! +//! A [`RuleSet`] is a versioned vector of `{id, text}` rules for one topic. +//! Version 1 is the vector the signed topic document carries; later versions +//! are whatever the topic's RLM writes (persisted by the store, never +//! compiled in). A [`Checklist`] is one inspection: `{id, pass, evidence}` +//! per rule of a named version. It is green only when every rule of that +//! version is present exactly once with evidence and `pass: true`. A missing +//! rule, an unknown id, a duplicate, an evidence-less pass, or a red item is +//! not green, and nothing spends behind a checklist that is not green. + +use proof_canon::{canonical_json, is_custom_id}; +use proof_task::{ChecklistRule, TopicDocument, MAX_CHECKLIST_RULES, MAX_RULE_TEXT_LEN}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Only accepted `schema_version` of `checklist.json`. +pub const CHECKLIST_SCHEMA: u32 = 1; + +/// Longest evidence string kept per item (ingest truncates, verify refuses longer). +pub const MAX_EVIDENCE_LEN: usize = 2_048; + +/// Who wrote a rule version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RuleSource { + /// The vector carried by the signed topic document (version 1). + TopicDocument, + /// Rewritten by the topic's RLM inside its VM, persisted by the store. + Rlm, + /// Operator edit. + Operator, +} + +/// One versioned rule vector for one topic. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RuleSet { + /// Topic id. + pub topic_id: String, + /// Monotonic version (1 = the signed document's vector). + pub version: u32, + /// Who wrote it. + pub source: RuleSource, + /// The rules, in evaluation order. + pub rules: Vec, +} + +/// Why a rule set or checklist is not usable. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ChecklistError { + /// JSON did not parse. + #[error("parse: {0}")] + Parse(String), + /// Schema drift. + #[error("checklist schema_version {got}, this build reads {CHECKLIST_SCHEMA}")] + WrongSchema { + /// What the document said. + got: u32, + }, + /// A rule vector is malformed (shape mirrors the topic validator). + #[error("rule set: {0}")] + BadRules(String), + /// A rule set with no rules cannot authorise anything. + #[error("rule set is empty; nothing to verify, nothing to spend behind")] + NoRules, + /// The checklist names another topic / version than the rules. + #[error("checklist is for another topic or rule version")] + WrongRuleSet, + /// A rule was never ticked. Absence of evidence is a failed gate. + #[error("checklist item {0:?} missing")] + Missing(String), + /// An item names a rule the version does not have. + #[error("checklist item {0:?} is not a rule of this version")] + Unknown(String), + /// A rule was ticked twice. + #[error("checklist item {0:?} duplicated")] + Duplicate(String), + /// A pass with nothing to show for it, or an oversized blob. + #[error("checklist item {0:?} has empty or oversized evidence")] + BadEvidence(String), + /// At least one item failed. No paid inference. + #[error("checklist red: {}", .0.join(", "))] + Failed(Vec), +} + +fn validate_rules(rules: &[ChecklistRule]) -> Result<(), ChecklistError> { + if rules.is_empty() { + return Err(ChecklistError::NoRules); + } + if rules.len() > MAX_CHECKLIST_RULES { + return Err(ChecklistError::BadRules("too many rules".into())); + } + for (i, r) in rules.iter().enumerate() { + if !is_custom_id(&r.id) { + return Err(ChecklistError::BadRules(format!("bad id {:?}", r.id))); + } + if rules[..i].iter().any(|p| p.id == r.id) { + return Err(ChecklistError::BadRules(format!("duplicate id {:?}", r.id))); + } + let t = r.text.trim(); + if t.is_empty() || t.chars().count() > MAX_RULE_TEXT_LEN { + return Err(ChecklistError::BadRules(format!("bad text for {:?}", r.id))); + } + } + Ok(()) +} + +impl RuleSet { + /// Version 1: the vector the signed topic carries. + /// + /// # Errors + /// + /// [`ChecklistError::NoRules`] / [`ChecklistError::BadRules`]. + pub fn from_topic(doc: &TopicDocument) -> Result { + let set = Self { + topic_id: doc.id.clone(), + version: 1, + source: RuleSource::TopicDocument, + rules: doc.checklist.clone(), + }; + set.validate()?; + Ok(set) + } + + /// A later version written by the RLM or the operator. + /// + /// # Errors + /// + /// [`ChecklistError::BadRules`] when `version` does not advance or the + /// rules are malformed. + pub fn next( + &self, + source: RuleSource, + rules: Vec, + ) -> Result { + let set = Self { + topic_id: self.topic_id.clone(), + version: self.version.saturating_add(1), + source, + rules, + }; + set.validate()?; + Ok(set) + } + + /// Shape check (same rules as the topic validator, plus non-empty). + /// + /// # Errors + /// + /// [`ChecklistError::NoRules`] / [`ChecklistError::BadRules`]. + pub fn validate(&self) -> Result<(), ChecklistError> { + if self.version == 0 { + return Err(ChecklistError::BadRules("version must be >= 1".into())); + } + validate_rules(&self.rules) + } + + /// Rule ids in evaluation order. + #[must_use] + pub fn ids(&self) -> Vec<&str> { + self.rules.iter().map(|r| r.id.as_str()).collect() + } + + /// SHA-256 hex of the canonical JSON (what a checklist binds to). + #[must_use] + pub fn digest(&self) -> String { + domain_digest(b"proof-rlm-rules-v1", self) + } +} + +fn domain_digest(domain: &[u8], value: &T) -> String { + let value = serde_json::to_value(value).unwrap_or(serde_json::Value::Null); + let mut h = Sha256::new(); + h.update(domain); + h.update([0xff]); + h.update(canonical_json(&value).as_bytes()); + hex::encode(h.finalize()) +} + +/// One ticked rule. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckItem { + /// Rule id from the rule set. + pub id: String, + /// Whether it passed. + pub pass: bool, + /// What the inspector saw (file, line, command output). Never a secret. + pub evidence: String, +} + +/// `checklist.json`: one inspection of one submission against one rule version. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Checklist { + /// Must equal [`CHECKLIST_SCHEMA`]. + pub schema_version: u32, + /// Topic the artefact was inspected for. + pub topic_id: String, + /// Rule version the items tick. + pub rules_version: u32, + /// Digest of that rule set, so a checklist cannot be replayed against edited rules. + pub rules_digest: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Artefact digest that was inspected. + pub artifact_digest: String, + /// Items, ideally in rule order. + pub items: Vec, +} + +impl Checklist { + /// Empty checklist for one submission against `rules`. + #[must_use] + pub fn new(rules: &RuleSet, submission_digest: &str, artifact_digest: &str) -> Self { + Self { + schema_version: CHECKLIST_SCHEMA, + topic_id: rules.topic_id.clone(), + rules_version: rules.version, + rules_digest: rules.digest(), + submission_digest: submission_digest.to_owned(), + artifact_digest: artifact_digest.to_owned(), + items: Vec::new(), + } + } + + /// Record one item (evidence truncated to [`MAX_EVIDENCE_LEN`]). + pub fn record(&mut self, id: &str, pass: bool, evidence: &str) -> &mut Self { + let mut evidence = evidence.trim().to_owned(); + if evidence.len() > MAX_EVIDENCE_LEN { + let mut cut = MAX_EVIDENCE_LEN; + while !evidence.is_char_boundary(cut) { + cut = cut.saturating_sub(1); + } + evidence.truncate(cut); + } + self.items.push(CheckItem { + id: id.to_owned(), + pass, + evidence, + }); + self + } + + /// Parse `checklist.json`. + /// + /// # Errors + /// + /// [`ChecklistError::Parse`]. + pub fn from_json(body: &str) -> Result { + serde_json::from_str(body).map_err(|e| ChecklistError::Parse(e.to_string())) + } + + /// Pretty JSON for the artefact bundle. + #[must_use] + pub fn to_json(&self) -> String { + serde_json::to_string_pretty(self).unwrap_or_else(|_| "{}".into()) + } + + /// Structural check against `rules`: schema, same topic + version + + /// digest, every rule exactly once, no unknown ids, evidence present. + /// + /// # Errors + /// + /// See [`ChecklistError`]. + pub fn verify_complete(&self, rules: &RuleSet) -> Result<(), ChecklistError> { + if self.schema_version != CHECKLIST_SCHEMA { + return Err(ChecklistError::WrongSchema { + got: self.schema_version, + }); + } + rules.validate()?; + if self.topic_id != rules.topic_id + || self.rules_version != rules.version + || !self.rules_digest.eq_ignore_ascii_case(&rules.digest()) + { + return Err(ChecklistError::WrongRuleSet); + } + let ids = rules.ids(); + for item in &self.items { + if !ids.contains(&item.id.as_str()) { + return Err(ChecklistError::Unknown(item.id.clone())); + } + } + for id in ids { + let mut seen = 0usize; + for item in self.items.iter().filter(|i| i.id == id) { + seen = seen.saturating_add(1); + let e = item.evidence.trim(); + if e.is_empty() || e.len() > MAX_EVIDENCE_LEN { + return Err(ChecklistError::BadEvidence(id.to_owned())); + } + } + match seen { + 0 => return Err(ChecklistError::Missing(id.to_owned())), + 1 => {} + _ => return Err(ChecklistError::Duplicate(id.to_owned())), + } + } + Ok(()) + } + + /// Complete **and** every item passed. + /// + /// # Errors + /// + /// A structural error, or [`ChecklistError::Failed`] naming every red id. + pub fn verify(&self, rules: &RuleSet) -> Result<(), ChecklistError> { + self.verify_complete(rules)?; + let failed = self.failed_ids(); + if failed.is_empty() { + Ok(()) + } else { + Err(ChecklistError::Failed(failed)) + } + } + + /// Ids recorded as `pass: false`, sorted, deduplicated. + #[must_use] + pub fn failed_ids(&self) -> Vec { + let mut out: Vec = self + .items + .iter() + .filter(|i| !i.pass) + .map(|i| i.id.clone()) + .collect(); + out.sort(); + out.dedup(); + out + } + + /// Whether this checklist authorises spend under `rules`. + #[must_use] + pub fn is_green(&self, rules: &RuleSet) -> bool { + self.verify(rules).is_ok() + } + + /// SHA-256 hex of the canonical JSON; binds a spend token to these exact items. + #[must_use] + pub fn digest(&self) -> String { + domain_digest(b"proof-rlm-checklist-v1", self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::{green, rules, topic}; + + #[test] + fn version_one_is_the_signed_documents_vector() { + let t = topic(); + let set = RuleSet::from_topic(&t).expect("rules"); + assert_eq!(set.version, 1); + assert_eq!(set.source, RuleSource::TopicDocument); + assert_eq!( + set.ids(), + t.checklist + .iter() + .map(|r| r.id.as_str()) + .collect::>() + ); + assert_eq!(set.digest().len(), 64); + let mut none = t.clone(); + none.checklist.clear(); + assert_eq!(RuleSet::from_topic(&none), Err(ChecklistError::NoRules)); + } + + #[test] + fn later_versions_advance_and_are_shape_checked() { + let v1 = rules(); + let v2 = v1 + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rewritten".into(), + text: "the rlm rewrote the rule".into(), + }], + ) + .expect("v2"); + assert_eq!(v2.version, 2); + assert_eq!(v2.source, RuleSource::Rlm); + assert_ne!(v2.digest(), v1.digest()); + assert!(matches!( + v1.next(RuleSource::Operator, Vec::new()), + Err(ChecklistError::NoRules) + )); + assert!(matches!( + v1.next( + RuleSource::Operator, + vec![ChecklistRule { + id: "Bad Id".into(), + text: "x".into() + }] + ), + Err(ChecklistError::BadRules(_)) + )); + let mut zero = v1; + zero.version = 0; + assert!(matches!(zero.validate(), Err(ChecklistError::BadRules(_)))); + } + + #[test] + fn a_complete_all_pass_checklist_is_green_for_its_rule_version() { + let set = rules(); + let c = green(&set, "d"); + c.verify(&set).expect("green"); + assert!(c.is_green(&set)); + assert!(c.failed_ids().is_empty()); + let round = Checklist::from_json(&c.to_json()).expect("round trip"); + assert_eq!(round, c); + assert_eq!(round.digest(), c.digest()); + // The same items are not green for an edited rule version. + let v2 = set.next(RuleSource::Rlm, set.rules.clone()).expect("v2"); + assert_eq!(c.verify(&v2), Err(ChecklistError::WrongRuleSet)); + } + + /// Any red item is red — one is enough, and the error names every red id. + #[test] + fn any_false_item_is_red() { + let set = rules(); + for id in set.ids() { + let mut c = green(&set, "d"); + for item in &mut c.items { + if item.id == id { + item.pass = false; + } + } + assert!(!c.is_green(&set)); + assert_eq!( + c.verify(&set), + Err(ChecklistError::Failed(vec![id.to_owned()])) + ); + } + let mut two = green(&set, "d"); + two.items[0].pass = false; + two.items[1].pass = false; + let mut want = vec![two.items[0].id.clone(), two.items[1].id.clone()]; + want.sort(); + assert_eq!(two.verify(&set), Err(ChecklistError::Failed(want))); + } + + /// Absence of evidence is a failed gate: a missing rule, an unknown id, a + /// duplicate vote, or an evidence-less pass all refuse. + #[test] + fn missing_unknown_duplicate_or_evidence_less_items_refuse() { + let set = rules(); + let first = set.ids()[0].to_owned(); + + let mut missing = green(&set, "d"); + missing.items.retain(|i| i.id != first); + assert_eq!( + missing.verify(&set), + Err(ChecklistError::Missing(first.clone())) + ); + + let mut unknown = green(&set, "d"); + unknown.record("not_a_rule", true, "x"); + assert_eq!( + unknown.verify(&set), + Err(ChecklistError::Unknown("not_a_rule".into())) + ); + + let mut dup = green(&set, "d"); + dup.record(&first, true, "again"); + assert_eq!( + dup.verify(&set), + Err(ChecklistError::Duplicate(first.clone())) + ); + + let mut blank = green(&set, "d"); + blank.items[0].evidence = " ".into(); + assert_eq!(blank.verify(&set), Err(ChecklistError::BadEvidence(first))); + + let mut schema = green(&set, "d"); + schema.schema_version = 2; + assert_eq!( + schema.verify(&set), + Err(ChecklistError::WrongSchema { got: 2 }) + ); + + let empty = Checklist::new(&set, "d", "a"); + assert!(matches!( + empty.verify(&set), + Err(ChecklistError::Missing(_)) + )); + assert!(Checklist::from_json("nope").is_err()); + } + + #[test] + fn evidence_is_truncated_on_record_and_digest_tracks_content() { + let set = rules(); + let first = set.ids()[0].to_owned(); + let mut c = green(&set, "d"); + c.items.retain(|i| i.id != first); + c.record(&first, true, &"é".repeat(MAX_EVIDENCE_LEN)); + assert!(c.items.last().expect("item").evidence.len() <= MAX_EVIDENCE_LEN); + c.verify(&set).expect("truncated evidence still verifies"); + let a = c.digest(); + c.items[0].evidence.push('!'); + assert_ne!(a, c.digest(), "digest must move with the evidence"); + } +} diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs new file mode 100644 index 000000000..89b0bf63d --- /dev/null +++ b/crates/proof-rlm/src/runner.rs @@ -0,0 +1,504 @@ +//! Custom-metric runner contract and the fail-closed registry. +//! +//! A topic names its metric by `custom_id`. A [`CustomRunner`] registered +//! under that id knows how to inspect an artefact against the topic's rule +//! set and, behind a [`SpendToken`], run it and report a `primary_value`. +//! Nothing is registered by default: [`RunnerRegistry::resolve`] on an +//! unknown id is [`RunnerError::Unregistered`], which the host turns into a +//! 503 before any row or rent. No runner in this repository knows a +//! benchmark, a model, or a repository; those are topic data. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use proof_canon::is_custom_id; +use proof_task::{ + Constraints, InferenceOffer, MetricDirection, MetricFamily, ProofPin, TopicDocument, +}; +use serde::{Deserialize, Serialize}; + +use crate::gate::SpendToken; +use crate::rules::{Checklist, RuleSet}; + +/// Only accepted `schema_version` of a run request. +pub const RUN_REQUEST_SCHEMA: u32 = 1; + +/// Only accepted `schema_version` of a run report. +pub const RUN_REPORT_SCHEMA: u32 = 1; + +/// Public fields of the RLM judge offer the runner may show the harness judge. +/// Never the origin, never a key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct JudgeRef { + /// Live judge offer id. + pub offer_id: String, + /// Judge model id. + pub model_ref: String, + /// Judge config commitment (config knobs + origin, hashed). + pub config_commitment: String, +} + +/// Sandbox policy the runner and the topic VM must honour for miner code. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicy { + /// Miner code runs only inside a Firecracker guest under the topic VM. + pub firecracker_required: bool, + /// Wall-clock deadline for one proof run (topic `eval_executor`, else pin). + pub deadline_s: u64, +} + +/// Everything a runner needs to inspect and run one submission once. +/// +/// Every value is copied from the signed topic, the live judge offer, the +/// rule set, and the submission. Nothing is a default of this crate. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CustomRunRequest { + /// Must equal [`RUN_REQUEST_SCHEMA`]. + pub schema_version: u32, + /// Topic id. + pub topic_id: String, + /// Custom metric id the topic names. + pub custom_id: String, + /// Primary metric name the report must fill. + pub primary: String, + /// Improvement direction. + pub direction: MetricDirection, + /// Relative win the topic demands over the bar. + pub epsilon_rel: f64, + /// Frozen submission digest. + pub submission_digest: String, + /// Artefact digest (sha256 of the recipe bytes). + pub artifact_digest: String, + /// Optional locator for the same bytes. + pub artifact_uri: Option, + /// Miner claim (English). + pub claim: String, + /// Topic constraints (sandbox flag, model pin, opaque slice / params). + pub constraints: Constraints, + /// Rule version the checklist must tick. + pub rules_version: u32, + /// Digest of that rule set. + pub rules_digest: String, + /// Seed every paid call must use (topic baseline seed). + pub seed: u64, + /// Public judge reference. + pub judge: JudgeRef, + /// Sandbox policy. + pub sandbox: SandboxPolicy, + /// Executor offer commitment the topic pins, if any. + pub executor_commitment: Option, +} + +/// Why a run did not happen or is not evidence. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RunnerError { + /// No runner registered under this custom id. Root cause for the 503. + #[error("custom metric {0:?} has no registered runner")] + Unregistered(String), + /// A runner exists but its backend (topic VM orchestrator) is not configured. + #[error("runner not wired: {0}")] + NotWired(String), + /// The topic is not a custom-family topic. + #[error("topic {0:?} is not a custom-family topic")] + NotCustom(String), + /// Registry key is not a well-formed custom id. + #[error("custom id {0:?} must match [a-z0-9][a-z0-9_-]{{1,63}}")] + BadCustomId(String), + /// The token covers another submission. + #[error("spend token does not cover this run")] + SpendTokenMismatch, + /// The backend failed. + #[error("runner backend: {0}")] + Backend(String), + /// The runner returned a document that is not evidence. + #[error("run report: {0}")] + Report(#[from] ReportError), +} + +impl CustomRunRequest { + /// Build the request from a custom-family topic, the live judge offer, + /// and the current rule set. + /// + /// # Errors + /// + /// [`RunnerError::NotCustom`] for `nll` / `throughput` topics. + #[allow(clippy::too_many_arguments)] + pub fn from_topic( + topic: &TopicDocument, + pin: &ProofPin, + offer: &InferenceOffer, + rules: &RuleSet, + submission_digest: &str, + artifact_digest: &str, + artifact_uri: Option<&str>, + claim: &str, + ) -> Result { + if topic.metric.family != MetricFamily::Custom { + return Err(RunnerError::NotCustom(topic.id.clone())); + } + Ok(Self { + schema_version: RUN_REQUEST_SCHEMA, + topic_id: topic.id.clone(), + custom_id: topic.metric.custom_id.trim().to_owned(), + primary: topic.metric.primary.trim().to_owned(), + direction: topic.metric.direction, + epsilon_rel: topic.metric.epsilon_rel, + submission_digest: submission_digest.trim().to_owned(), + artifact_digest: artifact_digest.trim().to_ascii_lowercase(), + artifact_uri: artifact_uri.map(|u| u.trim().to_owned()), + claim: claim.to_owned(), + constraints: topic.constraints.clone(), + rules_version: rules.version, + rules_digest: rules.digest(), + seed: topic.baseline.seed, + judge: JudgeRef { + offer_id: offer.offer_id.clone(), + model_ref: offer.config.model_ref.clone(), + config_commitment: offer.config_commitment.clone(), + }, + sandbox: SandboxPolicy { + firecracker_required: topic.constraints.firecracker_required, + deadline_s: topic + .eval_executor + .max_proof_deadline_s + .unwrap_or(pin.max_proof_deadline_s), + }, + executor_commitment: topic.eval_executor.require_offer_commitment.clone(), + }) + } +} + +/// Runner-authored measurement for one submission. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CustomRunReport { + /// Must equal [`RUN_REPORT_SCHEMA`]. + pub schema_version: u32, + /// Topic run for. + pub topic_id: String, + /// Custom metric id. + pub custom_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Artefact digest that was run. + pub artifact_digest: String, + /// Rule version the run was gated by. + pub rules_version: u32, + /// The primary metric value (becomes `custom_value`). + pub primary_value: f64, + /// Whether the miner's claim matches the measured numbers (runner/judge-filled). + pub claim_holds: bool, + /// Whether miner code ran inside the Firecracker guest. + pub sandboxed: bool, + /// Opaque runner evidence (per-task rows, timings). Shipped in the artefact. + pub evidence: BTreeMap, +} + +/// Why a report is not evidence. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ReportError { + /// JSON did not parse. + #[error("parse run report: {0}")] + Parse(String), + /// Schema drift. + #[error("run report schema_version {got}, this build reads {RUN_REPORT_SCHEMA}")] + WrongSchema { + /// What the document said. + got: u32, + }, + /// A binding field does not echo the request. + #[error("run report {0} does not match the run request")] + Mismatch(&'static str), + /// Primary is NaN / infinite. + #[error("run report primary_value is not finite")] + NotFinite, + /// The topic requires Firecracker and the run was not sandboxed. + #[error("run report says miner code ran outside the Firecracker guest")] + NotSandboxed, +} + +impl CustomRunReport { + /// Parse `report.json`. + /// + /// # Errors + /// + /// [`ReportError::Parse`]. + pub fn from_json(body: &str) -> Result { + serde_json::from_str(body).map_err(|e| ReportError::Parse(e.to_string())) + } + + /// Pretty JSON for the artefact bundle. + #[must_use] + pub fn to_json(&self) -> String { + serde_json::to_string_pretty(self).unwrap_or_else(|_| "{}".into()) + } + + /// Bind the report to the request that produced it. + /// + /// # Errors + /// + /// See [`ReportError`]. A report that fails here never becomes a `custom_value`. + pub fn verify(&self, req: &CustomRunRequest) -> Result<(), ReportError> { + if self.schema_version != RUN_REPORT_SCHEMA { + return Err(ReportError::WrongSchema { + got: self.schema_version, + }); + } + let pairs: [(&'static str, bool); 5] = [ + ("topic_id", self.topic_id.trim() == req.topic_id.trim()), + ("custom_id", self.custom_id.trim() == req.custom_id.trim()), + ( + "submission_digest", + self.submission_digest.trim() == req.submission_digest.trim(), + ), + ( + "artifact_digest", + self.artifact_digest + .trim() + .eq_ignore_ascii_case(req.artifact_digest.trim()), + ), + ("rules_version", self.rules_version == req.rules_version), + ]; + if let Some((field, _)) = pairs.iter().find(|(_, ok)| !ok) { + return Err(ReportError::Mismatch(field)); + } + if !self.primary_value.is_finite() { + return Err(ReportError::NotFinite); + } + if req.sandbox.firecracker_required && !self.sandboxed { + return Err(ReportError::NotSandboxed); + } + Ok(()) + } +} + +/// One file of the miner's artefact tree as inspected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactFile { + /// Relative path inside the artefact (`src/main.rs`). + pub path: String, + /// Bytes. + pub bytes: Vec, +} + +/// A log the runner captured (harness stdout, guest console, judge transcript). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LogFile { + /// File name under `logs/` in the artefact (single path segment). + pub name: String, + /// Raw bytes. + pub bytes: Vec, +} + +/// What inspection produced: the ticked checklist and the tree it looked at. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InspectOutcome { + /// One item per rule of the requested version. + pub checklist: Checklist, + /// The artefact tree as inspected (shipped in the artefact zip). + pub artifact: Vec, +} + +/// What a paid run produced. +#[derive(Debug, Clone, PartialEq)] +pub struct RunOutcome { + /// Runner-authored measurement. + pub report: CustomRunReport, + /// Captured logs. + pub logs: Vec, +} + +/// Inspects and runs submissions for one custom metric id. Paid work needs a token. +#[async_trait] +pub trait CustomRunner: Send + Sync { + /// Whether this runner could run right now (fail-closed). + /// + /// # Errors + /// + /// [`RunnerError::NotWired`] when its backend is not configured. + fn ready(&self) -> Result<(), RunnerError>; + + /// Tick every rule of `rules` over the artefact. **No paid inference.** + async fn inspect( + &self, + req: &CustomRunRequest, + rules: &RuleSet, + ) -> Result; + + /// Run the artefact. `spend` must cover the request. + async fn evaluate( + &self, + req: &CustomRunRequest, + spend: &SpendToken, + ) -> Result; +} + +/// `custom_id → runner`. Empty by default; unknown ids fail closed. +#[derive(Default)] +pub struct RunnerRegistry { + runners: BTreeMap>, +} + +impl RunnerRegistry { + /// An empty registry: every custom topic is unscorable until something registers. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Register `runner` under `custom_id` (replaces an earlier registration). + /// + /// # Errors + /// + /// [`RunnerError::BadCustomId`]. + pub fn register( + &mut self, + custom_id: &str, + runner: Arc, + ) -> Result<(), RunnerError> { + let id = custom_id.trim(); + if !is_custom_id(id) { + return Err(RunnerError::BadCustomId(id.to_owned())); + } + self.runners.insert(id.to_owned(), runner); + Ok(()) + } + + /// Builder form of [`Self::register`]; a bad id is dropped with the error + /// surfaced through [`Self::ids`] being unchanged. + #[must_use] + pub fn with(mut self, custom_id: &str, runner: Arc) -> Self { + let _ = self.register(custom_id, runner); + self + } + + /// The runner for `custom_id`. + /// + /// # Errors + /// + /// [`RunnerError::Unregistered`] — the fail-closed default. + pub fn resolve(&self, custom_id: &str) -> Result, RunnerError> { + self.runners + .get(custom_id.trim()) + .cloned() + .ok_or_else(|| RunnerError::Unregistered(custom_id.trim().to_owned())) + } + + /// Registered ids, sorted. + #[must_use] + pub fn ids(&self) -> Vec { + self.runners.keys().cloned().collect() + } + + /// Whether anything is registered. + #[must_use] + pub fn is_empty(&self) -> bool { + self.runners.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::{offer, pin, report_for, request, rules, topic, FakeOrchestrator}; + use crate::vm::{VmBackedRunner, VmTemplate}; + + #[test] + fn the_request_copies_topic_offer_and_rules_and_never_a_secret() { + let req = request(); + let t = topic(); + assert_eq!(req.custom_id, t.metric.custom_id); + assert_eq!(req.constraints, t.constraints); + assert_eq!(req.rules_version, 1); + assert_eq!(req.rules_digest, rules().digest()); + assert_eq!(req.seed, t.baseline.seed); + assert!(req.sandbox.firecracker_required); + assert_eq!(req.sandbox.deadline_s, pin().max_proof_deadline_s); + assert_eq!(req.judge.offer_id, offer().offer_id); + let dump = serde_json::to_string(&req).expect("json"); + assert!( + !dump.contains("127.0.0.1"), + "judge origin must not travel: {dump}" + ); + assert!(!dump.contains("base_url"), "{dump}"); + assert!(!dump.contains("api_key"), "{dump}"); + let mut plain = t; + plain.metric.family = MetricFamily::Nll; + assert!(matches!( + CustomRunRequest::from_topic(&plain, &pin(), &offer(), &rules(), "d", "a", None, ""), + Err(RunnerError::NotCustom(_)) + )); + let mut tight = topic(); + tight.eval_executor.max_proof_deadline_s = Some(600); + let req = + CustomRunRequest::from_topic(&tight, &pin(), &offer(), &rules(), "d", "a", None, "") + .expect("request"); + assert_eq!(req.sandbox.deadline_s, 600); + } + + #[test] + fn a_report_must_echo_the_request_and_honour_the_sandbox() { + let req = request(); + let r = report_for(&req, 0.7); + r.verify(&req).expect("bound"); + let round = CustomRunReport::from_json(&r.to_json()).expect("round trip"); + assert_eq!(round, r); + let mut other = report_for(&req, 0.7); + other.custom_id = "other_metric".into(); + assert_eq!(other.verify(&req), Err(ReportError::Mismatch("custom_id"))); + let mut stale = report_for(&req, 0.7); + stale.rules_version += 1; + assert_eq!( + stale.verify(&req), + Err(ReportError::Mismatch("rules_version")) + ); + let mut nan = report_for(&req, f64::NAN); + nan.primary_value = f64::NAN; + assert_eq!(nan.verify(&req), Err(ReportError::NotFinite)); + let mut host = report_for(&req, 0.7); + host.sandboxed = false; + assert_eq!(host.verify(&req), Err(ReportError::NotSandboxed)); + let mut relaxed = req.clone(); + relaxed.sandbox.firecracker_required = false; + host.verify(&relaxed) + .expect("topic did not require the guest"); + let mut schema = report_for(&req, 0.7); + schema.schema_version = 9; + assert_eq!( + schema.verify(&req), + Err(ReportError::WrongSchema { got: 9 }) + ); + assert!(CustomRunReport::from_json("[]").is_err()); + } + + /// The registry is empty by default and resolves nothing: the + /// fail-closed contract for every custom id nobody registered. + #[test] + fn the_registry_is_empty_by_default_and_unknown_ids_fail_closed() { + let reg = RunnerRegistry::new(); + assert!(reg.is_empty()); + assert!(reg.ids().is_empty()); + for id in ["any_metric", "another_metric", "yet_another_metric"] { + assert_eq!( + reg.resolve(id).err(), + Some(RunnerError::Unregistered(id.into())) + ); + } + let runner: Arc = Arc::new(VmBackedRunner::new( + FakeOrchestrator::new(0.5), + VmTemplate::unpinned(), + )); + let mut reg = RunnerRegistry::new(); + assert_eq!( + reg.register("Bad Id", runner.clone()), + Err(RunnerError::BadCustomId("Bad Id".into())) + ); + reg.register("topic_minted_metric", runner.clone()) + .expect("register"); + assert_eq!(reg.ids(), vec!["topic_minted_metric".to_owned()]); + reg.resolve("topic_minted_metric").expect("resolved"); + assert!(reg.resolve("other").is_err()); + let built = RunnerRegistry::new().with("also_minted", runner); + assert_eq!(built.ids(), vec!["also_minted".to_owned()]); + } +} diff --git a/crates/proof-rlm/src/state.rs b/crates/proof-rlm/src/state.rs new file mode 100644 index 000000000..fc0967771 --- /dev/null +++ b/crates/proof-rlm/src/state.rs @@ -0,0 +1,753 @@ +//! RLM lifecycle for one topic. +//! +//! ```text +//! draft → owner_presend → awaiting_owner_keys → provisioning → baselining +//! → open ⇄ evaluating → promoting → open … → closed +//! ``` +//! +//! The machine is a pure transition table plus two owner hooks: +//! +//! - **`owner_presend`** asks the owner (an `askUser`-style prompt) before +//! anything is sent anywhere. Without a configured hook the machine cannot +//! leave `owner_presend`; a decline returns it to `draft`. +//! - **`awaiting_owner_keys`** waits for the owner's paid-inference key file +//! to be present. The probe reports presence only; the value never enters +//! logs or the store. +//! +//! Nothing here spends, rents, or scores. It records where a topic is and +//! refuses moves the product rules forbid (for example `open` before a +//! sealed baseline). Every transition is meant to be persisted by the store +//! so the topic's history survives a control-plane restart. + +use std::path::{Path, PathBuf}; + +use proof_task::{MetricFamily, TopicDocument, TopicStatus}; +use serde::{Deserialize, Serialize}; + +/// Env var naming the owner's paid-inference key **file** used for the +/// baseline run. Only the name lives in git; the file is operator state and +/// is staged into the topic VM, never read by the control plane. +pub const OWNER_INFERENCE_KEY_FILE_ENV: &str = "PROOF_RLM_OWNER_INFERENCE_KEY_FILE"; + +/// Lifecycle states, in ship order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RlmState { + /// Topic drafted (signed or not, sealed or not). Nothing sent. + Draft, + /// Owner review before any send / spend. Needs an [`OwnerHook`] answer. + OwnerPresend, + /// Owner approved; waiting for the owner key file. + AwaitingOwnerKeys, + /// Topic VM being provisioned through the orchestrator. + Provisioning, + /// RLM writes rules and runs the baseline inside its VM; seals `custom_value`. + Baselining, + /// Accepting submissions and earning emission share. + Open, + /// One submission is being inspected and run. + Evaluating, + /// A pass beat the bar; artefact is being promoted. + Promoting, + /// Frozen. No new submissions, no emission share. + Closed, +} + +impl RlmState { + /// Every state, in ship order. + pub const ORDER: [RlmState; 9] = [ + Self::Draft, + Self::OwnerPresend, + Self::AwaitingOwnerKeys, + Self::Provisioning, + Self::Baselining, + Self::Open, + Self::Evaluating, + Self::Promoting, + Self::Closed, + ]; + + /// Wire name. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Draft => "draft", + Self::OwnerPresend => "owner_presend", + Self::AwaitingOwnerKeys => "awaiting_owner_keys", + Self::Provisioning => "provisioning", + Self::Baselining => "baselining", + Self::Open => "open", + Self::Evaluating => "evaluating", + Self::Promoting => "promoting", + Self::Closed => "closed", + } + } + + /// Parse a wire name. + #[must_use] + pub fn parse(s: &str) -> Option { + Self::ORDER.into_iter().find(|st| st.as_str() == s.trim()) + } + + /// Only `open` takes new submissions. + #[must_use] + pub const fn accepts_submissions(self) -> bool { + matches!(self, Self::Open) + } + + /// `closed` never moves again. + #[must_use] + pub const fn is_terminal(self) -> bool { + matches!(self, Self::Closed) + } +} + +/// Events that move the machine. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RlmEvent { + /// Operator asks for owner review of the draft. + SubmitForReview, + /// Owner answered yes at `owner_presend`. + OwnerApproved, + /// Owner answered no at `owner_presend`; back to draft. + OwnerDeclined, + /// Owner key file is present. + OwnerKeysPresent, + /// Topic VM provisioned; the RLM may write rules and baseline. + Provisioned, + /// Provisioning failed; back to draft for owner review. + ProvisionFailed, + /// Baseline measured and sealed into the topic. + BaselineSealed, + /// Baseline run failed; back to draft. + BaselineFailed, + /// A submission arrived on an open topic. + SubmissionReceived, + /// Verdict persisted; no promotion. + VerdictRecorded, + /// Verdict passed and beat the bar; promotion starts. + PromotionCandidate, + /// Artefact promoted; back to open. + Promoted, + /// Promotion refused (bar moved, artefact missing); back to open. + PromotionRefused, + /// Operator freezes the topic. + Close, +} + +impl RlmEvent { + /// Wire name. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::SubmitForReview => "submit_for_review", + Self::OwnerApproved => "owner_approved", + Self::OwnerDeclined => "owner_declined", + Self::OwnerKeysPresent => "owner_keys_present", + Self::Provisioned => "provisioned", + Self::ProvisionFailed => "provision_failed", + Self::BaselineSealed => "baseline_sealed", + Self::BaselineFailed => "baseline_failed", + Self::SubmissionReceived => "submission_received", + Self::VerdictRecorded => "verdict_recorded", + Self::PromotionCandidate => "promotion_candidate", + Self::Promoted => "promoted", + Self::PromotionRefused => "promotion_refused", + Self::Close => "close", + } + } +} + +/// Why a move was refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum StateError { + /// The table has no edge for this pair. + #[error("illegal transition {} --{event:?}-->", from.as_str())] + Illegal { + /// Current state. + from: RlmState, + /// Event that was applied. + event: RlmEvent, + }, + /// The owner hook could not answer. + #[error("owner hook: {0}")] + Hook(#[from] HookError), + /// The owner key file is absent or empty. + #[error("owner keys not present ({0})")] + KeysMissing(String), + /// The topic is not a custom-family topic (nothing for an RLM to run). + #[error("topic {0:?} is not a custom-family topic")] + NotCustom(String), +} + +/// The transition table. Every edge is explicit; anything else is illegal. +/// +/// # Errors +/// +/// [`StateError::Illegal`] for a pair the table does not name. +pub fn transition(from: RlmState, event: RlmEvent) -> Result { + use RlmEvent as E; + use RlmState as S; + let to = match (from, event) { + (S::Draft, E::SubmitForReview) => S::OwnerPresend, + (S::OwnerPresend, E::OwnerApproved) => S::AwaitingOwnerKeys, + (S::AwaitingOwnerKeys, E::OwnerKeysPresent) => S::Provisioning, + (S::Provisioning, E::Provisioned) => S::Baselining, + (S::Open, E::SubmissionReceived) => S::Evaluating, + (S::Evaluating, E::PromotionCandidate) => S::Promoting, + // Back to draft: the owner said no, or the ceremony failed. + (S::OwnerPresend, E::OwnerDeclined) + | (S::Provisioning, E::ProvisionFailed) + | (S::Baselining, E::BaselineFailed) => S::Draft, + // Back to open: sealed, verdict recorded, or promotion settled. + (S::Baselining, E::BaselineSealed) + | (S::Evaluating, E::VerdictRecorded) + | (S::Promoting, E::Promoted | E::PromotionRefused) => S::Open, + (S::Closed, _) => return Err(StateError::Illegal { from, event }), + (_, E::Close) => S::Closed, + _ => return Err(StateError::Illegal { from, event }), + }; + Ok(to) +} + +/// One recorded move. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Transition { + /// State before. + pub from: RlmState, + /// Event applied. + pub event: RlmEvent, + /// State after. + pub to: RlmState, + /// Operator-readable note (never a secret). + pub note: String, +} + +/// Where one topic is, plus how it got there. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Lifecycle { + /// Topic id. + pub topic_id: String, + /// Current state. + pub state: RlmState, + /// Every applied transition, oldest first. + pub history: Vec, +} + +impl Lifecycle { + /// A fresh draft. + #[must_use] + pub fn draft(topic_id: &str) -> Self { + Self::at(topic_id, RlmState::Draft) + } + + /// Start at `state` with no history (a topic loaded from a signed + /// document that already went through the ceremony off-host). + #[must_use] + pub fn at(topic_id: &str, state: RlmState) -> Self { + Self { + topic_id: topic_id.trim().to_owned(), + state, + history: Vec::new(), + } + } + + /// Lifecycle position implied by a signed topic document's `status`. + /// + /// An `open` document without a sealed baseline never maps to `open`: + /// it sits at `baselining`, because nobody is paid for beating a number + /// nobody measured. + #[must_use] + pub fn from_topic(doc: &TopicDocument) -> Self { + let state = match doc.status { + TopicStatus::Draft => RlmState::Draft, + TopicStatus::Open if doc.baseline.is_sealed() => RlmState::Open, + TopicStatus::Open => RlmState::Baselining, + TopicStatus::Closed => RlmState::Closed, + }; + Self::at(&doc.id, state) + } + + /// Apply one event. + /// + /// # Errors + /// + /// [`StateError::Illegal`]; the state is unchanged on error. + pub fn apply(&mut self, event: RlmEvent, note: &str) -> Result { + let to = transition(self.state, event)?; + self.history.push(Transition { + from: self.state, + event, + to, + note: note.trim().to_owned(), + }); + self.state = to; + Ok(to) + } +} + +/// What the owner is shown at `owner_presend`. Public topic fields only. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OwnerPrompt { + /// Topic id. + pub topic_id: String, + /// The research problem, as signed. + pub statement: String, + /// Custom metric the RLM will compute. + pub custom_id: String, + /// Model every paid call must name, when the topic pins one. + pub model_pin: Option, + /// Opaque task slice, when the topic names one. + pub task_slice: Option, + /// Whether miner code is confined to a Firecracker guest. + pub firecracker_required: bool, + /// Seed shared by baseline and challengers. + pub seed: u64, + /// Anti-cheat rule ids the topic ships (version 1). + pub checklist: Vec, + /// Operator-declared spend cap for the baseline run, if any. + pub spend_cap_usd: Option, +} + +impl OwnerPrompt { + /// Build the prompt from a custom-family topic document. + /// + /// # Errors + /// + /// [`StateError::NotCustom`] for `nll` / `throughput` topics. + pub fn from_topic(doc: &TopicDocument, spend_cap_usd: Option) -> Result { + if doc.metric.family != MetricFamily::Custom { + return Err(StateError::NotCustom(doc.id.clone())); + } + Ok(Self { + topic_id: doc.id.clone(), + statement: doc.statement.clone(), + custom_id: doc.metric.custom_id.trim().to_owned(), + model_pin: doc.constraints.model_pin.clone(), + task_slice: doc.constraints.task_slice.clone(), + firecracker_required: doc.constraints.firecracker_required, + seed: doc.baseline.seed, + checklist: doc.checklist.iter().map(|r| r.id.clone()).collect(), + spend_cap_usd, + }) + } + + /// Human text an `askUser`-style hook can show verbatim. + #[must_use] + pub fn render(&self) -> String { + let cap = self + .spend_cap_usd + .map_or_else(|| "unset".to_owned(), |c| format!("{c:.2} USD")); + format!( + "Proof topic {}: {} Metric {} (seed {}); model pin {}; task slice {}; firecracker \ + required: {}. Anti-cheat rules ticked before any paid inference: {}. Baseline \ + spend cap: {}. Approve provisioning a topic VM and running the baseline?", + self.topic_id, + self.statement.trim(), + self.custom_id, + self.seed, + self.model_pin.as_deref().unwrap_or("none"), + self.task_slice.as_deref().unwrap_or("none"), + self.firecracker_required, + if self.checklist.is_empty() { + "none".to_owned() + } else { + self.checklist.join(", ") + }, + cap + ) + } +} + +/// The owner's answer. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OwnerDecision { + /// Proceed to `awaiting_owner_keys`. + Approve, + /// Back to `draft`. + Decline { + /// Why (operator note). + reason: String, + }, +} + +/// Why a hook could not answer. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum HookError { + /// No hook configured on this host. Fail-closed: `owner_presend` cannot advance. + #[error("no owner hook configured; owner_presend cannot advance")] + NoHook, + /// The hook ran and failed. + #[error("owner hook failed: {0}")] + Failed(String), +} + +/// `askUser`-style hook: show the prompt, return the owner's decision. +pub trait OwnerHook: Send + Sync { + /// Ask the owner. Implementations must not answer on the owner's behalf. + /// + /// # Errors + /// + /// [`HookError`] when no answer could be obtained. + fn ask_owner(&self, prompt: &OwnerPrompt) -> Result; +} + +/// No hook wired: every ask refuses. +pub struct NoOwnerHook; + +impl OwnerHook for NoOwnerHook { + fn ask_owner(&self, _prompt: &OwnerPrompt) -> Result { + Err(HookError::NoHook) + } +} + +/// A decision the owner already recorded out of band (tests, `--owner-approved` flows). +pub struct StaticOwnerHook(pub OwnerDecision); + +impl OwnerHook for StaticOwnerHook { + fn ask_owner(&self, _prompt: &OwnerPrompt) -> Result { + Ok(self.0.clone()) + } +} + +/// Presence probe for the owner key. Presence only, never the value. +pub trait OwnerKeysProbe: Send + Sync { + /// `Ok` when the key is present and non-empty. + /// + /// # Errors + /// + /// [`HookError::Failed`] naming what is missing (a path or env name, never a value). + fn owner_keys_present(&self) -> Result<(), HookError>; +} + +/// Key lives in a file ([`OWNER_INFERENCE_KEY_FILE_ENV`]). +pub struct FileKeysProbe { + /// File to probe. + pub path: PathBuf, +} + +impl FileKeysProbe { + /// Probe `path`. + #[must_use] + pub fn new(path: &Path) -> Self { + Self { + path: path.to_path_buf(), + } + } + + /// Probe the file named by [`OWNER_INFERENCE_KEY_FILE_ENV`], if set. + #[must_use] + pub fn from_env() -> Option { + std::env::var(OWNER_INFERENCE_KEY_FILE_ENV) + .ok() + .map(|p| p.trim().to_owned()) + .filter(|p| !p.is_empty()) + .map(|p| Self::new(Path::new(&p))) + } +} + +impl OwnerKeysProbe for FileKeysProbe { + fn owner_keys_present(&self) -> Result<(), HookError> { + let present = std::fs::read_to_string(&self.path).is_ok_and(|s| !s.trim().is_empty()); + if present { + Ok(()) + } else { + Err(HookError::Failed(format!( + "{} ({}) missing or empty", + OWNER_INFERENCE_KEY_FILE_ENV, + self.path.display() + ))) + } + } +} + +/// Drive `owner_presend`: ask the owner, then approve or decline. +/// +/// # Errors +/// +/// [`StateError::Illegal`] when not at `owner_presend`; [`StateError::Hook`] +/// when the hook could not answer (state unchanged, nothing sent). +pub fn owner_presend( + lc: &mut Lifecycle, + hook: &dyn OwnerHook, + prompt: &OwnerPrompt, +) -> Result { + if lc.state != RlmState::OwnerPresend { + return Err(StateError::Illegal { + from: lc.state, + event: RlmEvent::OwnerApproved, + }); + } + match hook.ask_owner(prompt)? { + OwnerDecision::Approve => lc.apply(RlmEvent::OwnerApproved, "owner approved presend"), + OwnerDecision::Decline { reason } => lc.apply( + RlmEvent::OwnerDeclined, + &format!("owner declined: {reason}"), + ), + } +} + +/// Drive `awaiting_owner_keys`: advance only when the key file is present. +/// +/// # Errors +/// +/// [`StateError::Illegal`] when not at `awaiting_owner_keys`; +/// [`StateError::KeysMissing`] when the probe says no (state unchanged). +pub fn await_owner_keys( + lc: &mut Lifecycle, + probe: &dyn OwnerKeysProbe, +) -> Result { + if lc.state != RlmState::AwaitingOwnerKeys { + return Err(StateError::Illegal { + from: lc.state, + event: RlmEvent::OwnerKeysPresent, + }); + } + probe + .owner_keys_present() + .map_err(|e| StateError::KeysMissing(e.to_string()))?; + lc.apply(RlmEvent::OwnerKeysPresent, "owner key file present") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::topic; + + #[test] + fn the_happy_path_walks_every_state_in_ship_order() { + let mut lc = Lifecycle::draft("topic-a"); + let steps = [ + (RlmEvent::SubmitForReview, RlmState::OwnerPresend), + (RlmEvent::OwnerApproved, RlmState::AwaitingOwnerKeys), + (RlmEvent::OwnerKeysPresent, RlmState::Provisioning), + (RlmEvent::Provisioned, RlmState::Baselining), + (RlmEvent::BaselineSealed, RlmState::Open), + (RlmEvent::SubmissionReceived, RlmState::Evaluating), + (RlmEvent::PromotionCandidate, RlmState::Promoting), + (RlmEvent::Promoted, RlmState::Open), + (RlmEvent::SubmissionReceived, RlmState::Evaluating), + (RlmEvent::VerdictRecorded, RlmState::Open), + (RlmEvent::Close, RlmState::Closed), + ]; + for (event, want) in steps { + assert_eq!(lc.apply(event, "t").expect("legal"), want); + } + assert_eq!(lc.history.len(), steps.len()); + assert!(lc.state.is_terminal()); + assert!(!lc.state.accepts_submissions()); + let names: Vec<&str> = RlmState::ORDER.iter().map(|s| s.as_str()).collect(); + assert_eq!( + names, + [ + "draft", + "owner_presend", + "awaiting_owner_keys", + "provisioning", + "baselining", + "open", + "evaluating", + "promoting", + "closed" + ] + ); + for s in RlmState::ORDER { + assert_eq!(RlmState::parse(s.as_str()), Some(s)); + } + assert_eq!(RlmState::parse("nope"), None); + assert_eq!(RlmEvent::Close.as_str(), "close"); + } + + /// Skipping owner review, keys, provisioning, or the baseline is illegal. + #[test] + fn no_state_can_be_skipped_and_closed_is_terminal() { + let mut lc = Lifecycle::draft("t"); + for event in [ + RlmEvent::OwnerApproved, + RlmEvent::OwnerKeysPresent, + RlmEvent::Provisioned, + RlmEvent::BaselineSealed, + RlmEvent::SubmissionReceived, + RlmEvent::Promoted, + ] { + assert!(matches!( + lc.apply(event, ""), + Err(StateError::Illegal { + from: RlmState::Draft, + .. + }) + )); + } + assert_eq!( + lc.state, + RlmState::Draft, + "a refused move leaves state alone" + ); + assert!(lc.history.is_empty()); + for (from, event) in [ + (RlmState::Open, RlmEvent::BaselineSealed), + (RlmState::Open, RlmEvent::Promoted), + (RlmState::Closed, RlmEvent::SubmissionReceived), + (RlmState::Closed, RlmEvent::Close), + ] { + assert!(matches!( + transition(from, event), + Err(StateError::Illegal { .. }) + )); + } + for s in RlmState::ORDER { + if s != RlmState::Closed { + assert_eq!( + transition(s, RlmEvent::Close), + Ok(RlmState::Closed), + "{s:?}" + ); + } + } + assert_eq!( + transition(RlmState::Provisioning, RlmEvent::ProvisionFailed), + Ok(RlmState::Draft) + ); + assert_eq!( + transition(RlmState::Baselining, RlmEvent::BaselineFailed), + Ok(RlmState::Draft) + ); + assert_eq!( + transition(RlmState::Promoting, RlmEvent::PromotionRefused), + Ok(RlmState::Open) + ); + } + + /// Without a hook the machine cannot leave owner_presend: nothing is + /// sent on the owner's behalf. + #[test] + fn owner_presend_needs_an_answer_and_a_decline_returns_to_draft() { + let t = topic(); + let prompt = OwnerPrompt::from_topic(&t, Some(25.0)).expect("prompt"); + let text = prompt.render(); + for needle in [ + t.id.as_str(), + t.statement.trim(), + t.metric.custom_id.as_str(), + t.constraints.model_pin.as_deref().unwrap_or("none"), + t.constraints.task_slice.as_deref().unwrap_or("none"), + "seed 42", + t.checklist[0].id.as_str(), + "25.00 USD", + ] { + assert!(text.contains(needle), "{needle} missing in {text}"); + } + let mut lc = Lifecycle::draft(&t.id); + assert!(matches!( + owner_presend(&mut lc, &NoOwnerHook, &prompt), + Err(StateError::Illegal { .. }) + )); + lc.apply(RlmEvent::SubmitForReview, "").expect("review"); + assert!(matches!( + owner_presend(&mut lc, &NoOwnerHook, &prompt), + Err(StateError::Hook(HookError::NoHook)) + )); + assert_eq!(lc.state, RlmState::OwnerPresend); + + let decline = StaticOwnerHook(OwnerDecision::Decline { + reason: "budget".into(), + }); + assert_eq!( + owner_presend(&mut lc, &decline, &prompt).expect("declined"), + RlmState::Draft + ); + assert!(lc.history.last().expect("h").note.contains("budget")); + + lc.apply(RlmEvent::SubmitForReview, "") + .expect("review again"); + assert_eq!( + owner_presend(&mut lc, &StaticOwnerHook(OwnerDecision::Approve), &prompt) + .expect("approved"), + RlmState::AwaitingOwnerKeys + ); + } + + #[test] + fn prompts_are_for_custom_topics_only() { + let plain = TopicDocument { + id: "adamw-beater-v0".into(), + ..TopicDocument::default() + }; + assert!(matches!( + OwnerPrompt::from_topic(&plain, None), + Err(StateError::NotCustom(_)) + )); + let mut bare = topic(); + bare.constraints.model_pin = None; + bare.checklist.clear(); + let p = OwnerPrompt::from_topic(&bare, None).expect("prompt"); + let text = p.render(); + assert!(text.contains("model pin none"), "{text}"); + assert!(text.contains("inference: none"), "{text}"); + assert!(text.contains("cap: unset"), "{text}"); + } + + #[test] + fn owner_keys_are_probed_for_presence_only() { + let dir = std::env::temp_dir().join(format!( + "proof-rlm-keys-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let key = dir.join("owner_inference_key"); + let probe = FileKeysProbe::new(&key); + + let mut lc = Lifecycle::at("t", RlmState::AwaitingOwnerKeys); + let err = await_owner_keys(&mut lc, &probe).expect_err("missing file"); + assert!(matches!(err, StateError::KeysMissing(_)), "{err}"); + assert!(err.to_string().contains(OWNER_INFERENCE_KEY_FILE_ENV)); + assert_eq!(lc.state, RlmState::AwaitingOwnerKeys); + + std::fs::write(&key, " \n").expect("write"); + assert!(matches!( + await_owner_keys(&mut lc, &probe), + Err(StateError::KeysMissing(_)) + )); + + std::fs::write(&key, "not-a-real-secret-value\n").expect("write"); + assert_eq!( + await_owner_keys(&mut lc, &probe).expect("present"), + RlmState::Provisioning + ); + let dump = serde_json::to_string(&lc).expect("json"); + assert!( + !dump.contains("not-a-real-secret-value"), + "key value must never be recorded: {dump}" + ); + + let mut wrong = Lifecycle::draft("t"); + assert!(matches!( + await_owner_keys(&mut wrong, &probe), + Err(StateError::Illegal { .. }) + )); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_signed_open_topic_without_a_seal_sits_at_baselining() { + let mut doc = topic(); + doc.baseline.script_sha256.clear(); + doc.status = TopicStatus::Open; + assert_eq!(Lifecycle::from_topic(&doc).state, RlmState::Baselining); + doc.baseline.script_sha256 = "11".repeat(32); + doc.baseline.metrics_commitment = "22".repeat(32); + assert_eq!(Lifecycle::from_topic(&doc).state, RlmState::Open); + doc.status = TopicStatus::Draft; + assert_eq!(Lifecycle::from_topic(&doc).state, RlmState::Draft); + doc.status = TopicStatus::Closed; + assert_eq!(Lifecycle::from_topic(&doc).state, RlmState::Closed); + let json = serde_json::to_string(&Lifecycle::from_topic(&doc)).expect("json"); + assert!(json.contains("\"state\":\"closed\"")); + } +} diff --git a/crates/proof-rlm/src/vm.rs b/crates/proof-rlm/src/vm.rs new file mode 100644 index 000000000..715298a5a --- /dev/null +++ b/crates/proof-rlm/src/vm.rs @@ -0,0 +1,510 @@ +//! Topic-VM orchestrator boundary. +//! +//! Every topic's RLM runs **inside a VM attributed to that topic** — it +//! writes rules, runs the baseline, inspects and runs miner submissions +//! there, and never touches the control-plane host filesystem or secrets. +//! Miner code runs in a Firecracker guest under that VM when the topic says +//! `firecracker_required`. The control plane is the orchestrator: it asks +//! for a VM, hands it [`VmJob`]s (public topic data, digests, rule versions — +//! never host paths, never keys), reads back documents, and tears the VM down +//! or retains it by policy. +//! +//! The only orchestrator shipped here is [`UnwiredVmOrchestrator`]: it +//! refuses every call and names the env vars a live one would read. There is +//! no host-local execution path in this crate — a missing orchestrator is a +//! 503, not a fallback. + +use std::sync::Arc; + +use async_trait::async_trait; +use proof_canon::is_slug; +use proof_task::{ChecklistRule, TopicDocument}; +use serde::{Deserialize, Serialize}; + +use crate::gate::SpendToken; +use crate::rules::RuleSet; +use crate::runner::{ + CustomRunReport, CustomRunRequest, CustomRunner, InspectOutcome, RunOutcome, RunnerError, + SandboxPolicy, +}; + +/// Env var naming the orchestrator base URL (operator state, never git). +pub const VM_ORCHESTRATOR_URL_ENV: &str = "PROOF_VM_ORCHESTRATOR_URL"; + +/// Env var naming the orchestrator bearer **file**. Never logged. +pub const VM_ORCHESTRATOR_TOKEN_FILE_ENV: &str = "PROOF_VM_ORCHESTRATOR_TOKEN_FILE"; + +/// Env var naming the `sha256:` digest of the RLM VM image the orchestrator boots. +pub const RLM_VM_IMAGE_DIGEST_ENV: &str = "PROOF_RLM_VM_IMAGE_DIGEST"; + +/// What happens to a topic VM when the topic leaves service. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RetainPolicy { + /// Destroy the VM; artefacts already live in the topic-scoped store. + Destroy, + /// Keep the VM (and its scratch) for audit. + Retain, +} + +/// Host-independent VM shape: an image digest plus sizes. Operator config. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmTemplate { + /// `sha256:` digest of the RLM VM image. Empty = unpinned = never boots. + pub image_digest: String, + /// vCPUs (1..=64). + pub vcpus: u32, + /// Guest memory in MiB (512..=131072). + pub mem_mib: u32, +} + +impl VmTemplate { + /// A template with no image pin: validates false, so nothing boots. + #[must_use] + pub fn unpinned() -> Self { + Self { + image_digest: String::new(), + vcpus: 2, + mem_mib: 4_096, + } + } + + /// Template from [`RLM_VM_IMAGE_DIGEST_ENV`] (unpinned when unset). + #[must_use] + pub fn from_env() -> Self { + Self { + image_digest: std::env::var(RLM_VM_IMAGE_DIGEST_ENV) + .map(|s| s.trim().to_owned()) + .unwrap_or_default(), + ..Self::unpinned() + } + } + + /// Ranges and digest shape. + /// + /// # Errors + /// + /// [`VmError::Spec`] naming the first bad field. + pub fn validate(&self) -> Result<(), VmError> { + let hex = self + .image_digest + .trim() + .strip_prefix("sha256:") + .unwrap_or(""); + let checks: [(&'static str, bool); 3] = [ + ("image_digest", proof_canon::is_hex64(hex)), + ("vcpus", (1..=64).contains(&self.vcpus)), + ("mem_mib", (512..=131_072).contains(&self.mem_mib)), + ]; + match checks.iter().find(|(_, ok)| !ok) { + Some((field, _)) => Err(VmError::Spec(field)), + None => Ok(()), + } + } +} + +/// What the orchestrator is asked to create for one topic. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TopicVmSpec { + /// Topic the VM is attributed to (one topic ↔ its own VM). + pub topic_id: String, + /// Image + sizes. + pub template: VmTemplate, + /// Sandbox policy for miner code inside the VM. + pub sandbox: SandboxPolicy, + /// What to do with the VM when the topic closes. + pub retain: RetainPolicy, +} + +impl TopicVmSpec { + /// Spec for `topic` from a template and the topic's own sandbox policy. + #[must_use] + pub fn for_topic(topic_id: &str, template: VmTemplate, sandbox: SandboxPolicy) -> Self { + Self { + topic_id: topic_id.trim().to_owned(), + template, + sandbox, + retain: RetainPolicy::Destroy, + } + } + + /// Slug topic id + valid template. + /// + /// # Errors + /// + /// [`VmError::Spec`]. + pub fn validate(&self) -> Result<(), VmError> { + if !is_slug(&self.topic_id) { + return Err(VmError::Spec("topic_id")); + } + self.template.validate() + } +} + +/// One provisioned topic VM. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmHandle { + /// Topic the VM belongs to. + pub topic_id: String, + /// Orchestrator VM id. + pub vm_id: String, +} + +/// Work handed to the RLM inside its VM. Public data only: a job never +/// carries a host path, a key, or an origin. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "job", rename_all = "snake_case")] +pub enum VmJob { + /// Let the RLM read the signed topic and write (or rewrite) its rules. + ProposeRules { + /// The signed topic (public document). + topic: Box, + /// Rule version to supersede, if any. + current_version: Option, + }, + /// Run the baseline artefact so the operator can seal `custom_value`. + Baseline { + /// Request shaped exactly like a miner run. + request: CustomRunRequest, + }, + /// Tick every rule over a miner artefact. No paid inference. + Inspect { + /// The run request. + request: CustomRunRequest, + /// Rules to tick. + rules: RuleSet, + }, + /// Run a miner artefact behind a spend token. + Evaluate { + /// The run request. + request: CustomRunRequest, + /// Digest of the checklist that minted the token (audit binding). + checklist_digest: String, + /// Rule version the token was minted for. + rules_version: u32, + }, + /// Flush the VM's scratch into the topic-scoped artefact store. + Archive { + /// Topic id. + topic_id: String, + }, +} + +/// What a job produced. +#[derive(Debug, Clone, PartialEq)] +pub enum VmJobOutput { + /// Rules the RLM proposes; the store versions them. + Rules(Vec), + /// Baseline measurement. + Baseline(CustomRunReport), + /// Inspection result. + Inspected(InspectOutcome), + /// Paid run result. + Evaluated(RunOutcome), + /// Scratch archived. + Archived, +} + +/// Why the orchestrator refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum VmError { + /// No live orchestrator configured. + #[error("topic-vm orchestrator not wired: {0}")] + NotWired(String), + /// A spec field is missing or out of range. + #[error("topic-vm spec: {0} is missing or out of range")] + Spec(&'static str), + /// The orchestrator failed. + #[error("topic-vm orchestrator: {0}")] + Backend(String), + /// The job's output was not the shape the job asked for. + #[error("topic-vm returned the wrong output for {0}")] + WrongOutput(&'static str), +} + +/// Create / attach / run / teardown for topic VMs. +#[async_trait] +pub trait TopicVmOrchestrator: Send + Sync { + /// Whether the orchestrator is configured (fail-closed). + /// + /// # Errors + /// + /// [`VmError::NotWired`]. + fn ready(&self) -> Result<(), VmError>; + + /// Provision a VM for `spec.topic_id`. + async fn create(&self, spec: &TopicVmSpec) -> Result; + + /// The existing VM for `topic_id`, if any. + async fn attach(&self, topic_id: &str) -> Result, VmError>; + + /// Run one job inside the VM. + async fn run(&self, handle: &VmHandle, job: VmJob) -> Result; + + /// Tear down (or retain) the VM. `Ok(true)` only when the orchestrator + /// confirms the requested end state. + async fn teardown(&self, handle: &VmHandle, policy: RetainPolicy) -> Result; +} + +/// The CI-safe orchestrator: nothing is configured, every call refuses. +pub struct UnwiredVmOrchestrator; + +impl UnwiredVmOrchestrator { + fn refuse() -> VmError { + VmError::NotWired(format!( + "no orchestrator configured ({VM_ORCHESTRATOR_URL_ENV} / {VM_ORCHESTRATOR_TOKEN_FILE_ENV})" + )) + } +} + +#[async_trait] +impl TopicVmOrchestrator for UnwiredVmOrchestrator { + fn ready(&self) -> Result<(), VmError> { + Err(Self::refuse()) + } + + async fn create(&self, _spec: &TopicVmSpec) -> Result { + Err(Self::refuse()) + } + + async fn attach(&self, _topic_id: &str) -> Result, VmError> { + Err(Self::refuse()) + } + + async fn run(&self, _handle: &VmHandle, _job: VmJob) -> Result { + Err(Self::refuse()) + } + + async fn teardown(&self, _handle: &VmHandle, _policy: RetainPolicy) -> Result { + Err(Self::refuse()) + } +} + +fn map_vm(e: VmError) -> RunnerError { + match e { + VmError::NotWired(m) => RunnerError::NotWired(m), + other => RunnerError::Backend(other.to_string()), + } +} + +/// The generic runner: every inspect / evaluate is a job inside the topic's +/// VM. Registering it under a `custom_id` is an operator action; nothing +/// registers it by default. +pub struct VmBackedRunner { + orchestrator: Arc, + template: VmTemplate, +} + +impl VmBackedRunner { + /// Runner over `orchestrator` booting `template` for topics without a VM. + #[must_use] + pub fn new(orchestrator: Arc, template: VmTemplate) -> Self { + Self { + orchestrator, + template, + } + } + + /// The runner an unconfigured host would get: unwired orchestrator, + /// unpinned image. `ready()` names the orchestrator as the root cause. + #[must_use] + pub fn unwired() -> Self { + Self::new(Arc::new(UnwiredVmOrchestrator), VmTemplate::unpinned()) + } + + /// The topic's VM, created on first use. + async fn vm_for(&self, req: &CustomRunRequest) -> Result { + self.ready()?; + if let Some(h) = self + .orchestrator + .attach(&req.topic_id) + .await + .map_err(map_vm)? + { + return Ok(h); + } + let spec = + TopicVmSpec::for_topic(&req.topic_id, self.template.clone(), req.sandbox.clone()); + spec.validate().map_err(map_vm)?; + self.orchestrator.create(&spec).await.map_err(map_vm) + } +} + +#[async_trait] +impl CustomRunner for VmBackedRunner { + fn ready(&self) -> Result<(), RunnerError> { + self.orchestrator.ready().map_err(map_vm)?; + self.template.validate().map_err(map_vm) + } + + async fn inspect( + &self, + req: &CustomRunRequest, + rules: &RuleSet, + ) -> Result { + let vm = self.vm_for(req).await?; + let job = VmJob::Inspect { + request: req.clone(), + rules: rules.clone(), + }; + match self.orchestrator.run(&vm, job).await.map_err(map_vm)? { + VmJobOutput::Inspected(out) => Ok(out), + _ => Err(map_vm(VmError::WrongOutput("inspect"))), + } + } + + async fn evaluate( + &self, + req: &CustomRunRequest, + spend: &SpendToken, + ) -> Result { + if !spend.covers(&req.topic_id, &req.submission_digest) + || spend.rules_version() != req.rules_version + { + return Err(RunnerError::SpendTokenMismatch); + } + let vm = self.vm_for(req).await?; + let job = VmJob::Evaluate { + request: req.clone(), + checklist_digest: spend.checklist_digest().to_owned(), + rules_version: spend.rules_version(), + }; + match self.orchestrator.run(&vm, job).await.map_err(map_vm)? { + VmJobOutput::Evaluated(out) => { + out.report.verify(req)?; + Ok(out) + } + _ => Err(map_vm(VmError::WrongOutput("evaluate"))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::{pinned_template, request, rules, token_for, FakeOrchestrator}; + + #[tokio::test] + async fn the_unwired_runner_refuses_before_any_call() { + let runner = VmBackedRunner::unwired(); + let err = runner.ready().expect_err("unwired"); + assert!(matches!(err, RunnerError::NotWired(_)), "{err}"); + assert!(err.to_string().contains(VM_ORCHESTRATOR_URL_ENV), "{err}"); + let req = request(); + assert!(matches!( + runner.inspect(&req, &rules()).await, + Err(RunnerError::NotWired(_)) + )); + assert!(matches!( + runner.evaluate(&req, &token_for(&req)).await, + Err(RunnerError::NotWired(_)) + )); + assert!(UnwiredVmOrchestrator.ready().is_err()); + } + + #[tokio::test] + async fn an_unpinned_image_over_a_live_orchestrator_still_refuses() { + let runner = VmBackedRunner::new(FakeOrchestrator::new(0.5), VmTemplate::unpinned()); + assert_eq!( + runner.ready(), + Err(RunnerError::Backend( + VmError::Spec("image_digest").to_string() + )) + ); + assert!(VmTemplate::unpinned().validate().is_err()); + pinned_template().validate().expect("pinned"); + let mut tiny = pinned_template(); + tiny.mem_mib = 64; + assert_eq!(tiny.validate(), Err(VmError::Spec("mem_mib"))); + let spec = TopicVmSpec::for_topic( + "Bad Topic", + pinned_template(), + SandboxPolicy { + firecracker_required: true, + deadline_s: 60, + }, + ); + assert_eq!(spec.validate(), Err(VmError::Spec("topic_id"))); + } + + /// One topic, one VM: the first job creates it, later jobs attach. + #[tokio::test] + async fn one_topic_one_vm_and_jobs_carry_public_data_only() { + let orch = FakeOrchestrator::new(0.8); + let runner = VmBackedRunner::new(orch.clone(), pinned_template()); + runner.ready().expect("ready"); + let req = request(); + let inspected = runner.inspect(&req, &rules()).await.expect("inspect"); + assert!(inspected.checklist.is_green(&rules())); + let run = runner + .evaluate(&req, &token_for(&req)) + .await + .expect("evaluate"); + assert!((run.report.primary_value - 0.8).abs() < 1e-12); + assert_eq!( + orch.created(), + 1, + "the second job attached, it did not create" + ); + let jobs = orch.jobs(); + assert_eq!(jobs.len(), 2); + for job in &jobs { + let dump = serde_json::to_string(job).expect("json"); + for forbidden in ["/run/base", "/opt/base", "api_key", "127.0.0.1", "base_url"] { + assert!(!dump.contains(forbidden), "job leaked {forbidden}: {dump}"); + } + } + assert!(matches!(jobs[0], VmJob::Inspect { .. })); + assert!(matches!(jobs[1], VmJob::Evaluate { .. })); + } + + #[tokio::test] + async fn a_token_for_another_submission_or_rule_version_never_runs() { + let orch = FakeOrchestrator::new(0.8); + let runner = VmBackedRunner::new(orch.clone(), pinned_template()); + let req = request(); + let mut other = req.clone(); + other.submission_digest = "digest-b".into(); + assert_eq!( + runner.evaluate(&req, &token_for(&other)).await, + Err(RunnerError::SpendTokenMismatch) + ); + let mut stale = req.clone(); + stale.rules_version = 2; + assert_eq!( + runner.evaluate(&stale, &token_for(&req)).await, + Err(RunnerError::SpendTokenMismatch) + ); + assert!(orch.jobs().is_empty(), "no job before the token checks"); + } + + #[tokio::test] + async fn a_report_that_escaped_the_sandbox_is_not_evidence() { + let orch = FakeOrchestrator::new(0.8); + orch.set_sandboxed(false); + let runner = VmBackedRunner::new(orch, pinned_template()); + let req = request(); + let err = runner + .evaluate(&req, &token_for(&req)) + .await + .expect_err("unsandboxed"); + assert!(matches!( + err, + RunnerError::Report(crate::runner::ReportError::NotSandboxed) + )); + } + + #[test] + fn env_names_are_names_only() { + assert_eq!(VM_ORCHESTRATOR_URL_ENV, "PROOF_VM_ORCHESTRATOR_URL"); + assert_eq!( + VM_ORCHESTRATOR_TOKEN_FILE_ENV, + "PROOF_VM_ORCHESTRATOR_TOKEN_FILE" + ); + assert_eq!(RLM_VM_IMAGE_DIGEST_ENV, "PROOF_RLM_VM_IMAGE_DIGEST"); + assert!( + VmTemplate::from_env().image_digest.is_empty() + || VmTemplate::from_env().validate().is_ok() + ); + } +} From eb141d0e0ebc6b84e59b7a40c3d2a235666682d2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:09:48 +0000 Subject: [PATCH 04/14] feat(db): proof rlm store with postgres and memory backends Migration 0020_proof_rlm.sql: proof_topic_version (signed documents per re-sign), proof_rule_version (RLM-written rule vectors, versioned with the topic, digest-bound), proof_checklist (every inspection, red or green), proof_lifecycle_event, proof_baseline_measurement, proof_artefact (zip metadata), proof_promotion_event (the learning continuum: baseline -> runs -> best -> what the next run must beat). Append-only for base_app; the current best is the newest promotion row. proof-rlm-store: the RlmStore contract, PgRlmStore over crates/db (plain runtime sqlx), and MemoryRlmStore for CI / local. One contract test runs against both; the Postgres half runs on an isolated migrated schema when DATABASE_URL is set and is skipped otherwise. --- crates/db/migrations/0020_proof_rlm.sql | 118 +++++ crates/proof-rlm-store/Cargo.toml | 27 ++ crates/proof-rlm-store/src/lib.rs | 318 +++++++++++++ crates/proof-rlm-store/src/memory.rs | 182 ++++++++ crates/proof-rlm-store/src/pg.rs | 431 ++++++++++++++++++ .../proof-rlm-store/tests/store_contract.rs | 207 +++++++++ 6 files changed, 1283 insertions(+) create mode 100644 crates/db/migrations/0020_proof_rlm.sql create mode 100644 crates/proof-rlm-store/Cargo.toml create mode 100644 crates/proof-rlm-store/src/lib.rs create mode 100644 crates/proof-rlm-store/src/memory.rs create mode 100644 crates/proof-rlm-store/src/pg.rs create mode 100644 crates/proof-rlm-store/tests/store_contract.rs diff --git a/crates/db/migrations/0020_proof_rlm.sql b/crates/db/migrations/0020_proof_rlm.sql new file mode 100644 index 000000000..8117307cb --- /dev/null +++ b/crates/db/migrations/0020_proof_rlm.sql @@ -0,0 +1,118 @@ +-- Proof RLM: DB-backed topic versions, RLM-authored rule versions, +-- per-submission checklists, lifecycle transitions, artefact metadata, and +-- the promotion continuum (best pointer + history). +-- +-- Proof is a dynamic agentic challenge system. Nothing about a challenge is +-- compiled into the binary: every row here is data a signed topic document +-- or that topic's RLM (running in its own VM) produced. Rules the RLM writes +-- land in `proof_rule_version`, versioned with the topic, so the gate a +-- submission was ticked against is replayable months later — not only in +-- logs. `proof_promotion_event` is the learning continuum: baseline → miner +-- runs → best → what the next run has to beat. +-- +-- Append-only tables (`proof_rule_version`, `proof_checklist`, +-- `proof_lifecycle_event`, `proof_promotion_event`, +-- `proof_baseline_measurement`) get INSERT + SELECT only for `base_app`; the +-- current best pointer is the newest promotion row, never an UPDATE. +-- `topic_id` follows the topic slug CHECK; digests are lowercase 64 hex. + +CREATE TABLE proof_topic_version ( + topic_id TEXT NOT NULL, + version INTEGER NOT NULL, -- 1, 2, … per re-sign + status TEXT NOT NULL, -- draft | open | closed (document status) + document JSONB NOT NULL, -- the signed topic document, verbatim + signature TEXT NOT NULL, -- sr25519 hex over the canonical document + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (topic_id, version), + CONSTRAINT proof_topic_version_id_check CHECK (topic_id ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + CONSTRAINT proof_topic_version_status_check CHECK (status IN ('draft', 'open', 'closed')), + CONSTRAINT proof_topic_version_pos CHECK (version >= 1) +); + +CREATE TABLE proof_rule_version ( + topic_id TEXT NOT NULL, + version INTEGER NOT NULL, -- 1 = the signed document's checklist vector + source TEXT NOT NULL, -- topic_document | rlm | operator + rules JSONB NOT NULL, -- [{"id","text"}], evaluation order + digest TEXT NOT NULL, -- sha256 hex over the canonical rule set + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (topic_id, version), + CONSTRAINT proof_rule_version_source_check CHECK (source IN ('topic_document', 'rlm', 'operator')), + CONSTRAINT proof_rule_version_digest_check CHECK (digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT proof_rule_version_pos CHECK (version >= 1) +); + +CREATE TABLE proof_checklist ( + submission_digest TEXT PRIMARY KEY, -- frozen submission digest + topic_id TEXT NOT NULL, + rules_version INTEGER NOT NULL, + green BOOLEAN NOT NULL, -- complete and every rule passed + failed_ids JSONB NOT NULL, -- ["rule_id", …], empty when green + document JSONB NOT NULL, -- checklist.json verbatim + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_checklist_digest_check CHECK (submission_digest ~ '^[0-9a-f]{64}$') +); +CREATE INDEX proof_checklist_topic ON proof_checklist (topic_id, created_at); + +CREATE TABLE proof_lifecycle_event ( + id BIGSERIAL PRIMARY KEY, + topic_id TEXT NOT NULL, + from_state TEXT NOT NULL, + event TEXT NOT NULL, + to_state TEXT NOT NULL, + note TEXT NOT NULL DEFAULT '', -- operator-readable, never a secret + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_lifecycle_event_states_check CHECK ( + from_state IN ('draft','owner_presend','awaiting_owner_keys','provisioning','baselining','open','evaluating','promoting','closed') + AND to_state IN ('draft','owner_presend','awaiting_owner_keys','provisioning','baselining','open','evaluating','promoting','closed') + ) +); +CREATE INDEX proof_lifecycle_event_topic ON proof_lifecycle_event (topic_id, id); + +CREATE TABLE proof_baseline_measurement ( + topic_id TEXT NOT NULL, + rules_version INTEGER NOT NULL, + primary_value DOUBLE PRECISION NOT NULL, -- what the RLM measured before any submission + report JSONB NOT NULL, -- run report verbatim + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (topic_id, rules_version) +); + +CREATE TABLE proof_artefact ( + topic_id TEXT NOT NULL, + submission_id TEXT NOT NULL, -- store row id (pf_ + 16 hex) + submission_digest TEXT NOT NULL, + path TEXT NOT NULL, -- {root}/{topic_id}/{submission_id}.zip + sha256 TEXT NOT NULL, -- of the zip bytes + bytes BIGINT NOT NULL, + primary_value DOUBLE PRECISION, -- NULL on a pre-spend reject + checklist_green BOOLEAN NOT NULL, + promoted BOOLEAN NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (topic_id, submission_id), + CONSTRAINT proof_artefact_row_id_check CHECK (submission_id ~ '^pf_[0-9a-f]{16}$'), + CONSTRAINT proof_artefact_sha_check CHECK (sha256 ~ '^[0-9a-f]{64}$') +); + +CREATE TABLE proof_promotion_event ( + id BIGSERIAL PRIMARY KEY, + topic_id TEXT NOT NULL, + submission_id TEXT NOT NULL, + submission_digest TEXT NOT NULL, + primary_value DOUBLE PRECISION NOT NULL, + bar DOUBLE PRECISION, -- what it had to beat (sealed or previous best) + previous_best TEXT, -- displaced submission_id, NULL for the first crown + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_promotion_event_row_id_check CHECK (submission_id ~ '^pf_[0-9a-f]{16}$') +); +CREATE INDEX proof_promotion_event_topic ON proof_promotion_event (topic_id, id); + +GRANT SELECT, INSERT ON TABLE proof_topic_version TO base_app; +GRANT SELECT, INSERT ON TABLE proof_rule_version TO base_app; +GRANT SELECT, INSERT ON TABLE proof_checklist TO base_app; +GRANT SELECT, INSERT ON TABLE proof_lifecycle_event TO base_app; +GRANT USAGE, SELECT ON SEQUENCE proof_lifecycle_event_id_seq TO base_app; +GRANT SELECT, INSERT ON TABLE proof_baseline_measurement TO base_app; +GRANT SELECT, INSERT ON TABLE proof_artefact TO base_app; +GRANT SELECT, INSERT ON TABLE proof_promotion_event TO base_app; +GRANT USAGE, SELECT ON SEQUENCE proof_promotion_event_id_seq TO base_app; diff --git a/crates/proof-rlm-store/Cargo.toml b/crates/proof-rlm-store/Cargo.toml new file mode 100644 index 000000000..baea31762 --- /dev/null +++ b/crates/proof-rlm-store/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "proof-rlm-store" +description = "Proof RLM persistence: DB-backed topic versions, RLM-authored rule versions, checklists, lifecycle transitions, artefact metadata, and the promotion continuum (Postgres via sqlx, plus an in-memory store for CI/local)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +async-trait = "0.1" +db = { path = "../db" } +proof-rlm = { path = "../proof-rlm" } +proof-task = { path = "../proof-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "postgres", "json"] } +thiserror = "2" + +[dev-dependencies] +db = { path = "../db", features = ["testing"] } +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/proof-rlm-store/src/lib.rs b/crates/proof-rlm-store/src/lib.rs new file mode 100644 index 000000000..fae4b078d --- /dev/null +++ b/crates/proof-rlm-store/src/lib.rs @@ -0,0 +1,318 @@ +//! Proof RLM persistence. +//! +//! Everything a topic's RLM produces that must outlive a process — the signed +//! topic versions, the **rule versions the RLM writes**, every submission's +//! checklist, every lifecycle transition, artefact metadata, the baseline +//! measurement, and the promotion continuum (best pointer + history) — goes +//! through [`RlmStore`]. [`PgRlmStore`] is the production implementation +//! over `crates/db` migration `0020_proof_rlm.sql`; [`MemoryRlmStore`] is the +//! CI / local implementation with the same contract. Rules land here, not +//! only in logs. Nothing here knows a challenge. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +mod memory; +mod pg; + +use async_trait::async_trait; +use proof_rlm::{Checklist, CustomRunReport, Lifecycle, RlmEvent, RlmState, RuleSet}; +use proof_task::TopicDocument; +use serde::{Deserialize, Serialize}; + +pub use memory::MemoryRlmStore; +pub use pg::PgRlmStore; + +/// Store failures. +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + /// Lock poisoned (memory store). + #[error("rlm store lock poisoned")] + Poison, + /// A version did not advance by exactly one (or a first version was not 1). + #[error("rlm store: {0} version must advance by one")] + VersionGap(&'static str), + /// A row is malformed (bad id / digest shape, serialisation). + #[error("rlm store: {0}")] + Malformed(String), + /// Database failure. + #[error("rlm store: db: {0}")] + Db(String), +} + +impl From for StoreError { + fn from(e: sqlx::Error) -> Self { + Self::Db(e.to_string()) + } +} + +/// One persisted checklist. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChecklistRow { + /// Topic id. + pub topic_id: String, + /// Frozen submission digest (primary key). + pub submission_digest: String, + /// Rule version the items tick. + pub rules_version: u32, + /// Complete and every rule passed. + pub green: bool, + /// Red rule ids (empty when green). + pub failed_ids: Vec, + /// The document itself. + pub document: Checklist, +} + +impl ChecklistRow { + /// Row for `checklist` verified against `rules`. + #[must_use] + pub fn from_checklist(checklist: &Checklist, rules: &RuleSet) -> Self { + let complete = checklist.verify_complete(rules).is_ok(); + let failed = checklist.failed_ids(); + Self { + topic_id: checklist.topic_id.clone(), + submission_digest: checklist.submission_digest.clone(), + rules_version: checklist.rules_version, + green: complete && failed.is_empty(), + failed_ids: failed, + document: checklist.clone(), + } + } +} + +/// One lifecycle move for one topic. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransitionRow { + /// Topic id. + pub topic_id: String, + /// State before. + pub from: RlmState, + /// Event applied. + pub event: RlmEvent, + /// State after. + pub to: RlmState, + /// Operator-readable note (never a secret). + pub note: String, +} + +/// What the RLM measured before any submission (learning continuum start). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BaselineRow { + /// Topic id. + pub topic_id: String, + /// Rule version in force. + pub rules_version: u32, + /// Baseline primary. + pub primary_value: f64, + /// Run report verbatim. + pub report: CustomRunReport, +} + +/// Metadata of one artefact zip. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ArtefactRow { + /// Topic id. + pub topic_id: String, + /// Store row id (`pf_` + 16 hex). + pub submission_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Where the zip lives. + pub path: String, + /// SHA-256 hex of the zip bytes. + pub sha256: String, + /// Zip size. + pub bytes: u64, + /// Primary value, when a report exists. + pub primary_value: Option, + /// Whether the checklist was green. + pub checklist_green: bool, + /// Whether the run was promoted. + pub promoted: bool, +} + +/// One promotion: the continuum's "new best" event. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PromotionRow { + /// Topic id. + pub topic_id: String, + /// Winning row id. + pub submission_id: String, + /// Winning frozen digest. + pub submission_digest: String, + /// Winning primary. + pub primary_value: f64, + /// Bar it cleared (sealed baseline or previous best). + pub bar: Option, + /// Displaced best, if any. + pub previous_best: Option, +} + +/// Replay lifecycle rows into a [`Lifecycle`]. +#[must_use] +pub fn replay(topic_id: &str, rows: &[TransitionRow]) -> Option { + let first = rows.first()?; + let mut lc = Lifecycle::at(topic_id, first.from); + for r in rows { + lc.history.push(proof_rlm::Transition { + from: r.from, + event: r.event, + to: r.to, + note: r.note.clone(), + }); + lc.state = r.to; + } + Some(lc) +} + +/// Persistence contract shared by the Postgres and in-memory stores. +#[async_trait] +pub trait RlmStore: Send + Sync { + /// Persist a signed topic document as the next version; returns that version. + async fn put_topic_version(&self, doc: &TopicDocument) -> Result; + /// Newest persisted version of a topic. + async fn latest_topic( + &self, + topic_id: &str, + ) -> Result, StoreError>; + + /// Persist a rule version. Must be `current + 1` (or 1 for the first). + async fn put_rules(&self, rules: &RuleSet) -> Result<(), StoreError>; + /// Newest rule version. + async fn current_rules(&self, topic_id: &str) -> Result, StoreError>; + /// One rule version. + async fn rules_at(&self, topic_id: &str, version: u32) -> Result, StoreError>; + + /// Persist a submission's checklist. + async fn put_checklist(&self, row: &ChecklistRow) -> Result<(), StoreError>; + /// A submission's checklist. + async fn checklist(&self, submission_digest: &str) -> Result, StoreError>; + + /// Append a lifecycle move. + async fn record_transition(&self, row: &TransitionRow) -> Result<(), StoreError>; + /// Replay a topic's lifecycle. + async fn lifecycle(&self, topic_id: &str) -> Result, StoreError>; + + /// Persist the baseline measurement for a rule version. + async fn put_baseline(&self, row: &BaselineRow) -> Result<(), StoreError>; + /// Newest baseline measurement. + async fn baseline(&self, topic_id: &str) -> Result, StoreError>; + + /// Persist artefact metadata. + async fn put_artefact(&self, row: &ArtefactRow) -> Result<(), StoreError>; + /// Every artefact of a topic, oldest first. + async fn artefacts(&self, topic_id: &str) -> Result, StoreError>; + + /// Append a promotion event. + async fn record_promotion(&self, row: &PromotionRow) -> Result<(), StoreError>; + /// The current best (newest promotion), if any. + async fn best(&self, topic_id: &str) -> Result, StoreError>; + /// Promotion history, oldest first. + async fn promotions(&self, topic_id: &str) -> Result, StoreError>; +} + +fn is_row_id(id: &str) -> bool { + id.strip_prefix("pf_") + .is_some_and(|h| h.len() == 16 && h.bytes().all(|b| b.is_ascii_hexdigit())) +} + +fn check_artefact(row: &ArtefactRow) -> Result<(), StoreError> { + if !is_row_id(&row.submission_id) || !proof_canon_hex64(&row.sha256) { + return Err(StoreError::Malformed(format!( + "artefact row {} / {}", + row.submission_id, row.sha256 + ))); + } + Ok(()) +} + +fn check_promotion(row: &PromotionRow) -> Result<(), StoreError> { + if !is_row_id(&row.submission_id) || !row.primary_value.is_finite() { + return Err(StoreError::Malformed(format!( + "promotion row {}", + row.submission_id + ))); + } + Ok(()) +} + +fn proof_canon_hex64(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_hexdigit()) +} + +fn check_rules(rules: &RuleSet, current: Option) -> Result<(), StoreError> { + rules + .validate() + .map_err(|e| StoreError::Malformed(e.to_string()))?; + let want = current.map_or(1, |c| c.saturating_add(1)); + if rules.version != want { + return Err(StoreError::VersionGap("rules")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replay_rebuilds_state_from_rows() { + assert!(replay("t", &[]).is_none()); + let rows = [ + TransitionRow { + topic_id: "t".into(), + from: RlmState::Open, + event: RlmEvent::SubmissionReceived, + to: RlmState::Evaluating, + note: "a".into(), + }, + TransitionRow { + topic_id: "t".into(), + from: RlmState::Evaluating, + event: RlmEvent::VerdictRecorded, + to: RlmState::Open, + note: "b".into(), + }, + ]; + let lc = replay("t", &rows).expect("lifecycle"); + assert_eq!(lc.state, RlmState::Open); + assert_eq!(lc.history.len(), 2); + assert_eq!(lc.history[0].note, "a"); + } + + #[test] + fn row_shapes_are_checked_before_any_write() { + let good = ArtefactRow { + topic_id: "t".into(), + submission_id: "pf_0000000000000001".into(), + submission_digest: "d".into(), + path: "/x".into(), + sha256: "ab".repeat(32), + bytes: 1, + primary_value: None, + checklist_green: false, + promoted: false, + }; + check_artefact(&good).expect("good"); + let mut bad = good.clone(); + bad.submission_id = "nope".into(); + assert!(check_artefact(&bad).is_err()); + bad = good; + bad.sha256 = "zz".into(); + assert!(check_artefact(&bad).is_err()); + let promo = PromotionRow { + topic_id: "t".into(), + submission_id: "pf_0000000000000001".into(), + submission_digest: "d".into(), + primary_value: f64::NAN, + bar: None, + previous_best: None, + }; + assert!(check_promotion(&promo).is_err()); + } +} diff --git a/crates/proof-rlm-store/src/memory.rs b/crates/proof-rlm-store/src/memory.rs new file mode 100644 index 000000000..f21f65775 --- /dev/null +++ b/crates/proof-rlm-store/src/memory.rs @@ -0,0 +1,182 @@ +//! In-memory [`RlmStore`] for CI / local hosts. Same contract as Postgres, +//! no durability: a host that boots without `BASE_DATABASE_URL` logs that +//! rules, checklists, transitions, and promotions will not survive a restart. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use proof_rlm::{Lifecycle, RuleSet}; +use proof_task::TopicDocument; + +use crate::{ + check_artefact, check_promotion, check_rules, replay, ArtefactRow, BaselineRow, ChecklistRow, + PromotionRow, RlmStore, StoreError, TransitionRow, +}; + +#[derive(Default)] +struct Inner { + topics: BTreeMap>, + rules: BTreeMap>, + checklists: BTreeMap, + transitions: BTreeMap>, + baselines: BTreeMap>, + artefacts: BTreeMap>, + promotions: BTreeMap>, +} + +/// In-memory store. +#[derive(Clone, Default)] +pub struct MemoryRlmStore { + inner: Arc>, +} + +impl MemoryRlmStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> Result, StoreError> { + self.inner.lock().map_err(|_| StoreError::Poison) + } +} + +#[async_trait] +impl RlmStore for MemoryRlmStore { + async fn put_topic_version(&self, doc: &TopicDocument) -> Result { + let mut g = self.lock()?; + let versions = g.topics.entry(doc.id.clone()).or_default(); + versions.push(doc.clone()); + u32::try_from(versions.len()).map_err(|_| StoreError::VersionGap("topic")) + } + + async fn latest_topic( + &self, + topic_id: &str, + ) -> Result, StoreError> { + let g = self.lock()?; + Ok(g.topics.get(topic_id).and_then(|v| { + v.last() + .cloned() + .map(|d| (u32::try_from(v.len()).unwrap_or(u32::MAX), d)) + })) + } + + async fn put_rules(&self, rules: &RuleSet) -> Result<(), StoreError> { + let mut g = self.lock()?; + let versions = g.rules.entry(rules.topic_id.clone()).or_default(); + check_rules(rules, versions.last().map(|r| r.version))?; + versions.push(rules.clone()); + Ok(()) + } + + async fn current_rules(&self, topic_id: &str) -> Result, StoreError> { + Ok(self + .lock()? + .rules + .get(topic_id) + .and_then(|v| v.last().cloned())) + } + + async fn rules_at(&self, topic_id: &str, version: u32) -> Result, StoreError> { + Ok(self + .lock()? + .rules + .get(topic_id) + .and_then(|v| v.iter().find(|r| r.version == version).cloned())) + } + + async fn put_checklist(&self, row: &ChecklistRow) -> Result<(), StoreError> { + self.lock()? + .checklists + .insert(row.submission_digest.clone(), row.clone()); + Ok(()) + } + + async fn checklist(&self, submission_digest: &str) -> Result, StoreError> { + Ok(self.lock()?.checklists.get(submission_digest).cloned()) + } + + async fn record_transition(&self, row: &TransitionRow) -> Result<(), StoreError> { + self.lock()? + .transitions + .entry(row.topic_id.clone()) + .or_default() + .push(row.clone()); + Ok(()) + } + + async fn lifecycle(&self, topic_id: &str) -> Result, StoreError> { + let g = self.lock()?; + Ok(g.transitions + .get(topic_id) + .and_then(|rows| replay(topic_id, rows))) + } + + async fn put_baseline(&self, row: &BaselineRow) -> Result<(), StoreError> { + if !row.primary_value.is_finite() { + return Err(StoreError::Malformed("baseline primary".into())); + } + let mut g = self.lock()?; + let rows = g.baselines.entry(row.topic_id.clone()).or_default(); + rows.retain(|b| b.rules_version != row.rules_version); + rows.push(row.clone()); + rows.sort_by_key(|b| b.rules_version); + Ok(()) + } + + async fn baseline(&self, topic_id: &str) -> Result, StoreError> { + Ok(self + .lock()? + .baselines + .get(topic_id) + .and_then(|v| v.last().cloned())) + } + + async fn put_artefact(&self, row: &ArtefactRow) -> Result<(), StoreError> { + check_artefact(row)?; + let mut g = self.lock()?; + let rows = g.artefacts.entry(row.topic_id.clone()).or_default(); + rows.retain(|a| a.submission_id != row.submission_id); + rows.push(row.clone()); + Ok(()) + } + + async fn artefacts(&self, topic_id: &str) -> Result, StoreError> { + Ok(self + .lock()? + .artefacts + .get(topic_id) + .cloned() + .unwrap_or_default()) + } + + async fn record_promotion(&self, row: &PromotionRow) -> Result<(), StoreError> { + check_promotion(row)?; + self.lock()? + .promotions + .entry(row.topic_id.clone()) + .or_default() + .push(row.clone()); + Ok(()) + } + + async fn best(&self, topic_id: &str) -> Result, StoreError> { + Ok(self + .lock()? + .promotions + .get(topic_id) + .and_then(|v| v.last().cloned())) + } + + async fn promotions(&self, topic_id: &str) -> Result, StoreError> { + Ok(self + .lock()? + .promotions + .get(topic_id) + .cloned() + .unwrap_or_default()) + } +} diff --git a/crates/proof-rlm-store/src/pg.rs b/crates/proof-rlm-store/src/pg.rs new file mode 100644 index 000000000..038501321 --- /dev/null +++ b/crates/proof-rlm-store/src/pg.rs @@ -0,0 +1,431 @@ +//! Postgres [`RlmStore`] over `crates/db` migration `0020_proof_rlm.sql`. +//! +//! Plain runtime `sqlx::query` (no compile-time database). Append-only +//! tables are inserted, never updated; "current" is always the newest row. + +use async_trait::async_trait; +use db::PgPool; +use proof_rlm::{Checklist, CustomRunReport, Lifecycle, RlmEvent, RlmState, RuleSet, RuleSource}; +use proof_task::{ChecklistRule, TopicDocument}; +use serde_json::Value; + +use crate::{ + check_artefact, check_promotion, check_rules, replay, ArtefactRow, BaselineRow, ChecklistRow, + PromotionRow, RlmStore, StoreError, TransitionRow, +}; + +/// Postgres-backed store. +#[derive(Clone)] +pub struct PgRlmStore { + pool: PgPool, +} + +impl PgRlmStore { + /// Store over an already-migrated pool. + #[must_use] + pub fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// The pool. + #[must_use] + pub fn pool(&self) -> &PgPool { + &self.pool + } +} + +fn malformed(e: E) -> StoreError { + StoreError::Malformed(e.to_string()) +} + +fn to_u32(v: i32) -> Result { + u32::try_from(v).map_err(malformed) +} + +fn to_i32(v: u32) -> Result { + i32::try_from(v).map_err(malformed) +} + +fn source_str(s: RuleSource) -> &'static str { + match s { + RuleSource::TopicDocument => "topic_document", + RuleSource::Rlm => "rlm", + RuleSource::Operator => "operator", + } +} + +fn parse_source(s: &str) -> Result { + match s { + "topic_document" => Ok(RuleSource::TopicDocument), + "rlm" => Ok(RuleSource::Rlm), + "operator" => Ok(RuleSource::Operator), + other => Err(StoreError::Malformed(format!("rule source {other:?}"))), + } +} + +fn parse_state(s: &str) -> Result { + RlmState::parse(s).ok_or_else(|| StoreError::Malformed(format!("state {s:?}"))) +} + +fn parse_event(s: &str) -> Result { + serde_json::from_value(Value::String(s.to_owned())).map_err(malformed) +} + +#[derive(sqlx::FromRow)] +struct RuleRow { + topic_id: String, + version: i32, + source: String, + rules: Value, +} + +impl RuleRow { + fn into_set(self) -> Result { + let rules: Vec = serde_json::from_value(self.rules).map_err(malformed)?; + Ok(RuleSet { + topic_id: self.topic_id, + version: to_u32(self.version)?, + source: parse_source(&self.source)?, + rules, + }) + } +} + +#[derive(sqlx::FromRow)] +struct ChecklistDbRow { + topic_id: String, + submission_digest: String, + rules_version: i32, + green: bool, + failed_ids: Value, + document: Value, +} + +#[derive(sqlx::FromRow)] +struct TransitionDbRow { + topic_id: String, + from_state: String, + event: String, + to_state: String, + note: String, +} + +#[derive(sqlx::FromRow)] +struct BaselineDbRow { + topic_id: String, + rules_version: i32, + primary_value: f64, + report: Value, +} + +#[derive(sqlx::FromRow)] +struct ArtefactDbRow { + topic_id: String, + submission_id: String, + submission_digest: String, + path: String, + sha256: String, + bytes: i64, + primary_value: Option, + checklist_green: bool, + promoted: bool, +} + +#[derive(sqlx::FromRow)] +struct PromotionDbRow { + topic_id: String, + submission_id: String, + submission_digest: String, + primary_value: f64, + bar: Option, + previous_best: Option, +} + +impl From for PromotionRow { + fn from(r: PromotionDbRow) -> Self { + Self { + topic_id: r.topic_id, + submission_id: r.submission_id, + submission_digest: r.submission_digest, + primary_value: r.primary_value, + bar: r.bar, + previous_best: r.previous_best, + } + } +} + +#[async_trait] +impl RlmStore for PgRlmStore { + async fn put_topic_version(&self, doc: &TopicDocument) -> Result { + let document = serde_json::to_value(doc).map_err(malformed)?; + let status = serde_json::to_value(doc.status) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .unwrap_or_default(); + let version: i32 = sqlx::query_scalar( + "INSERT INTO proof_topic_version (topic_id, version, status, document, signature) \ + VALUES ($1, (SELECT COALESCE(MAX(version), 0) + 1 FROM proof_topic_version WHERE topic_id = $1), $2, $3, $4) \ + RETURNING version", + ) + .bind(&doc.id) + .bind(status) + .bind(document) + .bind(&doc.signature) + .fetch_one(&self.pool) + .await?; + to_u32(version) + } + + async fn latest_topic( + &self, + topic_id: &str, + ) -> Result, StoreError> { + let row: Option<(i32, Value)> = sqlx::query_as( + "SELECT version, document FROM proof_topic_version WHERE topic_id = $1 \ + ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + row.map(|(v, doc)| Ok((to_u32(v)?, serde_json::from_value(doc).map_err(malformed)?))) + .transpose() + } + + async fn put_rules(&self, rules: &RuleSet) -> Result<(), StoreError> { + let current = self + .current_rules(&rules.topic_id) + .await? + .map(|r| r.version); + check_rules(rules, current)?; + sqlx::query( + "INSERT INTO proof_rule_version (topic_id, version, source, rules, digest) \ + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&rules.topic_id) + .bind(to_i32(rules.version)?) + .bind(source_str(rules.source)) + .bind(serde_json::to_value(&rules.rules).map_err(malformed)?) + .bind(rules.digest()) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn current_rules(&self, topic_id: &str) -> Result, StoreError> { + let row: Option = sqlx::query_as( + "SELECT topic_id, version, source, rules FROM proof_rule_version \ + WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + row.map(RuleRow::into_set).transpose() + } + + async fn rules_at(&self, topic_id: &str, version: u32) -> Result, StoreError> { + let row: Option = sqlx::query_as( + "SELECT topic_id, version, source, rules FROM proof_rule_version \ + WHERE topic_id = $1 AND version = $2", + ) + .bind(topic_id) + .bind(to_i32(version)?) + .fetch_optional(&self.pool) + .await?; + row.map(RuleRow::into_set).transpose() + } + + async fn put_checklist(&self, row: &ChecklistRow) -> Result<(), StoreError> { + sqlx::query( + "INSERT INTO proof_checklist (submission_digest, topic_id, rules_version, green, failed_ids, document) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(&row.submission_digest) + .bind(&row.topic_id) + .bind(to_i32(row.rules_version)?) + .bind(row.green) + .bind(serde_json::to_value(&row.failed_ids).map_err(malformed)?) + .bind(serde_json::to_value(&row.document).map_err(malformed)?) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn checklist(&self, submission_digest: &str) -> Result, StoreError> { + let row: Option = sqlx::query_as( + "SELECT topic_id, submission_digest, rules_version, green, failed_ids, document \ + FROM proof_checklist WHERE submission_digest = $1", + ) + .bind(submission_digest) + .fetch_optional(&self.pool) + .await?; + row.map(|r| { + let document: Checklist = serde_json::from_value(r.document).map_err(malformed)?; + Ok(ChecklistRow { + topic_id: r.topic_id, + submission_digest: r.submission_digest, + rules_version: to_u32(r.rules_version)?, + green: r.green, + failed_ids: serde_json::from_value(r.failed_ids).map_err(malformed)?, + document, + }) + }) + .transpose() + } + + async fn record_transition(&self, row: &TransitionRow) -> Result<(), StoreError> { + sqlx::query( + "INSERT INTO proof_lifecycle_event (topic_id, from_state, event, to_state, note) \ + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&row.topic_id) + .bind(row.from.as_str()) + .bind(row.event.as_str()) + .bind(row.to.as_str()) + .bind(&row.note) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn lifecycle(&self, topic_id: &str) -> Result, StoreError> { + let rows: Vec = sqlx::query_as( + "SELECT topic_id, from_state, event, to_state, note FROM proof_lifecycle_event \ + WHERE topic_id = $1 ORDER BY id", + ) + .bind(topic_id) + .fetch_all(&self.pool) + .await?; + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + out.push(TransitionRow { + topic_id: r.topic_id, + from: parse_state(&r.from_state)?, + event: parse_event(&r.event)?, + to: parse_state(&r.to_state)?, + note: r.note, + }); + } + Ok(replay(topic_id, &out)) + } + + async fn put_baseline(&self, row: &BaselineRow) -> Result<(), StoreError> { + if !row.primary_value.is_finite() { + return Err(StoreError::Malformed("baseline primary".into())); + } + sqlx::query( + "INSERT INTO proof_baseline_measurement (topic_id, rules_version, primary_value, report) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(&row.topic_id) + .bind(to_i32(row.rules_version)?) + .bind(row.primary_value) + .bind(serde_json::to_value(&row.report).map_err(malformed)?) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn baseline(&self, topic_id: &str) -> Result, StoreError> { + let row: Option = sqlx::query_as( + "SELECT topic_id, rules_version, primary_value, report FROM proof_baseline_measurement \ + WHERE topic_id = $1 ORDER BY rules_version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + row.map(|r| { + let report: CustomRunReport = serde_json::from_value(r.report).map_err(malformed)?; + Ok(BaselineRow { + topic_id: r.topic_id, + rules_version: to_u32(r.rules_version)?, + primary_value: r.primary_value, + report, + }) + }) + .transpose() + } + + async fn put_artefact(&self, row: &ArtefactRow) -> Result<(), StoreError> { + check_artefact(row)?; + sqlx::query( + "INSERT INTO proof_artefact (topic_id, submission_id, submission_digest, path, sha256, bytes, \ + primary_value, checklist_green, promoted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind(&row.topic_id) + .bind(&row.submission_id) + .bind(&row.submission_digest) + .bind(&row.path) + .bind(&row.sha256) + .bind(i64::try_from(row.bytes).map_err(malformed)?) + .bind(row.primary_value) + .bind(row.checklist_green) + .bind(row.promoted) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn artefacts(&self, topic_id: &str) -> Result, StoreError> { + let rows: Vec = sqlx::query_as( + "SELECT topic_id, submission_id, submission_digest, path, sha256, bytes, primary_value, \ + checklist_green, promoted FROM proof_artefact WHERE topic_id = $1 ORDER BY created_at, submission_id", + ) + .bind(topic_id) + .fetch_all(&self.pool) + .await?; + rows.into_iter() + .map(|r| { + Ok(ArtefactRow { + topic_id: r.topic_id, + submission_id: r.submission_id, + submission_digest: r.submission_digest, + path: r.path, + sha256: r.sha256, + bytes: u64::try_from(r.bytes).map_err(malformed)?, + primary_value: r.primary_value, + checklist_green: r.checklist_green, + promoted: r.promoted, + }) + }) + .collect() + } + + async fn record_promotion(&self, row: &PromotionRow) -> Result<(), StoreError> { + check_promotion(row)?; + sqlx::query( + "INSERT INTO proof_promotion_event (topic_id, submission_id, submission_digest, primary_value, bar, previous_best) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(&row.topic_id) + .bind(&row.submission_id) + .bind(&row.submission_digest) + .bind(row.primary_value) + .bind(row.bar) + .bind(&row.previous_best) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn best(&self, topic_id: &str) -> Result, StoreError> { + let row: Option = sqlx::query_as( + "SELECT topic_id, submission_id, submission_digest, primary_value, bar, previous_best \ + FROM proof_promotion_event WHERE topic_id = $1 ORDER BY id DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(PromotionRow::from)) + } + + async fn promotions(&self, topic_id: &str) -> Result, StoreError> { + let rows: Vec = sqlx::query_as( + "SELECT topic_id, submission_id, submission_digest, primary_value, bar, previous_best \ + FROM proof_promotion_event WHERE topic_id = $1 ORDER BY id", + ) + .bind(topic_id) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(PromotionRow::from).collect()) + } +} diff --git a/crates/proof-rlm-store/tests/store_contract.rs b/crates/proof-rlm-store/tests/store_contract.rs new file mode 100644 index 000000000..9b3a86f08 --- /dev/null +++ b/crates/proof-rlm-store/tests/store_contract.rs @@ -0,0 +1,207 @@ +//! One contract, two stores. The memory store always runs; the Postgres store +//! runs against an isolated migrated schema when `DATABASE_URL` is set (same +//! gating as `crates/db/tests`) and is skipped otherwise. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::too_many_lines)] + +use proof_rlm::fixtures::{green, report_for, request, rules, topic}; +use proof_rlm::{RlmEvent, RlmState, RuleSource}; +use proof_rlm_store::{ + ArtefactRow, BaselineRow, ChecklistRow, MemoryRlmStore, PgRlmStore, PromotionRow, RlmStore, + StoreError, TransitionRow, +}; +use proof_task::ChecklistRule; + +async fn contract(store: &dyn RlmStore) { + let t = topic(); + + // Topic versions advance per persisted document. + assert!(store.latest_topic(&t.id).await.unwrap().is_none()); + assert_eq!(store.put_topic_version(&t).await.unwrap(), 1); + let mut resigned = t.clone(); + resigned.statement.push_str(" (v2)"); + assert_eq!(store.put_topic_version(&resigned).await.unwrap(), 2); + let (v, latest) = store.latest_topic(&t.id).await.unwrap().unwrap(); + assert_eq!(v, 2); + assert!(latest.statement.ends_with("(v2)")); + + // Rules: v1 from the document, v2 from the RLM, gaps refused. + let v1 = rules(); + assert!(store.current_rules(&t.id).await.unwrap().is_none()); + let v2 = v1 + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rlm_rule".into(), + text: "rewritten by the rlm".into(), + }], + ) + .unwrap(); + assert!(matches!( + store.put_rules(&v2).await, + Err(StoreError::VersionGap("rules")) + )); + store.put_rules(&v1).await.unwrap(); + assert!(matches!( + store.put_rules(&v1).await, + Err(StoreError::VersionGap("rules")) + )); + store.put_rules(&v2).await.unwrap(); + assert_eq!(store.current_rules(&t.id).await.unwrap().unwrap(), v2); + assert_eq!(store.rules_at(&t.id, 1).await.unwrap().unwrap(), v1); + assert!(store.rules_at(&t.id, 3).await.unwrap().is_none()); + + // Checklists are keyed by the frozen digest. + let digest = "ab".repeat(32); + let mut c = green(&v1, &digest); + c.items[0].pass = false; + let row = ChecklistRow::from_checklist(&c, &v1); + assert!(!row.green); + assert_eq!(row.failed_ids, vec![v1.rules[0].id.clone()]); + store.put_checklist(&row).await.unwrap(); + assert_eq!(store.checklist(&digest).await.unwrap().unwrap(), row); + assert!(store.checklist(&"cd".repeat(32)).await.unwrap().is_none()); + + // Lifecycle replays from appended rows. + assert!(store.lifecycle(&t.id).await.unwrap().is_none()); + for (from, event, to) in [ + ( + RlmState::Open, + RlmEvent::SubmissionReceived, + RlmState::Evaluating, + ), + ( + RlmState::Evaluating, + RlmEvent::PromotionCandidate, + RlmState::Promoting, + ), + (RlmState::Promoting, RlmEvent::Promoted, RlmState::Open), + ] { + store + .record_transition(&TransitionRow { + topic_id: t.id.clone(), + from, + event, + to, + note: format!("{event:?}"), + }) + .await + .unwrap(); + } + let lc = store.lifecycle(&t.id).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Open); + assert_eq!(lc.history.len(), 3); + assert_eq!(lc.history[1].event, RlmEvent::PromotionCandidate); + + // Baseline per rule version; newest wins. + let req = request(); + assert!(store.baseline(&t.id).await.unwrap().is_none()); + store + .put_baseline(&BaselineRow { + topic_id: t.id.clone(), + rules_version: 1, + primary_value: 0.4, + report: report_for(&req, 0.4), + }) + .await + .unwrap(); + store + .put_baseline(&BaselineRow { + topic_id: t.id.clone(), + rules_version: 2, + primary_value: 0.45, + report: report_for(&req, 0.45), + }) + .await + .unwrap(); + let b = store.baseline(&t.id).await.unwrap().unwrap(); + assert_eq!(b.rules_version, 2); + assert!((b.primary_value - 0.45).abs() < 1e-12); + assert!(matches!( + store + .put_baseline(&BaselineRow { + topic_id: t.id.clone(), + rules_version: 3, + primary_value: f64::NAN, + report: report_for(&req, 0.0), + }) + .await, + Err(StoreError::Malformed(_)) + )); + + // Artefact metadata and the promotion continuum. + let art = ArtefactRow { + topic_id: t.id.clone(), + submission_id: "pf_0000000000000001".into(), + submission_digest: digest.clone(), + path: "/artefacts/topic-a/pf_0000000000000001.zip".into(), + sha256: "ef".repeat(32), + bytes: 1_024, + primary_value: Some(0.7), + checklist_green: true, + promoted: true, + }; + store.put_artefact(&art).await.unwrap(); + assert!(matches!( + store + .put_artefact(&ArtefactRow { + submission_id: "nope".into(), + ..art.clone() + }) + .await, + Err(StoreError::Malformed(_)) + )); + assert_eq!(store.artefacts(&t.id).await.unwrap(), vec![art.clone()]); + + assert!(store.best(&t.id).await.unwrap().is_none()); + let first = PromotionRow { + topic_id: t.id.clone(), + submission_id: "pf_0000000000000001".into(), + submission_digest: digest.clone(), + primary_value: 0.7, + bar: Some(0.45), + previous_best: None, + }; + store.record_promotion(&first).await.unwrap(); + let second = PromotionRow { + submission_id: "pf_0000000000000002".into(), + primary_value: 0.8, + bar: Some(0.7), + previous_best: Some("pf_0000000000000001".into()), + ..first.clone() + }; + store.record_promotion(&second).await.unwrap(); + assert_eq!(store.best(&t.id).await.unwrap().unwrap(), second); + assert_eq!( + store.promotions(&t.id).await.unwrap(), + vec![first, second.clone()] + ); + assert!(matches!( + store + .record_promotion(&PromotionRow { + primary_value: f64::INFINITY, + ..second + }) + .await, + Err(StoreError::Malformed(_)) + )); + // Another topic sees none of it. + assert!(store.best("topic-b").await.unwrap().is_none()); + assert!(store.artefacts("topic-b").await.unwrap().is_empty()); +} + +#[tokio::test] +async fn memory_store_honours_the_contract() { + contract(&MemoryRlmStore::new()).await; +} + +#[tokio::test] +async fn postgres_store_honours_the_contract_when_a_database_is_present() { + if std::env::var_os("DATABASE_URL").is_none() { + eprintln!("DATABASE_URL unset; skipping the Postgres contract"); + return; + } + let pool = db::test_pool().await.expect("isolated migrated schema"); + contract(&PgRlmStore::new(pool.pool().clone())).await; + pool.drop_schema().await.expect("drop schema"); +} From 9d395de4051ed5ae74aebd49108031a974adb75d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:09:49 +0000 Subject: [PATCH 05/14] feat(proof-rlm-scorer): rlm scorer, artefact store, topic setup driver RlmScorer is the LiveScorer for the custom family: resolve custom_id in the registry (unknown or unwired -> RunnerUnwired, 503, no row), load the topic's current rule version from the store (seeding v1 from the signed document), inspect through the runner -> checklist persisted, red -> reject document with no paid inference, green -> spend token -> evaluate -> custom_value. Evaluations are serialised per topic and every lifecycle move is written to the store; a stale evaluating/promoting mirror is recovered rather than refused forever. ArtefactStore writes {root}/{topic_id}/{submission_id}.zip (manifest, artifact/, report, checklist, baseline_ref, logs) deterministically with path validation, best.json on promotion, and public events.jsonl; metadata and promotion rows mirror into the RLM store. TopicSetup drives draft -> owner_presend -> awaiting_owner_keys -> provisioning -> baselining over the VM boundary: owner hook, key probe, provision or attach, RLM ProposeRules -> rules vN in store, Baseline job -> measurement in store; mark_sealed closes baselining -> open. A re-run resumes from the persisted state. Router-level tests cover the unregistered 503, score / red-reject / below-best paths through the registry with a fake orchestrator, and the setup driver end to end. --- crates/proof-rlm-scorer/Cargo.toml | 40 ++ crates/proof-rlm-scorer/src/artefact.rs | 734 +++++++++++++++++++++++ crates/proof-rlm-scorer/src/lib.rs | 41 ++ crates/proof-rlm-scorer/src/scorer.rs | 609 +++++++++++++++++++ crates/proof-rlm-scorer/src/setup.rs | 320 ++++++++++ crates/proof-rlm-scorer/tests/rlm_e2e.rs | 511 ++++++++++++++++ 6 files changed, 2255 insertions(+) create mode 100644 crates/proof-rlm-scorer/Cargo.toml create mode 100644 crates/proof-rlm-scorer/src/artefact.rs create mode 100644 crates/proof-rlm-scorer/src/lib.rs create mode 100644 crates/proof-rlm-scorer/src/scorer.rs create mode 100644 crates/proof-rlm-scorer/src/setup.rs create mode 100644 crates/proof-rlm-scorer/tests/rlm_e2e.rs diff --git a/crates/proof-rlm-scorer/Cargo.toml b/crates/proof-rlm-scorer/Cargo.toml new file mode 100644 index 000000000..60c57165f --- /dev/null +++ b/crates/proof-rlm-scorer/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "proof-rlm-scorer" +description = "Proof RLM host side: RlmScorer (LiveScorer over the custom-runner registry, the RLM store, and the topic-VM boundary) and the topic-scoped artefact store ({root}/{topic_id}/{submission_id}.zip, best pointer, public events)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +async-trait = "0.1" +hex = "0.4" +proof-canon = { path = "../proof-canon" } +proof-eval = { path = "../proof-eval" } +proof-rlm = { path = "../proof-rlm" } +proof-rlm-store = { path = "../proof-rlm-store" } +proof-score = { path = "../proof-score" } +proof-task = { path = "../proof-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" +tokio = { version = "1", features = ["sync", "rt"] } +tracing = "0.1" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[dev-dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +hex = "0.4" +http-body-util = "0.1" +proof-http = { path = "../proof-http" } +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } +proof-store = { path = "../proof-store" } +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/proof-rlm-scorer/src/artefact.rs b/crates/proof-rlm-scorer/src/artefact.rs new file mode 100644 index 000000000..6cb4d7c10 --- /dev/null +++ b/crates/proof-rlm-scorer/src/artefact.rs @@ -0,0 +1,734 @@ +//! Topic-scoped artefact store: `{root}/{topic_id}/{submission_id}.zip`. +//! +//! ```text +//! manifest.json what this bundle is (ids, digests, primary, promoted) +//! checklist.json the rule items with evidence (rule version bound) +//! report.json runner-authored measurement (absent on a pre-spend reject) +//! baseline_ref.json the sealed baseline this run was compared against +//! artifact/… the miner's tree as inspected +//! logs/… runner logs (harness stdout, guest console, judge transcript) +//! ``` +//! +//! Beside the zips: `best.json` (current best pointer) and `events.jsonl` +//! (append-only public promotion events). Every path component is validated +//! before it touches the filesystem, the zip is written deterministically +//! (sorted entries, fixed timestamps), and nothing here knows a challenge. + +use std::io::{Cursor, Write}; +use std::path::{Path, PathBuf}; + +use proof_canon::is_slug; +use proof_rlm::{ArtifactFile, Checklist, CustomRunReport, LogFile}; +use proof_task::{ProofPin, TopicDocument}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, ZipWriter}; + +/// Env var naming the artefact root directory. +pub const ARTEFACT_ROOT_ENV: &str = "PROOF_ARTEFACT_ROOT"; + +/// Default artefact root. +pub const DEFAULT_ARTEFACT_ROOT: &str = "/artefacts"; + +/// Only accepted `manifest.json` schema. +pub const ARTEFACT_MANIFEST_SCHEMA: u32 = 1; + +/// Manifest entry. +pub const MANIFEST_FILE: &str = "manifest.json"; +/// Checklist entry. +pub const CHECKLIST_FILE: &str = "checklist.json"; +/// Report entry. +pub const REPORT_FILE: &str = "report.json"; +/// Baseline reference entry. +pub const BASELINE_REF_FILE: &str = "baseline_ref.json"; +/// Miner tree prefix. +pub const ARTIFACT_DIR: &str = "artifact"; +/// Logs prefix. +pub const LOGS_DIR: &str = "logs"; +/// Per-topic best pointer (next to the zips). +pub const BEST_FILE: &str = "best.json"; +/// Per-topic public event log (append-only JSON lines). +pub const EVENTS_FILE: &str = "events.jsonl"; + +/// Longest entry path inside the zip. +pub const MAX_ENTRY_PATH: usize = 256; + +/// The sealed baseline this run was compared against (public fields only). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BaselineRef { + /// Topic id. + pub topic_id: String, + /// SHA-256 of the sealed baseline script. + pub script_sha256: String, + /// Commitment over the sealed metric vector. + pub metrics_commitment: String, + /// Holdout commitment of the topic. + pub holdout_commitment: String, + /// Eval image digest the host pins. + pub eval_image_digest: String, +} + +impl BaselineRef { + /// Public seal references from the topic and pin. + #[must_use] + pub fn from_topic(topic: &TopicDocument, pin: &ProofPin) -> Self { + Self { + topic_id: topic.id.clone(), + script_sha256: topic.baseline.script_sha256.clone(), + metrics_commitment: topic.baseline.metrics_commitment.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + eval_image_digest: pin.eval_image_digest.clone(), + } + } +} + +/// `manifest.json`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ArtefactManifest { + /// Must equal [`ARTEFACT_MANIFEST_SCHEMA`]. + pub schema_version: u32, + /// Topic id. + pub topic_id: String, + /// Custom metric id. + pub custom_id: String, + /// Store row id (`pf_…`). + pub submission_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Miner artefact digest. + pub artifact_digest: String, + /// Rule version the checklist ticked. + pub rules_version: u32, + /// Primary value, when a report exists. + pub primary_value: Option, + /// Whether the checklist was green (spend happened only if true). + pub checklist_green: bool, + /// Whether this run was promoted. + pub promoted: bool, + /// Every entry in the zip, sorted. + pub entries: Vec, +} + +/// `best.json`: the topic's current best. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BestRef { + /// Topic id. + pub topic_id: String, + /// Winning row id. + pub submission_id: String, + /// Winning frozen digest. + pub submission_digest: String, + /// Winning primary. + pub primary_value: f64, + /// Bar it cleared, when known. + pub bar: Option, + /// Zip file name next to this pointer. + pub artefact: String, +} + +/// One line of `events.jsonl`. Public data only. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum PublicEvent { + /// A run was scored and its artefact written. + Scored { + /// Row id. + submission_id: String, + /// Primary value, when a report exists. + primary_value: Option, + /// Whether the checklist was green. + checklist_green: bool, + }, + /// A run became the topic's best. + Promoted { + /// Row id. + submission_id: String, + /// Primary value. + primary_value: f64, + /// Bar it cleared. + bar: Option, + /// Displaced best, if any. + previous_best: Option, + }, +} + +/// Everything that goes into one zip, before the row id is known. +#[derive(Debug, Clone, PartialEq)] +pub struct ArtefactBundle { + /// Topic id. + pub topic_id: String, + /// Custom metric id. + pub custom_id: String, + /// Frozen submission digest. + pub submission_digest: String, + /// Miner artefact digest. + pub artifact_digest: String, + /// Whether the checklist was green under its rule version. + pub checklist_green: bool, + /// The checklist (red on a pre-spend reject). + pub checklist: Checklist, + /// Runner report (absent when the checklist refused spend). + pub report: Option, + /// Sealed baseline reference. + pub baseline_ref: BaselineRef, + /// Miner tree. + pub artifact: Vec, + /// Runner logs. + pub logs: Vec, +} + +/// Why a bundle could not be written. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ArtefactError { + /// Topic id is not a slug. + #[error("artefact: topic id {0:?} is not a slug")] + BadTopicId(String), + /// Submission id is not `pf_` + 16 hex. + #[error("artefact: submission id {0:?} is not a store row id")] + BadSubmissionId(String), + /// A tree / log path would escape or collide. + #[error("artefact: entry path {0:?} is not a safe relative path")] + BadEntryPath(String), + /// Zip encoding failed. + #[error("artefact: zip: {0}")] + Zip(String), + /// Filesystem failure. + #[error("artefact: io: {0}")] + Io(String), +} + +fn is_row_id(id: &str) -> bool { + id.strip_prefix("pf_") + .is_some_and(|h| h.len() == 16 && h.bytes().all(|b| b.is_ascii_hexdigit())) +} + +/// Relative, no `.`/`..` segments, conservative charset, bounded length. +#[must_use] +pub fn is_safe_entry_path(path: &str) -> bool { + !path.is_empty() + && path.len() <= MAX_ENTRY_PATH + && !path.starts_with('/') + && !path.ends_with('/') + && path.split('/').all(|seg| { + !seg.is_empty() + && seg != "." + && seg != ".." + && seg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + }) +} + +/// `{root}/{topic_id}/{submission_id}.zip` after validating both ids. +/// +/// # Errors +/// +/// [`ArtefactError::BadTopicId`] / [`ArtefactError::BadSubmissionId`]. +pub fn artefact_path( + root: &Path, + topic_id: &str, + submission_id: &str, +) -> Result { + if !is_slug(topic_id) { + return Err(ArtefactError::BadTopicId(topic_id.to_owned())); + } + if !is_row_id(submission_id) { + return Err(ArtefactError::BadSubmissionId(submission_id.to_owned())); + } + Ok(root.join(topic_id).join(format!("{submission_id}.zip"))) +} + +fn options() -> SimpleFileOptions { + SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .last_modified_time(zip::DateTime::default()) + .unix_permissions(0o644) +} + +fn pretty(v: &T) -> Vec { + serde_json::to_string_pretty(v) + .unwrap_or_else(|_| "{}".into()) + .into_bytes() +} + +impl ArtefactBundle { + /// Sorted entry names this bundle will contain. + #[must_use] + pub fn entries(&self) -> Vec { + let mut out = vec![ + MANIFEST_FILE.to_owned(), + CHECKLIST_FILE.to_owned(), + BASELINE_REF_FILE.to_owned(), + ]; + if self.report.is_some() { + out.push(REPORT_FILE.to_owned()); + } + out.extend( + self.artifact + .iter() + .map(|f| format!("{ARTIFACT_DIR}/{}", f.path)), + ); + out.extend(self.logs.iter().map(|l| format!("{LOGS_DIR}/{}", l.name))); + out.sort(); + out + } + + /// `manifest.json` for `submission_id`. + #[must_use] + pub fn manifest(&self, submission_id: &str, promoted: bool) -> ArtefactManifest { + ArtefactManifest { + schema_version: ARTEFACT_MANIFEST_SCHEMA, + topic_id: self.topic_id.clone(), + custom_id: self.custom_id.clone(), + submission_id: submission_id.to_owned(), + submission_digest: self.submission_digest.clone(), + artifact_digest: self.artifact_digest.clone(), + rules_version: self.checklist.rules_version, + primary_value: self.report.as_ref().map(|r| r.primary_value), + checklist_green: self.checklist_green, + promoted, + entries: self.entries(), + } + } + + /// Deterministic zip bytes. + /// + /// # Errors + /// + /// [`ArtefactError::BadEntryPath`] for an unsafe tree / log name, + /// [`ArtefactError::Zip`] on encoder failure. + pub fn zip_bytes(&self, submission_id: &str, promoted: bool) -> Result, ArtefactError> { + let mut files: Vec<(String, Vec)> = vec![ + ( + MANIFEST_FILE.into(), + pretty(&self.manifest(submission_id, promoted)), + ), + (CHECKLIST_FILE.into(), self.checklist.to_json().into_bytes()), + (BASELINE_REF_FILE.into(), pretty(&self.baseline_ref)), + ]; + if let Some(r) = &self.report { + files.push((REPORT_FILE.into(), r.to_json().into_bytes())); + } + for f in &self.artifact { + if !is_safe_entry_path(&f.path) { + return Err(ArtefactError::BadEntryPath(f.path.clone())); + } + files.push((format!("{ARTIFACT_DIR}/{}", f.path), f.bytes.clone())); + } + for l in &self.logs { + if !is_safe_entry_path(&l.name) || l.name.contains('/') { + return Err(ArtefactError::BadEntryPath(l.name.clone())); + } + files.push((format!("{LOGS_DIR}/{}", l.name), l.bytes.clone())); + } + files.sort_by(|a, b| a.0.cmp(&b.0)); + for pair in files.windows(2) { + if pair[0].0 == pair[1].0 { + return Err(ArtefactError::BadEntryPath(pair[0].0.clone())); + } + } + let mut cursor = Cursor::new(Vec::new()); + { + let mut w = ZipWriter::new(&mut cursor); + for (name, bytes) in &files { + w.start_file(name.as_str(), options()) + .map_err(|e| ArtefactError::Zip(e.to_string()))?; + w.write_all(bytes) + .map_err(|e| ArtefactError::Zip(e.to_string()))?; + } + w.finish().map_err(|e| ArtefactError::Zip(e.to_string()))?; + } + Ok(cursor.into_inner()) + } +} + +/// A written zip: where it is and what it hashes to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WrittenArtefact { + /// Zip path. + pub path: PathBuf, + /// SHA-256 hex of the zip bytes. + pub sha256: String, + /// Zip size. + pub bytes: u64, +} + +/// Filesystem store rooted at [`ARTEFACT_ROOT_ENV`] / [`DEFAULT_ARTEFACT_ROOT`]. +#[derive(Debug, Clone)] +pub struct ArtefactStore { + root: PathBuf, +} + +impl ArtefactStore { + /// Store under `root`. + #[must_use] + pub fn new(root: &Path) -> Self { + Self { + root: root.to_path_buf(), + } + } + + /// Store under `PROOF_ARTEFACT_ROOT`, else `/artefacts`. + #[must_use] + pub fn from_env() -> Self { + let root = std::env::var(ARTEFACT_ROOT_ENV) + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| DEFAULT_ARTEFACT_ROOT.to_owned()); + Self::new(Path::new(&root)) + } + + /// Root directory. + #[must_use] + pub fn root(&self) -> &Path { + &self.root + } + + fn topic_dir(&self, topic_id: &str) -> Result { + if !is_slug(topic_id) { + return Err(ArtefactError::BadTopicId(topic_id.to_owned())); + } + let dir = self.root.join(topic_id); + std::fs::create_dir_all(&dir).map_err(|e| ArtefactError::Io(e.to_string()))?; + Ok(dir) + } + + /// Write `{root}/{topic_id}/{submission_id}.zip`. + /// + /// # Errors + /// + /// Id / path validation, zip, or io failures. Nothing is written on error. + pub fn write( + &self, + bundle: &ArtefactBundle, + submission_id: &str, + promoted: bool, + ) -> Result { + let path = artefact_path(&self.root, &bundle.topic_id, submission_id)?; + let bytes = bundle.zip_bytes(submission_id, promoted)?; + self.topic_dir(&bundle.topic_id)?; + std::fs::write(&path, &bytes).map_err(|e| ArtefactError::Io(e.to_string()))?; + Ok(WrittenArtefact { + path, + sha256: hex::encode(Sha256::digest(&bytes)), + bytes: bytes.len() as u64, + }) + } + + /// Write `{root}/{topic_id}/best.json`. + /// + /// # Errors + /// + /// Id validation or io failures. + pub fn mark_best(&self, best: &BestRef) -> Result { + artefact_path(&self.root, &best.topic_id, &best.submission_id)?; + let path = self.topic_dir(&best.topic_id)?.join(BEST_FILE); + let mut body = pretty(best); + body.push(b'\n'); + std::fs::write(&path, body).map_err(|e| ArtefactError::Io(e.to_string()))?; + Ok(path) + } + + /// Read the current best pointer, if any. + #[must_use] + pub fn best(&self, topic_id: &str) -> Option { + if !is_slug(topic_id) { + return None; + } + let body = std::fs::read_to_string(self.root.join(topic_id).join(BEST_FILE)).ok()?; + serde_json::from_str(&body).ok() + } + + /// Append one public event to `{root}/{topic_id}/events.jsonl`. + /// + /// # Errors + /// + /// Id validation or io failures. + pub fn append_event(&self, topic_id: &str, event: &PublicEvent) -> Result<(), ArtefactError> { + let path = self.topic_dir(topic_id)?.join(EVENTS_FILE); + let mut line = + serde_json::to_string(event).map_err(|e| ArtefactError::Io(e.to_string()))?; + line.push('\n'); + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|e| ArtefactError::Io(e.to_string()))?; + file.write_all(line.as_bytes()) + .map_err(|e| ArtefactError::Io(e.to_string())) + } + + /// Public events, oldest first (empty when none). + #[must_use] + pub fn events(&self, topic_id: &str) -> Vec { + if !is_slug(topic_id) { + return Vec::new(); + } + std::fs::read_to_string(self.root.join(topic_id).join(EVENTS_FILE)) + .map(|body| { + body.lines() + .filter_map(|l| serde_json::from_str(l).ok()) + .collect() + }) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use proof_rlm::fixtures::{green, report_for, request, rules}; + + use super::*; + + fn tmp(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "proof-rlm-artefacts-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("dir"); + dir + } + + fn bundle(with_report: bool) -> ArtefactBundle { + let req = request(); + let set = rules(); + let mut checklist = green(&set, &req.submission_digest); + if !with_report { + checklist.items[0].pass = false; + } + ArtefactBundle { + topic_id: req.topic_id.clone(), + custom_id: req.custom_id.clone(), + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.clone(), + checklist_green: with_report, + checklist, + report: with_report.then(|| report_for(&req, 0.6)), + baseline_ref: BaselineRef { + topic_id: req.topic_id.clone(), + script_sha256: "11".repeat(32), + metrics_commitment: "22".repeat(32), + holdout_commitment: "33".repeat(32), + eval_image_digest: format!("sha256:{}", "ab".repeat(32)), + }, + artifact: vec![ + ArtifactFile { + path: "src/main.rs".into(), + bytes: b"fn main() {}\n".to_vec(), + }, + ArtifactFile { + path: "Cargo.toml".into(), + bytes: b"[package]\n".to_vec(), + }, + ], + logs: vec![LogFile { + name: "run.log".into(), + bytes: b"ok\n".to_vec(), + }], + } + } + + fn entries_of(bytes: &[u8]) -> Vec<(String, Vec)> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes.to_vec())).expect("zip"); + let mut out = Vec::new(); + for i in 0..archive.len() { + let mut f = archive.by_index(i).expect("entry"); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).expect("read"); + out.push((f.name().to_owned(), buf)); + } + out + } + + #[test] + fn the_zip_has_the_documented_layout_and_is_deterministic() { + let b = bundle(true); + let a = b.zip_bytes("pf_0000000000000001", false).expect("zip"); + let again = b.zip_bytes("pf_0000000000000001", false).expect("zip"); + assert_eq!(a, again, "same bundle, same bytes"); + let entries = entries_of(&a); + let names: Vec<&str> = entries.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, + [ + "artifact/Cargo.toml", + "artifact/src/main.rs", + "baseline_ref.json", + "checklist.json", + "logs/run.log", + "manifest.json", + "report.json", + ] + ); + let manifest: ArtefactManifest = serde_json::from_slice(&entries[5].1).expect("manifest"); + assert_eq!(manifest.submission_id, "pf_0000000000000001"); + assert_eq!(manifest.rules_version, 1); + assert!((manifest.primary_value.expect("primary") - 0.6).abs() < 1e-12); + assert!(manifest.checklist_green); + assert!(!manifest.promoted); + assert_eq!(manifest.entries, names); + let report = CustomRunReport::from_json(std::str::from_utf8(&entries[6].1).expect("utf8")) + .expect("report"); + assert_eq!(report.topic_id, b.topic_id); + } + + /// A pre-spend reject still ships a bundle: the red checklist is the + /// evidence, and there is no report to ship. + #[test] + fn a_red_checklist_bundle_has_no_report() { + let b = bundle(false); + let bytes = b.zip_bytes("pf_0000000000000002", false).expect("zip"); + let names: Vec = entries_of(&bytes).into_iter().map(|(n, _)| n).collect(); + assert!(!names.iter().any(|n| n == REPORT_FILE), "{names:?}"); + let m = b.manifest("pf_0000000000000002", false); + assert!(!m.checklist_green); + assert_eq!(m.primary_value, None); + } + + #[test] + fn entry_paths_cannot_escape_or_collide() { + for bad in [ + "", + "/etc/passwd", + "../x", + "a/../b", + "a/./b", + "a//b", + "trailing/", + "sp ace", + "back\\slash", + &"x".repeat(MAX_ENTRY_PATH + 1), + ] { + assert!(!is_safe_entry_path(bad), "{bad:?}"); + } + for good in ["Cargo.toml", "src/main.rs", ".gitignore", "a-b_c.d/e"] { + assert!(is_safe_entry_path(good), "{good:?}"); + } + let mut b = bundle(true); + b.artifact.push(ArtifactFile { + path: "../escape".into(), + bytes: Vec::new(), + }); + assert_eq!( + b.zip_bytes("pf_0000000000000003", false), + Err(ArtefactError::BadEntryPath("../escape".into())) + ); + let mut nested_log = bundle(true); + nested_log.logs[0].name = "a/b.log".into(); + assert!(matches!( + nested_log.zip_bytes("pf_0000000000000003", false), + Err(ArtefactError::BadEntryPath(_)) + )); + let mut dup = bundle(true); + dup.artifact.push(ArtifactFile { + path: "Cargo.toml".into(), + bytes: b"again".to_vec(), + }); + assert!(matches!( + dup.zip_bytes("pf_0000000000000003", false), + Err(ArtefactError::BadEntryPath(_)) + )); + } + + #[test] + fn ids_are_validated_before_touching_the_filesystem() { + let root = Path::new("/artefacts"); + assert_eq!( + artefact_path(root, "topic-a", "pf_000000000000002a").expect("path"), + PathBuf::from("/artefacts/topic-a/pf_000000000000002a.zip") + ); + assert!(matches!( + artefact_path(root, "../etc", "pf_0000000000000001"), + Err(ArtefactError::BadTopicId(_)) + )); + assert!(matches!( + artefact_path(root, "topic-a", "../../x"), + Err(ArtefactError::BadSubmissionId(_)) + )); + assert!(matches!( + artefact_path(root, "topic-a", "pf_1"), + Err(ArtefactError::BadSubmissionId(_)) + )); + } + + #[test] + fn the_store_writes_zips_the_best_pointer_and_public_events() { + let root = tmp("store"); + let store = ArtefactStore::new(&root); + assert_eq!(store.root(), root.as_path()); + let b = bundle(true); + let written = store.write(&b, "pf_0000000000000007", true).expect("write"); + assert_eq!( + written.path, + root.join("topic-a").join("pf_0000000000000007.zip") + ); + let bytes = std::fs::read(&written.path).expect("read"); + assert_eq!( + bytes, + b.zip_bytes("pf_0000000000000007", true).expect("zip") + ); + assert_eq!(written.sha256, hex::encode(Sha256::digest(&bytes))); + assert_eq!(written.bytes, bytes.len() as u64); + assert!(store.best("topic-a").is_none()); + let pointer = store + .mark_best(&BestRef { + topic_id: "topic-a".into(), + submission_id: "pf_0000000000000007".into(), + submission_digest: b.submission_digest.clone(), + primary_value: 0.6, + bar: Some(0.5), + artefact: "pf_0000000000000007.zip".into(), + }) + .expect("pointer"); + assert_eq!(pointer, root.join("topic-a").join(BEST_FILE)); + assert_eq!( + store.best("topic-a").expect("best").submission_id, + "pf_0000000000000007" + ); + assert!(store.best("../x").is_none()); + store + .append_event( + "topic-a", + &PublicEvent::Scored { + submission_id: "pf_0000000000000007".into(), + primary_value: Some(0.6), + checklist_green: true, + }, + ) + .expect("event"); + store + .append_event( + "topic-a", + &PublicEvent::Promoted { + submission_id: "pf_0000000000000007".into(), + primary_value: 0.6, + bar: Some(0.5), + previous_best: None, + }, + ) + .expect("event"); + let events = store.events("topic-a"); + assert_eq!(events.len(), 2); + assert!(matches!(events[1], PublicEvent::Promoted { .. })); + assert!(store.events("topic-b").is_empty()); + assert!(store.append_event("Bad Topic", &events[0]).is_err()); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn the_root_comes_from_the_env_or_the_default() { + assert_eq!( + ArtefactStore::new(Path::new(DEFAULT_ARTEFACT_ROOT)).root(), + Path::new("/artefacts") + ); + assert_eq!(ARTEFACT_ROOT_ENV, "PROOF_ARTEFACT_ROOT"); + } +} diff --git a/crates/proof-rlm-scorer/src/lib.rs b/crates/proof-rlm-scorer/src/lib.rs new file mode 100644 index 000000000..76868b4eb --- /dev/null +++ b/crates/proof-rlm-scorer/src/lib.rs @@ -0,0 +1,41 @@ +//! Proof RLM engine — host side. No challenge content. +//! +//! - [`RlmScorer`] is the `LiveScorer` for the whole `custom` metric family: +//! resolve the topic's `custom_id` in the runner registry (unknown → +//! `RunnerUnwired`, 503, no row), load the topic's current rule version +//! from the store, inspect → checklist (persisted), red → reject without +//! paid inference, green → spend token → evaluate → `custom_value`. It is +//! wired through `proof_eval::FamilyMux` so no custom topic ever falls +//! back to the digest-pinned harvest. +//! - [`ArtefactStore`] writes `{root}/{topic_id}/{submission_id}.zip` +//! (`manifest.json`, `artifact/`, `report.json`, `checklist.json`, +//! `baseline_ref.json`, `logs/`) once the row is persisted, `best.json` +//! on promotion, and `events.jsonl` (public events) — with metadata and +//! the promotion continuum mirrored into the RLM store. +//! - [`TopicSetup`] drives `draft → … → baselining` over the topic-VM +//! boundary (owner hook, key probe, provision, RLM rule proposal → +//! store, baseline → store), and `mark_sealed` closes the loop to `open`. +//! +//! Core types live in `proof-rlm`; persistence in `proof-rlm-store`. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::too_many_arguments +)] + +mod artefact; +mod scorer; +mod setup; + +pub use artefact::{ + artefact_path, is_safe_entry_path, ArtefactBundle, ArtefactError, ArtefactManifest, + ArtefactStore, BaselineRef, BestRef, PublicEvent, WrittenArtefact, ARTEFACT_MANIFEST_SCHEMA, + ARTEFACT_ROOT_ENV, ARTIFACT_DIR, BASELINE_REF_FILE, BEST_FILE, CHECKLIST_FILE, + DEFAULT_ARTEFACT_ROOT, EVENTS_FILE, LOGS_DIR, MANIFEST_FILE, MAX_ENTRY_PATH, REPORT_FILE, +}; +pub use scorer::RlmScorer; +pub use setup::{SetupError, SetupOutcome, TopicSetup}; diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs new file mode 100644 index 000000000..93908e2af --- /dev/null +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -0,0 +1,609 @@ +//! [`RlmScorer`]: the `LiveScorer` for the whole `custom` metric family. +//! +//! Per submission, in this order and never reordered: +//! +//! 1. resolve the topic's `custom_id` in the [`RunnerRegistry`] — unknown or +//! unwired → `RunnerUnwired` (503, no row); +//! 2. load the topic's **current rule version** from the store (seeding +//! version 1 from the signed document the first time); +//! 3. **inspect** through the runner (a job in the topic VM) → checklist, +//! persisted red or green; +//! 4. red → **reject document, no paid inference**; +//! 5. green → [`SpendToken`] → **evaluate** (the only paid step) → report → +//! `custom_value = primary_value`; +//! 6. on persist: artefact zip + metadata row + public event; on promotion: +//! promotion row, `best.json`, lifecycle `promoting → open`. +//! +//! Evaluations are serialised per topic so the lifecycle mirror is exact +//! and every transition is written to the store. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use proof_eval::{EvalError, LiveScorer, ProofEvalDocument, PROOF_METRICS_SCHEMA}; +use proof_rlm::{ + authorize_spend, decide_promote, ArtifactFile, Checklist, CustomRunReport, CustomRunRequest, + Lifecycle, LogFile, PromoteDecision, RlmEvent, RlmState, RuleSet, RunnerError, RunnerRegistry, +}; +use proof_rlm_store::{ArtefactRow, ChecklistRow, PromotionRow, RlmStore, TransitionRow}; +use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind}; +use proof_task::{HoldoutRecord, InferenceOffer, MetricFamily, ProofPin, TopicDocument}; + +use crate::artefact::{ArtefactBundle, ArtefactStore, BaselineRef, BestRef, PublicEvent}; + +/// Scored-but-not-yet-persisted state for one submission. +struct Pending { + bundle: ArtefactBundle, + decision: Option, +} + +/// Family scorer over the runner registry, the RLM store, and the artefact store. +pub struct RlmScorer { + registry: Arc, + store: Arc, + artefacts: Option, + pending: Mutex>, + locks: Mutex>>>, +} + +fn unwired(custom_id: &str, detail: String) -> EvalError { + EvalError::RunnerUnwired { + custom_id: custom_id.to_owned(), + detail, + } +} + +fn map_runner(custom_id: &str, e: RunnerError) -> EvalError { + match e { + RunnerError::Unregistered(_) | RunnerError::NotWired(_) => { + unwired(custom_id, e.to_string()) + } + other => EvalError::Backend(other.to_string()), + } +} + +fn store_err(e: E) -> EvalError { + EvalError::Backend(format!("rlm store: {e}")) +} + +impl RlmScorer { + /// Scorer over `registry` and `store`, no artefact store. + #[must_use] + pub fn new(registry: Arc, store: Arc) -> Self { + Self { + registry, + store, + artefacts: None, + pending: Mutex::new(BTreeMap::new()), + locks: Mutex::new(BTreeMap::new()), + } + } + + /// Where artefact zips go. `None` keeps bundles in memory only. + #[must_use] + pub fn with_artefacts(mut self, store: Option) -> Self { + self.artefacts = store; + self + } + + /// Bundles scored but not yet persisted. + #[must_use] + pub fn pending_len(&self) -> usize { + self.pending.lock().map_or(0, |m| m.len()) + } + + fn topic_lock(&self, topic_id: &str) -> Arc> { + self.locks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entry(topic_id.to_owned()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + + /// Make sure the signed document this run is scored under is in the store + /// (a new signature is a new topic version). + async fn ensure_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + let latest = self + .store + .latest_topic(&topic.id) + .await + .map_err(store_err)?; + if latest.is_none_or(|(_, d)| d.signature != topic.signature) { + self.store + .put_topic_version(topic) + .await + .map_err(store_err)?; + } + Ok(()) + } + + /// Persisted lifecycle, or the position the signed document implies. + async fn lifecycle(&self, topic: &TopicDocument) -> Result { + Ok(self + .store + .lifecycle(&topic.id) + .await + .map_err(store_err)? + .unwrap_or_else(|| Lifecycle::from_topic(topic))) + } + + /// Apply `event` to the persisted lifecycle and record the move. + async fn apply( + &self, + topic: &TopicDocument, + event: RlmEvent, + note: &str, + ) -> Result { + let mut lc = self.lifecycle(topic).await?; + let to = lc + .apply(event, note) + .map_err(|e| EvalError::Backend(format!("topic lifecycle: {e}")))?; + let last = lc.history.last().cloned(); + if let Some(t) = last { + self.store + .record_transition(&TransitionRow { + topic_id: topic.id.clone(), + from: t.from, + event: t.event, + to: t.to, + note: t.note, + }) + .await + .map_err(store_err)?; + } + Ok(to) + } + + async fn apply_logged(&self, topic: &TopicDocument, event: RlmEvent, note: &str) { + if let Err(e) = self.apply(topic, event, note).await { + tracing::warn!(topic_id = %topic.id, %e, "rlm lifecycle event refused"); + } + } + + /// Called with the topic lock held: a persisted `evaluating` / + /// `promoting` means the previous run's row never landed. Close that + /// phase rather than refusing the topic forever. + async fn recover_stale(&self, topic: &TopicDocument) -> Result<(), EvalError> { + match self.lifecycle(topic).await?.state { + RlmState::Evaluating => { + self.apply( + topic, + RlmEvent::VerdictRecorded, + "previous run never persisted; recovered", + ) + .await?; + } + RlmState::Promoting => { + self.apply( + topic, + RlmEvent::PromotionRefused, + "previous promotion never persisted; recovered", + ) + .await?; + } + _ => {} + } + Ok(()) + } + + /// The topic's current rule version, seeding version 1 from the signed + /// document on first use so the gate a run was ticked against is in the + /// store, not only in the document. + async fn rules_for(&self, topic: &TopicDocument) -> Result { + if let Some(current) = self + .store + .current_rules(&topic.id) + .await + .map_err(store_err)? + { + return Ok(current); + } + let v1 = RuleSet::from_topic(topic).map_err(|e| { + EvalError::Backend(format!( + "topic carries no usable anti-cheat rules ({e}); refuse" + )) + })?; + self.store.put_rules(&v1).await.map_err(store_err)?; + Ok(v1) + } + + fn agent( + topic: &TopicDocument, + kind: ProofKind, + reproduced: bool, + claim_holds: bool, + rationale: String, + ) -> AgentVerdict { + AgentVerdict { + verdict: kind, + reproduced, + claim_holds_public: claim_holds, + contamination: false, + canary_hit: false, + flops_used: 0, + flops_budget: topic.flops_budget, + cheat_codes: if kind == ProofKind::Clean { + Vec::new() + } else { + vec![ProofCheatCode::Other] + }, + rationale, + topic_id: topic.id.clone(), + family: topic.metric.family, + } + } + + fn document( + pin: &ProofPin, + topic: &TopicDocument, + req: &CustomRunRequest, + agent: AgentVerdict, + custom_value: Option, + ) -> ProofEvalDocument { + ProofEvalDocument { + schema_version: PROOF_METRICS_SCHEMA, + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.clone(), + topic_id: topic.id.clone(), + eval_image_digest: pin.eval_image_digest.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + agent, + harness: HarnessMetrics { + custom_value, + ..HarnessMetrics::default() + }, + } + } + + fn stash( + &self, + pin: &ProofPin, + topic: &TopicDocument, + req: &CustomRunRequest, + checklist: Checklist, + checklist_green: bool, + artifact: Vec, + report: Option, + logs: Vec, + ) { + let bundle = ArtefactBundle { + topic_id: topic.id.clone(), + custom_id: req.custom_id.clone(), + submission_digest: req.submission_digest.clone(), + artifact_digest: req.artifact_digest.clone(), + checklist_green, + checklist, + report, + baseline_ref: BaselineRef::from_topic(topic, pin), + artifact, + logs, + }; + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + req.submission_digest.clone(), + Pending { + bundle, + decision: None, + }, + ); + } + + async fn evaluate( + &self, + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + frozen_digest: &str, + artifact_digest: &str, + claim: &str, + ) -> Result { + let custom_id = topic.metric.custom_id.trim().to_owned(); + let runner = self + .registry + .resolve(&custom_id) + .map_err(|e| map_runner(&custom_id, e))?; + let rules = self.rules_for(topic).await?; + let req = CustomRunRequest::from_topic( + topic, + pin, + offer, + &rules, + frozen_digest, + artifact_digest, + None, + claim, + ) + .map_err(|e| map_runner(&custom_id, e))?; + let inspected = runner + .inspect(&req, &rules) + .await + .map_err(|e| map_runner(&custom_id, e))?; + let checklist = inspected.checklist; + let row = ChecklistRow::from_checklist(&checklist, &rules); + self.store.put_checklist(&row).await.map_err(store_err)?; + if let Err(red) = checklist.verify(&rules) { + let rationale = format!( + "anti-cheat {red} (rules v{}); no paid inference", + rules.version + ); + let agent = Self::agent(topic, ProofKind::Reject, false, false, rationale); + self.stash( + pin, + topic, + &req, + checklist, + false, + inspected.artifact, + None, + Vec::new(), + ); + return Ok(Self::document(pin, topic, &req, agent, None)); + } + let token = authorize_spend(&checklist, &rules, &req.topic_id, &req.submission_digest) + .map_err(|e| EvalError::Backend(e.to_string()))?; + let run = runner + .evaluate(&req, &token) + .await + .map_err(|e| map_runner(&custom_id, e))?; + run.report + .verify(&req) + .map_err(|e| EvalError::NoVerdict(e.to_string()))?; + let rationale = format!( + "{}: {} = {:.6}; checklist green (rules v{}); sandboxed={}", + req.custom_id, + req.primary, + run.report.primary_value, + rules.version, + run.report.sandboxed + ); + let agent = Self::agent( + topic, + ProofKind::Clean, + true, + run.report.claim_holds, + rationale, + ); + let doc = Self::document(pin, topic, &req, agent, Some(run.report.primary_value)); + self.stash( + pin, + topic, + &req, + checklist, + true, + inspected.artifact, + Some(run.report), + run.logs, + ); + Ok(doc) + } + + async fn persist( + &self, + topic_id: &str, + submission_digest: &str, + submission_id: &str, + promoted: bool, + ) { + let Some(pending) = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(submission_digest) + else { + return; + }; + let bundle = pending.bundle; + let primary = bundle.report.as_ref().map(|r| r.primary_value); + let Some(store) = &self.artefacts else { + return; + }; + match store.write(&bundle, submission_id, promoted) { + Ok(written) => { + let row = ArtefactRow { + topic_id: topic_id.to_owned(), + submission_id: submission_id.to_owned(), + submission_digest: submission_digest.to_owned(), + path: written.path.display().to_string(), + sha256: written.sha256, + bytes: written.bytes, + primary_value: primary, + checklist_green: bundle.checklist_green, + promoted, + }; + if let Err(e) = self.store.put_artefact(&row).await { + tracing::error!(topic_id, submission_id, %e, "artefact metadata not persisted"); + } + let _ = store.append_event( + topic_id, + &PublicEvent::Scored { + submission_id: submission_id.to_owned(), + primary_value: primary, + checklist_green: bundle.checklist_green, + }, + ); + tracing::info!(topic_id, submission_id, path = %written.path.display(), "artefact written"); + } + Err(e) => tracing::error!(topic_id, submission_id, %e, "artefact not written"), + } + if !promoted { + return; + } + let (Some(primary_value), Some(PromoteDecision::Promote { bar, .. })) = + (primary, pending.decision) + else { + tracing::error!( + topic_id, + submission_id, + "promoted without a primary or a decision" + ); + return; + }; + let previous_best = self + .store + .best(topic_id) + .await + .ok() + .flatten() + .map(|b| b.submission_id); + let row = PromotionRow { + topic_id: topic_id.to_owned(), + submission_id: submission_id.to_owned(), + submission_digest: submission_digest.to_owned(), + primary_value, + bar: Some(bar), + previous_best: previous_best.clone(), + }; + if let Err(e) = self.store.record_promotion(&row).await { + tracing::error!(topic_id, submission_id, %e, "promotion not persisted"); + } + if let Err(e) = store.mark_best(&BestRef { + topic_id: topic_id.to_owned(), + submission_id: submission_id.to_owned(), + submission_digest: submission_digest.to_owned(), + primary_value, + bar: Some(bar), + artefact: format!("{submission_id}.zip"), + }) { + tracing::error!(topic_id, submission_id, %e, "best pointer not written"); + } + let _ = store.append_event( + topic_id, + &PublicEvent::Promoted { + submission_id: submission_id.to_owned(), + primary_value, + bar: Some(bar), + previous_best, + }, + ); + } +} + +#[async_trait] +impl LiveScorer for RlmScorer { + async fn score( + &self, + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + frozen_digest: &str, + artifact_digest: &str, + _holdout: &[HoldoutRecord], + claim: &str, + ) -> Result { + self.ready_for_topic(topic)?; + let lock = self.topic_lock(&topic.id); + let _serial = lock.lock().await; + self.ensure_topic(topic).await?; + self.recover_stale(topic).await?; + self.apply(topic, RlmEvent::SubmissionReceived, frozen_digest) + .await?; + let out = self + .evaluate(pin, topic, offer, frozen_digest, artifact_digest, claim) + .await; + if out.is_err() { + // No row will follow a refusal, so the verdict phase is over now. + self.apply_logged(topic, RlmEvent::VerdictRecorded, "refused; no row") + .await; + } + out + } + + fn ready(&self) -> Result<(), EvalError> { + Ok(()) + } + + fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + if topic.metric.family != MetricFamily::Custom { + return Err(EvalError::Backend(format!( + "rlm scorer asked to score non-custom topic {}", + topic.id + ))); + } + let custom_id = topic.metric.custom_id.trim(); + let runner = self + .registry + .resolve(custom_id) + .map_err(|e| map_runner(custom_id, e))?; + runner.ready().map_err(|e| map_runner(custom_id, e)) + } + + fn custom_ids(&self) -> Vec { + self.registry.ids() + } + + async fn auto_promote( + &self, + topic: &TopicDocument, + submission_digest: &str, + pass: bool, + primary: Option, + bar: Option, + ) -> bool { + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(p) = pending.get_mut(submission_digest) else { + return false; + }; + let reported = p.bundle.report.as_ref().map(|r| r.primary_value); + // The payout primary and the report must agree, or neither is evidence. + let agree = matches!((primary, reported), (Some(a), Some(b)) if (a - b).abs() < 1e-9); + let decision = decide_promote( + pass && agree, + p.bundle.checklist_green, + &p.bundle.checklist.failed_ids(), + reported, + bar, + topic.metric.direction, + topic.metric.epsilon_rel, + ); + let promote = decision.is_promote(); + p.decision = Some(decision); + promote + } + + async fn on_persisted( + &self, + topic_id: &str, + submission_digest: &str, + submission_id: &str, + promoted: bool, + ) { + let owned = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains_key(submission_digest); + if !owned { + return; + } + let topic = self + .store + .latest_topic(topic_id) + .await + .ok() + .flatten() + .map(|(_, d)| d); + if promoted { + if let Some(t) = &topic { + self.apply_logged(t, RlmEvent::PromotionCandidate, submission_id) + .await; + } + } + self.persist(topic_id, submission_digest, submission_id, promoted) + .await; + if let Some(t) = &topic { + let event = if promoted { + RlmEvent::Promoted + } else { + RlmEvent::VerdictRecorded + }; + self.apply_logged(t, event, submission_id).await; + } + } +} diff --git a/crates/proof-rlm-scorer/src/setup.rs b/crates/proof-rlm-scorer/src/setup.rs new file mode 100644 index 000000000..4e238c6bf --- /dev/null +++ b/crates/proof-rlm-scorer/src/setup.rs @@ -0,0 +1,320 @@ +//! Topic setup driver: the agentic lifecycle from `draft` to a sealed +//! baseline, with the RLM doing its work **inside the topic VM**. +//! +//! ```text +//! draft ──submit_for_review──▶ owner_presend ──(owner hook)──▶ awaiting_owner_keys +//! ──(key probe)──▶ provisioning ──(orchestrator.create)──▶ baselining +//! ──(ProposeRules job → rules vN in store; Baseline job → measurement in store)──▶ +//! returns SetupOutcome; the operator seals custom_value, re-signs `open`, +//! and calls `mark_sealed` (baselining → open). +//! ``` +//! +//! The control plane never runs RLM logic: it forwards jobs through +//! [`TopicVmOrchestrator`] and persists what comes back. Every transition +//! lands in the store. A missing orchestrator, a declined owner, or a +//! missing key file stops the driver where it is, with the reason, and a +//! re-run resumes from the persisted state. + +use std::sync::Arc; + +use proof_rlm::{ + await_owner_keys, owner_presend, CustomRunReport, CustomRunRequest, Lifecycle, OwnerHook, + OwnerKeysProbe, OwnerPrompt, RlmEvent, RlmState, RuleSet, RuleSource, SandboxPolicy, + StateError, TopicVmOrchestrator, TopicVmSpec, VmError, VmHandle, VmJob, VmJobOutput, + VmTemplate, +}; +use proof_rlm_store::{BaselineRow, RlmStore, StoreError, TransitionRow}; +use proof_task::{InferenceOffer, MetricFamily, ProofPin, TopicDocument}; + +/// Why setup stopped. +#[derive(Debug, thiserror::Error)] +pub enum SetupError { + /// Not a custom-family topic (nothing for an RLM to do). + #[error("topic {0:?} is not a custom-family topic")] + NotCustom(String), + /// Lifecycle refused (owner declined, keys missing, illegal move). + #[error("lifecycle: {0}")] + State(#[from] StateError), + /// The orchestrator refused or failed. + #[error("topic vm: {0}")] + Vm(#[from] VmError), + /// The store refused. + #[error("store: {0}")] + Store(#[from] StoreError), + /// The RLM proposed no usable rules. + #[error("rules: {0}")] + Rules(#[from] proof_rlm::ChecklistError), + /// The baseline report did not bind to the request. + #[error("baseline report: {0}")] + Report(#[from] proof_rlm::ReportError), + /// The owner declined at presend; the topic is back at draft. + #[error("owner declined; topic {0:?} returned to draft")] + Declined(String), +} + +/// What setup produced for the operator to seal. +#[derive(Debug, Clone, PartialEq)] +pub struct SetupOutcome { + /// Topic id. + pub topic_id: String, + /// The topic's VM. + pub vm: VmHandle, + /// Rule version in force after the RLM wrote its rules. + pub rules_version: u32, + /// Baseline primary the operator seals as `custom_value`. + pub baseline_primary: f64, +} + +/// Everything the driver needs; no secrets. +pub struct TopicSetup { + /// VM boundary. + pub orchestrator: Arc, + /// Persistence. + pub store: Arc, + /// RLM VM image + sizes. + pub template: VmTemplate, + /// `askUser`-style owner hook. + pub owner: Arc, + /// Owner key presence probe. + pub keys: Arc, + /// Spend cap shown to the owner, if any. + pub spend_cap_usd: Option, +} + +impl TopicSetup { + async fn lifecycle(&self, topic: &TopicDocument) -> Result { + Ok(self + .store + .lifecycle(&topic.id) + .await? + .unwrap_or_else(|| Lifecycle::from_topic(topic))) + } + + /// Persist every transition after `from`. + async fn record(&self, lc: &Lifecycle, from: usize) -> Result<(), SetupError> { + for t in lc.history.iter().skip(from) { + self.store + .record_transition(&TransitionRow { + topic_id: lc.topic_id.clone(), + from: t.from, + event: t.event, + to: t.to, + note: t.note.clone(), + }) + .await?; + } + Ok(()) + } + + /// Apply one event and persist it. + async fn step( + &self, + lc: &mut Lifecycle, + event: RlmEvent, + note: &str, + ) -> Result { + let mark = lc.history.len(); + let to = lc.apply(event, note)?; + self.record(lc, mark).await?; + Ok(to) + } + + /// `draft → owner_presend → awaiting_owner_keys → provisioning`. + async fn owner_phase( + &self, + topic: &TopicDocument, + lc: &mut Lifecycle, + ) -> Result<(), SetupError> { + let prompt = OwnerPrompt::from_topic(topic, self.spend_cap_usd)?; + if lc.state == RlmState::Draft { + self.step(lc, RlmEvent::SubmitForReview, "setup").await?; + } + if lc.state == RlmState::OwnerPresend { + let mark = lc.history.len(); + let to = owner_presend(lc, self.owner.as_ref(), &prompt)?; + self.record(lc, mark).await?; + if to == RlmState::Draft { + return Err(SetupError::Declined(topic.id.clone())); + } + } + if lc.state == RlmState::AwaitingOwnerKeys { + await_owner_keys(lc, self.keys.as_ref())?; + self.record(lc, lc.history.len().saturating_sub(1)).await?; + } + Ok(()) + } + + /// Provision (or attach to) the topic's own VM: `provisioning → baselining`. + async fn provision( + &self, + topic: &TopicDocument, + pin: &ProofPin, + lc: &mut Lifecycle, + ) -> Result { + self.orchestrator.ready()?; + if let Some(h) = self.orchestrator.attach(&topic.id).await? { + if lc.state == RlmState::Provisioning { + self.step(lc, RlmEvent::Provisioned, &h.vm_id).await?; + } + return Ok(h); + } + let sandbox = SandboxPolicy { + firecracker_required: topic.constraints.firecracker_required, + deadline_s: topic + .eval_executor + .max_proof_deadline_s + .unwrap_or(pin.max_proof_deadline_s), + }; + let spec = TopicVmSpec::for_topic(&topic.id, self.template.clone(), sandbox); + spec.validate()?; + match self.orchestrator.create(&spec).await { + Ok(h) => { + if lc.state == RlmState::Provisioning { + self.step(lc, RlmEvent::Provisioned, &h.vm_id).await?; + } + Ok(h) + } + Err(e) => { + if lc.state == RlmState::Provisioning { + self.step(lc, RlmEvent::ProvisionFailed, &e.to_string()) + .await?; + } + Err(e.into()) + } + } + } + + /// The RLM writes its rules inside the VM; the store versions them. + async fn propose_rules( + &self, + topic: &TopicDocument, + vm: &VmHandle, + ) -> Result { + let current = self.store.current_rules(&topic.id).await?; + let job = VmJob::ProposeRules { + topic: Box::new(topic.clone()), + current_version: current.as_ref().map(|r| r.version), + }; + let VmJobOutput::Rules(proposed) = self.orchestrator.run(vm, job).await? else { + return Err(VmError::WrongOutput("propose_rules").into()); + }; + let rules = if let Some(cur) = current { + cur.next(RuleSource::Rlm, proposed)? + } else { + let set = RuleSet { + topic_id: topic.id.clone(), + version: 1, + source: RuleSource::Rlm, + rules: proposed, + }; + set.validate()?; + set + }; + self.store.put_rules(&rules).await?; + Ok(rules) + } + + /// Baseline inside the VM, shaped exactly like a miner run, persisted. + async fn baseline( + &self, + topic: &TopicDocument, + pin: &ProofPin, + offer: &InferenceOffer, + rules: &RuleSet, + vm: &VmHandle, + lc: &mut Lifecycle, + ) -> Result { + let request = CustomRunRequest::from_topic( + topic, + pin, + offer, + rules, + &format!("baseline-{}", rules.digest()), + &topic.baseline.script_sha256, + None, + "operator baseline", + ) + .map_err(|e| SetupError::Vm(VmError::Backend(e.to_string())))?; + let job = VmJob::Baseline { + request: request.clone(), + }; + let report = match self.orchestrator.run(vm, job).await { + Ok(VmJobOutput::Baseline(r)) => r, + Ok(_) => return Err(VmError::WrongOutput("baseline").into()), + Err(e) => { + self.step(lc, RlmEvent::BaselineFailed, &e.to_string()) + .await?; + return Err(e.into()); + } + }; + report.verify(&request)?; + self.store + .put_baseline(&BaselineRow { + topic_id: topic.id.clone(), + rules_version: rules.version, + primary_value: report.primary_value, + report: report.clone(), + }) + .await?; + Ok(report) + } + + /// Drive `draft → … → baselining` and run the RLM's rule + baseline jobs. + /// + /// # Errors + /// + /// See [`SetupError`]. The lifecycle is left where the failure happened + /// (persisted), so a re-run resumes rather than restarts. + pub async fn run( + &self, + topic: &TopicDocument, + pin: &ProofPin, + offer: &InferenceOffer, + ) -> Result { + if topic.metric.family != MetricFamily::Custom { + return Err(SetupError::NotCustom(topic.id.clone())); + } + if self.store.latest_topic(&topic.id).await?.is_none() { + self.store.put_topic_version(topic).await?; + } + let mut lc = self.lifecycle(topic).await?; + self.owner_phase(topic, &mut lc).await?; + if lc.state != RlmState::Provisioning && lc.state != RlmState::Baselining { + return Err(StateError::Illegal { + from: lc.state, + event: RlmEvent::Provisioned, + } + .into()); + } + let vm = self.provision(topic, pin, &mut lc).await?; + let rules = self.propose_rules(topic, &vm).await?; + let report = self + .baseline(topic, pin, offer, &rules, &vm, &mut lc) + .await?; + Ok(SetupOutcome { + topic_id: topic.id.clone(), + vm, + rules_version: rules.version, + baseline_primary: report.primary_value, + }) + } + + /// The operator sealed the baseline and re-signed `status: open`: + /// `baselining → open`, recorded, and the new document version stored. + /// + /// # Errors + /// + /// [`SetupError::State`] when the topic is not at `baselining`. + pub async fn mark_sealed(&self, topic: &TopicDocument) -> Result { + let mut lc = self.lifecycle(topic).await?; + let to = self + .step( + &mut lc, + RlmEvent::BaselineSealed, + "operator sealed the baseline", + ) + .await?; + self.store.put_topic_version(topic).await?; + Ok(to) + } +} diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs new file mode 100644 index 000000000..4878cabc7 --- /dev/null +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -0,0 +1,511 @@ +//! Full control-plane path for the generic RLM engine, without any VM, +//! Lium, or paid inference: `POST /v1/submissions` on an open custom-family +//! topic through `FamilyMux` → `RlmScorer` → registry → generic +//! `VmBackedRunner` → fake orchestrator, with the memory RLM store and a +//! temp artefact root. +//! +//! Covers: an unregistered `custom_id` is a 503 with no row; a green +//! checklist scores and is crowned against the sealed value; a red checklist +//! is a persisted reject with **zero** paid runs; a later pass below the best +//! stays `awaiting_admin`; every scored row leaves its zip, the crown leaves +//! `best.json` + a promotion row, and the store holds rules v1, every +//! checklist, and the lifecycle. Then the setup driver walks +//! `draft → … → baselining` with RLM-written rules and a baseline in the +//! store, and `mark_sealed` opens the topic. Every id here is a placeholder. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::doc_markdown, + clippy::similar_names, + clippy::too_many_lines +)] + +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::Router; +use http_body_util::BodyExt; +use proof_eval::{EvalBackend, EvalError, FamilyMux, LiveScorer, ProofEvalDocument}; +use proof_http::{hash_admin_token, proof_router, AppState}; +use proof_rlm::fixtures::{offer, pin, pinned_template, topic, FakeOrchestrator}; +use proof_rlm::{ + FileKeysProbe, OwnerDecision, RlmEvent, RlmState, RunnerRegistry, StaticOwnerHook, + VmBackedRunner, VmJob, +}; +use proof_rlm_scorer::{ArtefactStore, RlmScorer, SetupError, TopicSetup}; +use proof_rlm_store::{MemoryRlmStore, RlmStore}; +use proof_score::SealedBaseline; +use proof_store::MemoryStore; +use proof_task::{ + holdout_commitment, synthetic_holdout, HoldoutRecord, InferenceOffer, ProofPin, TopicDocument, + TopicStatus, STRATUM_SIZE, +}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; + +/// Default route: a harvest that is ready and never asked to score here. +struct IdleHarvest; + +#[async_trait::async_trait] +impl LiveScorer for IdleHarvest { + async fn score( + &self, + _pin: &ProofPin, + _topic: &TopicDocument, + _offer: &InferenceOffer, + _frozen: &str, + _artifact: &str, + _holdout: &[HoldoutRecord], + _claim: &str, + ) -> Result { + Err(EvalError::Backend("idle harvest".into())) + } +} + +fn digest(label: &str) -> String { + hex::encode(Sha256::digest(label.as_bytes())) +} + +fn tmp_root(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "proof-rlm-e2e-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +struct Stack { + app: Router, + orchestrator: Arc, + rlm_store: Arc, + root: std::path::PathBuf, + topic: TopicDocument, +} + +fn stack(register: bool) -> Stack { + let pin = pin(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let mut t = topic(); + t.holdout_commitment = holdout_commitment(&recs); + let registered = if register { + vec![t.metric.custom_id.clone()] + } else { + Vec::new() + }; + let registered_ref: Vec<&str> = registered.iter().map(String::as_str).collect(); + if register { + t.validate(&pin, ®istered_ref) + .expect("open custom topic validates"); + } + + let store = MemoryStore::new(); + store.put_topic(t.clone()).unwrap(); + store.load_holdout(&t.id, recs).unwrap(); + store + .set_baseline( + &t.id, + SealedBaseline { + custom_value: Some(0.5), + ..SealedBaseline::default() + }, + ) + .unwrap(); + + let orchestrator = FakeOrchestrator::new(0.7); + let mut registry = RunnerRegistry::new(); + if register { + registry + .register( + &t.metric.custom_id, + Arc::new(VmBackedRunner::new(orchestrator.clone(), pinned_template())), + ) + .unwrap(); + } + let rlm_store = Arc::new(MemoryRlmStore::new()); + let root = tmp_root("stack"); + let scorer = RlmScorer::new(Arc::new(registry), rlm_store.clone()) + .with_artefacts(Some(ArtefactStore::new(&root))); + let mux = FamilyMux::new(Arc::new(IdleHarvest)).with_custom_family(Arc::new(scorer)); + let app = proof_router(AppState { + store, + pin, + backend: EvalBackend::Lium, + live_scorer: Some(Arc::new(mux)), + offer: Some(offer()), + judge_api_key: Some("test-judge-key".into()), + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }); + Stack { + app, + orchestrator, + rlm_store, + root, + topic: t, + } +} + +async fn json_req( + app: Router, + method: &str, + uri: &str, + body: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let req = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let v = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) +} + +fn submit_body(topic_id: &str, label: &str) -> serde_json::Value { + serde_json::json!({ + "miner_hotkey": digest("miner"), + "artifact_digest": digest(label), + "claim": "placeholder claim", + "declared_flops": 1u64, + "topic_id": topic_id, + "manifest": { "train_dataset_ids": ["placeholder-corpus"] }, + }) +} + +fn zip_names(path: &std::path::Path) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut names: Vec = (0..archive.len()) + .map(|i| archive.by_index(i).unwrap().name().to_owned()) + .collect(); + names.sort(); + names +} + +fn paid_runs(orchestrator: &FakeOrchestrator) -> usize { + orchestrator + .jobs() + .iter() + .filter(|j| matches!(j, VmJob::Evaluate { .. })) + .count() +} + +#[tokio::test] +async fn an_unregistered_custom_id_is_503_with_no_row() { + let Stack { app, topic, .. } = stack(false); + let (st, status) = json_req(app.clone(), "GET", "/v1/status", serde_json::json!({})).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["open_topics"][0], topic.id, "{status}"); + assert!( + status["scorable_topics"].as_array().unwrap().is_empty(), + "{status}" + ); + assert!( + status["registered_custom"].as_array().unwrap().is_empty(), + "{status}" + ); + assert_eq!(status["can_score"], false, "{status}"); + + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&topic.id, "artifact-a"), + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let msg = body["error"].as_str().unwrap_or_default(); + assert!(msg.contains(&topic.metric.custom_id), "{body}"); + assert!(msg.contains("no registered runner"), "{body}"); + let (_, list) = json_req(app, "GET", "/v1/submissions", serde_json::json!({})).await; + assert!(list["items"].as_array().unwrap().is_empty(), "{list}"); +} + +#[tokio::test] +async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { + let Stack { + app, + orchestrator, + rlm_store, + root, + topic, + } = stack(true); + let tid = topic.id.clone(); + + // 0. Open, registered, scorable. + let (st, status) = json_req(app.clone(), "GET", "/v1/status", serde_json::json!({})).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["can_score"], true, "{status}"); + assert_eq!(status["scorable_topics"][0], tid, "{status}"); + assert_eq!( + status["registered_custom"][0], topic.metric.custom_id, + "{status}" + ); + let (_, topics) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + ) + .await; + assert_eq!( + topics["items"][0]["metric"]["custom_id"], + topic.metric.custom_id + ); + assert_eq!( + topics["items"][0]["constraints"]["firecracker_required"], + true + ); + assert_eq!(topics["items"][0]["checklist"][0]["id"], "rule_a"); + assert!( + !topics.to_string().contains("content_sha256"), + "holdout leak" + ); + + // 1. Green checklist, primary 0.70 > 0.50 * 1.02: scored and crowned. + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-a"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + assert_eq!(created["state"], "champion", "{created}"); + let id_a = created["id"].as_str().unwrap().to_owned(); + assert_eq!(paid_runs(&orchestrator), 1); + assert_eq!(orchestrator.created(), 1, "one topic, one vm"); + + let (_, row) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id_a}"), + serde_json::json!({}), + ) + .await; + assert_eq!(row["verdict"]["pass"], true, "{row}"); + assert!((row["verdict"]["harness"]["custom_value"].as_f64().unwrap() - 0.7).abs() < 1e-12); + assert!(row["verdict"]["agent"]["rationale"] + .as_str() + .unwrap() + .contains("rules v1")); + let dump = row.to_string(); + assert!( + !dump.contains("api_key") && !dump.contains("127.0.0.1"), + "{dump}" + ); + + let zip_a = root.join(&tid).join(format!("{id_a}.zip")); + assert_eq!( + zip_names(&zip_a), + [ + "artifact/src/main.rs", + "baseline_ref.json", + "checklist.json", + "logs/run.log", + "manifest.json", + "report.json", + ] + ); + let best = ArtefactStore::new(&root).best(&tid).expect("best.json"); + assert_eq!(best.submission_id, id_a); + let promo = rlm_store.best(&tid).await.unwrap().expect("promotion row"); + assert_eq!(promo.submission_id, id_a); + assert!((promo.primary_value - 0.7).abs() < 1e-12); + assert_eq!(promo.previous_best, None); + let rules = rlm_store + .current_rules(&tid) + .await + .unwrap() + .expect("rules v1 in store"); + assert_eq!(rules.version, 1); + assert_eq!(rules.rules, topic.checklist); + let (_, doc) = rlm_store + .latest_topic(&tid) + .await + .unwrap() + .expect("topic in store"); + assert_eq!(doc.signature, topic.signature); + + // 2. Red checklist: persisted reject, no paid run, red checklist in store. + orchestrator.set_red(Some("rule_b")); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-b"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], false, "{created}"); + assert_eq!(created["state"], "rejected", "{created}"); + let id_b = created["id"].as_str().unwrap().to_owned(); + assert_eq!(paid_runs(&orchestrator), 1, "a red checklist must not pay"); + let (_, row_b) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id_b}"), + serde_json::json!({}), + ) + .await; + assert!(row_b["verdict"]["agent"]["rationale"] + .as_str() + .unwrap() + .contains("rule_b")); + assert!(row_b["verdict"]["harness"]["custom_value"].is_null()); + let names_b = zip_names(&root.join(&tid).join(format!("{id_b}.zip"))); + assert!(!names_b.iter().any(|n| n == "report.json"), "{names_b:?}"); + let digest_b = row_b["submission_digest"].as_str().unwrap(); + let cl = rlm_store + .checklist(digest_b) + .await + .unwrap() + .expect("checklist row"); + assert!(!cl.green); + assert_eq!(cl.failed_ids, vec!["rule_b".to_owned()]); + + // 3. Green again but 0.60: beats the seal (pass) yet not the best + // 0.70 * 1.02, so it stays awaiting_admin and the crown stays. + orchestrator.set_red(None); + orchestrator.set_primary(0.6); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-c"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + assert_eq!(created["state"], "awaiting_admin", "{created}"); + assert_eq!(paid_runs(&orchestrator), 2); + assert_eq!( + ArtefactStore::new(&root).best(&tid).unwrap().submission_id, + id_a + ); + assert_eq!(rlm_store.promotions(&tid).await.unwrap().len(), 1); + assert_eq!(rlm_store.artefacts(&tid).await.unwrap().len(), 3); + assert_eq!(ArtefactStore::new(&root).events(&tid).len(), 4); + + // 4. Lifecycle mirror is back at open with every move persisted. + let lc = rlm_store.lifecycle(&tid).await.unwrap().expect("lifecycle"); + assert_eq!(lc.state, RlmState::Open); + let events: Vec = lc.history.iter().map(|h| h.event).collect(); + assert_eq!( + events, + [ + RlmEvent::SubmissionReceived, + RlmEvent::PromotionCandidate, + RlmEvent::Promoted, + RlmEvent::SubmissionReceived, + RlmEvent::VerdictRecorded, + RlmEvent::SubmissionReceived, + RlmEvent::VerdictRecorded, + ] + ); + // Jobs never carried a host path or a secret. + for job in orchestrator.jobs() { + let dump = serde_json::to_string(&job).unwrap(); + for forbidden in ["/run/base", "/opt/base", "api_key", "127.0.0.1"] { + assert!(!dump.contains(forbidden), "job leaked {forbidden}"); + } + } + let _ = std::fs::remove_dir_all(&root); +} + +/// The agentic setup: owner hook, key probe, VM provision, RLM-written +/// rules and baseline persisted, then the operator seals and opens. +#[tokio::test] +async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { + let root = tmp_root("setup"); + let key = root.join("owner_key"); + let orchestrator = FakeOrchestrator::new(0.42); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let mut setup = TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Decline { + reason: "not yet".into(), + })), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: Some(10.0), + }; + + // A decline returns to draft; nothing is provisioned. + let err = setup + .run(&draft, &pin(), &offer()) + .await + .expect_err("declined"); + assert!(matches!(err, SetupError::Declined(_)), "{err}"); + assert_eq!( + rlm_store.lifecycle(&draft.id).await.unwrap().unwrap().state, + RlmState::Draft + ); + assert_eq!(orchestrator.created(), 0); + + // Approved but no key file: stops at awaiting_owner_keys, nothing provisioned. + setup.owner = Arc::new(StaticOwnerHook(OwnerDecision::Approve)); + let err = setup + .run(&draft, &pin(), &offer()) + .await + .expect_err("no key"); + assert!(err.to_string().contains("owner keys not present"), "{err}"); + assert_eq!( + rlm_store.lifecycle(&draft.id).await.unwrap().unwrap().state, + RlmState::AwaitingOwnerKeys + ); + assert_eq!(orchestrator.created(), 0); + + // Key present: provision, RLM rules v1 in store, baseline in store. + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let out = setup.run(&draft, &pin(), &offer()).await.expect("setup"); + assert_eq!(out.rules_version, 1); + assert!((out.baseline_primary - 0.42).abs() < 1e-12); + assert_eq!(orchestrator.created(), 1); + let rules = rlm_store.current_rules(&draft.id).await.unwrap().unwrap(); + assert_eq!(rules.source, proof_rlm::RuleSource::Rlm); + assert_eq!(rules.rules[0].id, "rlm_rule"); + let baseline = rlm_store.baseline(&draft.id).await.unwrap().unwrap(); + assert_eq!(baseline.rules_version, 1); + let lc = rlm_store.lifecycle(&draft.id).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Baselining); + assert!(lc.history.iter().any(|h| h.event == RlmEvent::Provisioned)); + let dump = serde_json::to_string(&lc).unwrap(); + assert!( + !dump.contains("not-a-real-secret"), + "key value must never be recorded" + ); + assert!(orchestrator.jobs().iter().any(|j| matches!( + j, + VmJob::ProposeRules { + current_version: None, + .. + } + ))); + + // The operator seals custom_value (0.42) and re-signs open. + let mut open = draft.clone(); + open.status = TopicStatus::Open; + open.baseline.metrics_commitment = "22".repeat(32); + assert_eq!(setup.mark_sealed(&open).await.unwrap(), RlmState::Open); + let (v, latest) = rlm_store.latest_topic(&draft.id).await.unwrap().unwrap(); + assert_eq!(v, 2); + assert_eq!(latest.status, TopicStatus::Open); + let _ = std::fs::remove_dir_all(&root); +} From c27ef83c991d983c8cc4148af645da8e13c9a1ed Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:10:09 +0000 Subject: [PATCH 06/14] feat(proof): route custom topics through the rlm engine proof-http: submit refuses a topic whose runner is not registered or wired (503, no row) right after host readiness; publish validates custom ids against the host's registered set (draft with any id, open only with a runner); /v1/status gains scorable_topics and registered_custom, and can_score means at least one open topic is scorable here; a pass the family scorer crowns persists as champion and the async on_persisted hook fires with the row id. bins/proof-challenge: BASE_DATABASE_URL(_FILE) selects the Postgres RLM store (migrated at boot; configured-but-unreachable is fatal, unset falls back to memory with a warning); the live harvest is wrapped in FamilyMux with an RlmScorer over an empty RunnerRegistry, so every custom topic answers 503 until an operator / RLM registers a runner. No runner, benchmark, model, or repository is compiled in. PROOF_ARTEFACT_ROOT (default /artefacts) names where zips land. --- Cargo.lock | 62 +++ bins/proof-challenge/Cargo.toml | 4 + bins/proof-challenge/src/main.rs | 181 ++++++++- bins/proof-challenge/tests/submit_e2e.rs | 1 + crates/proof-challenge/src/lib.rs | 2 +- crates/proof-http/src/lib.rs | 468 +++++++++++++++++++---- crates/proof-rlm-scorer/Cargo.toml | 1 + crates/proof-rlm-scorer/src/scorer.rs | 16 +- crates/proof-rlm-scorer/src/setup.rs | 2 +- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 23 +- crates/proof-rlm/src/runner.rs | 29 +- 11 files changed, 698 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54a66e386..f5254cb9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3450,12 +3450,16 @@ dependencies = [ "challenge-keys", "clap", "crypto", + "db", "harvest-pod", "hex", "prism-lium", "proof-challenge", "proof-eval", "proof-harvest", + "proof-rlm", + "proof-rlm-scorer", + "proof-rlm-store", "proof-task", "reqwest 0.12.28", "serde_json", @@ -3545,6 +3549,64 @@ dependencies = [ "tower", ] +[[package]] +name = "proof-rlm" +version = "0.1.0" +dependencies = [ + "async-trait", + "hex", + "proof-canon", + "proof-score", + "proof-task", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "proof-rlm-scorer" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "hex", + "http-body-util", + "proof-canon", + "proof-eval", + "proof-executor", + "proof-http", + "proof-rlm", + "proof-rlm-store", + "proof-score", + "proof-store", + "proof-task", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", + "tower", + "tracing", + "zip", +] + +[[package]] +name = "proof-rlm-store" +version = "0.1.0" +dependencies = [ + "async-trait", + "db", + "proof-rlm", + "proof-task", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "proof-score" version = "0.1.0" diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index 6a2b53f33..ff00fbb71 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -16,11 +16,15 @@ path = "src/main.rs" axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } challenge-keys = { path = "../../crates/challenge-keys" } clap = { version = "4", features = ["derive", "env"] } +db = { path = "../../crates/db" } harvest-pod = { path = "../../crates/harvest-pod" } prism-lium = { path = "../../crates/prism-lium" } proof-challenge = { path = "../../crates/proof-challenge" } proof-eval = { path = "../../crates/proof-eval" } proof-harvest = { path = "../../crates/proof-harvest" } +proof-rlm = { path = "../../crates/proof-rlm" } +proof-rlm-scorer = { path = "../../crates/proof-rlm-scorer" } +proof-rlm-store = { path = "../../crates/proof-rlm-store" } proof-task = { path = "../../crates/proof-task" } serde_json = "1" telemetry = { path = "../../crates/telemetry" } diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index a55122b0b..7e18d296e 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -23,8 +23,11 @@ use proof_challenge::{ BaselineMeasurement, EvalBackend, EvalExecutorOffer, HarvestOverrides, InferenceOffer, LiveScorer, MemoryStore, ProofPin, TopicDocument, CHALLENGE_ID, SCORING_VERSION, }; -use proof_eval::supported_custom; +use proof_eval::{custom_ids_ref, registered_custom, FamilyMux}; use proof_harvest::{HarvestLimits, LiumProofHarvest}; +use proof_rlm::RunnerRegistry; +use proof_rlm_scorer::{ArtefactStore, RlmScorer}; +use proof_rlm_store::{MemoryRlmStore, PgRlmStore, RlmStore}; use tokio::net::TcpListener; /// Operator Proof challenge service CLI. @@ -88,6 +91,16 @@ struct Cli { /// Holdout shard bytes (`` files). Not the record catalog. #[arg(long, env = "PROOF_HOLDOUT_STORE")] holdout_store: Option, + /// Root for per-submission artefact zips (`{root}/{topic_id}/{submission_id}.zip`). + #[arg(long, env = "PROOF_ARTEFACT_ROOT", default_value = "/artefacts")] + artefact_root: PathBuf, + /// Postgres URL for the RLM store (topic versions, rule versions, + /// checklists, lifecycle, artefact metadata, promotions). Unset → in-memory. + #[arg(long, env = "BASE_DATABASE_URL")] + database_url: Option, + /// File holding the Postgres URL (preferred on a droplet). + #[arg(long, env = "BASE_DATABASE_URL_FILE")] + database_url_file: Option, } fn main() -> ExitCode { @@ -129,19 +142,6 @@ fn run(cli: &Cli) -> Result<(), String> { ); } - let store = MemoryStore::new(); - match load_topics(&store, &pin, cli.topics_file.as_deref()) { - Ok(n) => tracing::info!(topics = n, "signed topics loaded"), - Err(e) => tracing::warn!("topics unavailable ({e}); submissions will 400/503 until fixed"), - } - match load_holdouts(&store, cli.holdout_file.as_deref()) { - Ok(n) => tracing::info!(topics = n, "holdouts verified against topic commitments"), - Err(e) => tracing::warn!("holdouts unavailable ({e}); submissions will 503 until fixed"), - } - match load_baselines(&store, &pin, cli.baseline_file.as_deref()) { - Ok(n) => tracing::info!(topics = n, "sealed baselines recorded"), - Err(e) => tracing::warn!("baselines unavailable ({e}); submissions will 503 until fixed"), - } let offer = match load_offer(&pin, cli.inference_offer_file.as_deref()) { Ok(o) => { tracing::info!(offer_id = %o.offer_id, status = ?o.status, "inference offer loaded"); @@ -169,16 +169,24 @@ fn run(cli: &Cli) -> Result<(), String> { .build() .map_err(|e| e.to_string())?; + let rlm_store = rt.block_on(resolve_rlm_store(cli))?; let live_scorer = build_live_scorer( backend, cli.eval_timeout_secs, judge_api_key.clone(), cli.proxy_model_dir.clone(), cli.holdout_store.clone(), - ); + ) + .map(|harvest| with_custom_family(harvest, rlm_store, &cli.artefact_root)); match backend { EvalBackend::Lium if live_scorer.is_some() => { tracing::info!("live harvest wired: digest-pinned proof-eval image on Lium"); + tracing::info!( + registered_custom = ?registered_custom(live_scorer.as_deref()), + artefact_root = %cli.artefact_root.display(), + "custom-family topics route to the rlm scorer; an id with no registered runner \ + answers 503 (no runner is compiled in)" + ); } EvalBackend::Lium => tracing::warn!( "live harvest not wired; every submission will 503. Set the Lium credentials \ @@ -187,6 +195,21 @@ fn run(cli: &Cli) -> Result<(), String> { EvalBackend::Sim => {} } + let store = MemoryStore::new(); + let registered = registered_custom(live_scorer.as_deref()); + match load_topics(&store, &pin, cli.topics_file.as_deref(), ®istered) { + Ok(n) => tracing::info!(topics = n, "signed topics loaded"), + Err(e) => tracing::warn!("topics unavailable ({e}); submissions will 400/503 until fixed"), + } + match load_holdouts(&store, cli.holdout_file.as_deref()) { + Ok(n) => tracing::info!(topics = n, "holdouts verified against topic commitments"), + Err(e) => tracing::warn!("holdouts unavailable ({e}); submissions will 503 until fixed"), + } + match load_baselines(&store, &pin, cli.baseline_file.as_deref()) { + Ok(n) => tracing::info!(topics = n, "sealed baselines recorded"), + Err(e) => tracing::warn!("baselines unavailable ({e}); submissions will 503 until fixed"), + } + let state = AppState { store, pin, @@ -249,6 +272,62 @@ fn build_live_scorer( )) } +/// Route the `custom` metric family to the RLM scorer over the default harvest. +/// +/// The runner registry starts **empty**: no benchmark, model, or repository is +/// compiled in, so every custom topic answers 503 (`RunnerUnwired`) until an +/// operator or the topic's RLM registers a runner under its `custom_id`. It +/// never falls back to the digest-pinned harvest and never spends. +fn with_custom_family( + harvest: Arc, + rlm_store: Arc, + artefact_root: &Path, +) -> Arc { + let scorer = RlmScorer::new(Arc::new(RunnerRegistry::new()), rlm_store) + .with_artefacts(Some(ArtefactStore::new(artefact_root))); + Arc::new(FamilyMux::new(harvest).with_custom_family(Arc::new(scorer))) +} + +fn database_url(cli: &Cli) -> Result, String> { + if let Some(url) = cli.database_url.as_deref().map(str::trim) { + if !url.is_empty() { + return Ok(Some(url.to_owned())); + } + } + let Some(path) = cli.database_url_file.as_deref() else { + return Ok(None); + }; + let raw = std::fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err("BASE_DATABASE_URL_FILE is empty".into()); + } + Ok(Some(trimmed.to_owned())) +} + +/// Postgres RLM store when a database is configured, in-memory otherwise. +/// +/// A configured but unreachable database is fatal: falling back to memory +/// would silently drop every rule version, checklist, and promotion on +/// restart. +async fn resolve_rlm_store(cli: &Cli) -> Result, String> { + let Some(url) = database_url(cli)? else { + tracing::warn!( + "no database configured; rlm rules, checklists, lifecycle, and promotions are not \ + persisted across restarts" + ); + return Ok(Arc::new(MemoryRlmStore::new())); + }; + let pool = db::connect(&url) + .await + .map_err(|e| format!("database connect failed: {e}"))?; + db::migrate(&pool) + .await + .map_err(|e| format!("database migrate failed: {e}"))?; + tracing::info!("rlm store persists to postgres"); + Ok(Arc::new(PgRlmStore::new(pool))) +} + fn load_inference_api_key(path: Option<&Path>) -> Option { let p = path?; std::fs::read_to_string(p) @@ -278,13 +357,18 @@ fn load_pin(path: Option<&Path>) -> Result { Ok(pin) } -fn load_topics(store: &MemoryStore, pin: &ProofPin, path: Option<&Path>) -> Result { +fn load_topics( + store: &MemoryStore, + pin: &ProofPin, + path: Option<&Path>, + registered_custom: &[String], +) -> Result { let p = path.ok_or("PROOF_TOPICS_FILE not set")?; let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; let docs = TopicDocument::many_from_json(&body).map_err(|e| e.to_string())?; let n = docs.len(); for doc in docs { - doc.validate(pin, &supported_custom()) + doc.validate(pin, &custom_ids_ref(registered_custom)) .map_err(|e| format!("topic {}: {e}", doc.id))?; doc.verify_signature(pin) .map_err(|e| format!("topic {}: {e}", doc.id))?; @@ -502,6 +586,69 @@ mod tests { std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); } + /// No runner is compiled in: every custom id refuses through the mux + /// (`RunnerUnwired`, the 503 root cause), the harvest still owns the + /// nll / throughput route, and nothing is registered. + #[test] + fn live_scorer_registers_no_custom_runner_and_refuses_every_custom_id() { + let _guard = LIUM_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let pubkey = stub_ssh_pubkey("proof-families"); + std::env::set_var("LIUM_API_KEY", "test-key-not-a-real-secret"); + std::env::set_var("LIUM_SSH_PUBLIC_KEY_FILE", &pubkey); + let harvest = + build_live_scorer(EvalBackend::Lium, 900, None, None, None).expect("harvest wired"); + std::env::remove_var("LIUM_API_KEY"); + std::env::remove_var("LIUM_SSH_PUBLIC_KEY_FILE"); + + let root = std::env::temp_dir().join("proof-families-artefacts"); + let mux = with_custom_family(harvest, Arc::new(MemoryRlmStore::new()), &root); + assert!(registered_custom(Some(mux.as_ref())).is_empty()); + for id in ["any_metric", "another_metric"] { + let mut custom = TopicDocument::default(); + custom.metric.family = proof_task::MetricFamily::Custom; + custom.metric.custom_id = id.into(); + let err = mux.ready_for_topic(&custom).expect_err("unregistered"); + assert!( + matches!(err, proof_eval::EvalError::RunnerUnwired { .. }), + "{err}" + ); + assert!(err.to_string().contains("no registered runner"), "{err}"); + } + // The default route is the harvest itself, whose readiness is about + // proxy weights and holdout shards, not the runner registry. + let nll = TopicDocument::default(); + let err = mux.ready_for_topic(&nll).expect_err("no proxy dir staged"); + assert!( + matches!(err, proof_eval::EvalError::ProxyModelMissing), + "{err}" + ); + } + + #[test] + fn database_url_comes_from_the_value_or_the_file_or_nowhere() { + let mut cli = Cli::try_parse_from(["proof-challenge"]).expect("cli"); + assert_eq!(database_url(&cli).expect("none"), None); + cli.database_url = Some(" ".into()); + assert_eq!(database_url(&cli).expect("blank is none"), None); + let dir = std::env::temp_dir().join(format!("proof-db-url-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let file = dir.join("url"); + std::fs::write(&file, "postgres://placeholder/db\n").expect("write"); + cli.database_url_file = Some(file.clone()); + assert_eq!( + database_url(&cli).expect("file"), + Some("postgres://placeholder/db".into()) + ); + std::fs::write(&file, "\n").expect("write"); + assert!( + database_url(&cli).is_err(), + "an empty file is a config error" + ); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn inference_api_key_file_is_read_not_existence_only() { let dir = std::env::temp_dir().join(format!( diff --git a/bins/proof-challenge/tests/submit_e2e.rs b/bins/proof-challenge/tests/submit_e2e.rs index 11760cb1b..c0f94455e 100644 --- a/bins/proof-challenge/tests/submit_e2e.rs +++ b/bins/proof-challenge/tests/submit_e2e.rs @@ -88,6 +88,7 @@ fn dt_topic() -> TopicDocument { no_nvlink: true, no_nccl_fast_fabric: true, max_inter_node_gbps: Some(12.5), + ..Constraints::default() }, metric: MetricSpec { family: MetricFamily::Throughput, diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index df2ff8516..82a722dba 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -16,7 +16,7 @@ use proof_score::{payout_lattices, MinerTopicRun, SealedBaseline}; use proof_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; pub use proof_eval::{ - force_sim, resolve_eval_backend, scoring_readiness, sim_stub_win, supported_custom, + force_sim, registered_custom, resolve_eval_backend, scoring_readiness, sim_stub_win, BaselineMeasurement, EvalBackend, LiveScorer, }; pub use proof_executor::{ diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 51297c06f..70db29e8a 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -31,13 +31,13 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use proof_eval::{ - contamination_evidence, eval_after_freeze, force_sim, scoring_readiness, - secret_backed_base_url, supported_custom, EvalBackend, EvalError, LiveScorer, + contamination_evidence, custom_ids_ref, eval_after_freeze, force_sim, registered_custom, + scoring_readiness, secret_backed_base_url, EvalBackend, EvalError, LiveScorer, }; use proof_executor::{require_open_executor, EvalExecutorOffer, ExecutorPlan}; use proof_score::{ - judge_topic, primary_from_harness, AgentVerdict, GateFail, HarnessMetrics, MinerTopicRun, - ProofKind, ProofVerdict, + judge_topic, novelty_bar, primary_from_harness, AgentVerdict, GateFail, HarnessMetrics, + MinerTopicRun, ProofKind, ProofVerdict, }; use proof_store::{ freeze_submission_digest, ArtifactManifest, MemoryStore, Submission, SubmissionState, @@ -114,9 +114,18 @@ impl AppState { }) } - fn can_score(&self) -> bool { + /// Custom metric ids with a registered runner on this host. There is no + /// compiled-in list: a topic mints its id, a runner registered under it + /// makes the topic scorable. + fn registered_custom(&self) -> Vec { + registered_custom(self.live()) + } + + /// Whether the host-wide gates (digest, harvest, judge offer, executor, + /// key, any open sealed topic) pass. + fn host_ready(&self) -> bool { let open = self.store.any_open_scorable(self.epoch).unwrap_or(false); - if scoring_readiness( + scoring_readiness( &self.pin, self.backend, self.live(), @@ -125,21 +134,38 @@ impl AppState { self.executor_offer().as_ref(), self.judge_api_key.as_deref(), ) - .is_err() - { - return false; + .is_ok() + } + + /// Open topics this host can score right now: judge config resolves and, + /// on a live host, the family's scorer is wired (a custom topic whose + /// runner is not registered is open but not scorable). + fn scorable_topics(&self) -> Vec { + if !self.host_ready() { + return Vec::new(); } let secret = secret_backed_base_url(); - self.store.topics().unwrap_or_default().iter().any(|t| { - t.is_open_at(self.epoch) - && resolve_inference( - &self.pin, - Some(&t.inference), - secret.as_deref(), - self.offer.as_ref(), - ) - .ready_to_score() - }) + self.store + .topics() + .unwrap_or_default() + .iter() + .filter(|t| { + t.is_open_at(self.epoch) + && resolve_inference( + &self.pin, + Some(&t.inference), + secret.as_deref(), + self.offer.as_ref(), + ) + .ready_to_score() + && self.live().is_none_or(|s| s.ready_for_topic(t).is_ok()) + }) + .map(|t| t.id.clone()) + .collect() + } + + fn can_score(&self) -> bool { + !self.scorable_topics().is_empty() } } @@ -192,6 +218,8 @@ async fn status(State(st): State) -> impl IntoResponse { "live_harvest_wired": st.live_scorer.is_some(), "baseline_sealed": baseline_sealed, "open_topics": open, + "scorable_topics": st.scorable_topics(), + "registered_custom": st.registered_custom(), "epoch": st.epoch, })) } @@ -316,6 +344,11 @@ async fn submit( st.judge_api_key.as_deref(), ) .map_err(|e| eval_err(&e))?; + // A custom topic whose runner is not registered on this host is a 503 + // here, before any row or rent, not a rejected row downstream. + if let Some(live) = st.live() { + live.ready_for_topic(&topic).map_err(|e| eval_err(&e))?; + } let Some(offer) = st.offer.as_ref() else { return Err(eval_err(&EvalError::InferenceOfferMissing)); }; @@ -394,13 +427,14 @@ async fn submit( .await .map_err(|e| eval_err(&e))?; + let registered = st.registered_custom(); let verdict = judge_topic( &topic, &eval.agent, &eval.harness, &sealed, &hits, - &supported_custom(), + &custom_ids_ref(®istered), ); let receipt_json = serde_json::to_string(&eval.receipt).unwrap_or_default(); persist_scored( @@ -408,6 +442,7 @@ async fn submit( executor.as_ref(), eval.executor.as_ref(), body, + &topic, hotkey, artifact, nonce, @@ -416,6 +451,7 @@ async fn submit( receipt_json, eval.backend, ) + .await } #[allow(clippy::too_many_arguments)] @@ -507,11 +543,12 @@ fn persist_pre_eval_reject( } #[allow(clippy::too_many_arguments)] -fn persist_scored( +async fn persist_scored( st: &AppState, executor: Option<&EvalExecutorOffer>, plan: Option<&ExecutorPlan>, body: SubmitBody, + topic: &TopicDocument, hotkey: String, artifact: String, nonce: String, @@ -520,13 +557,24 @@ fn persist_scored( receipt_json: String, backend: EvalBackend, ) -> Result<(StatusCode, Json), (StatusCode, Json)> { - let topic_id = body.topic_id.trim().to_owned(); + let topic_id = topic.id.clone(); let pass = verdict.pass; - let primary = st - .store - .topic(&topic_id) - .ok() - .and_then(|t| primary_from_harness(&t, &verdict.harness)); + let primary = primary_from_harness(topic, &verdict.harness); + // Automatic promotion is a family decision (custom: pass + green checklist + // + relative win over the bar). Default scorers never crown. + let mut promoted = false; + if pass { + if let Some(live) = st.live() { + let bar = novelty_bar( + topic, + st.store.baseline(&topic_id).ok().flatten().as_ref(), + st.store.champion_primary(topic).ok().flatten(), + ); + promoted = live + .auto_promote(topic, &submission_digest, pass, primary, bar) + .await; + } + } let artifact_digest = artifact.clone(); let detail = if pass { None @@ -564,7 +612,9 @@ fn persist_scored( manifest: body.manifest, nonce, submission_digest, - state: if pass { + state: if promoted { + SubmissionState::Champion + } else if pass { SubmissionState::AwaitingAdmin } else { SubmissionState::Rejected @@ -584,6 +634,10 @@ fn persist_scored( near_duplicate: false, }, ); + if let Some(live) = st.live() { + live.on_persisted(&topic_id, &row.submission_digest, &row.id, promoted) + .await; + } Ok(( StatusCode::CREATED, Json(SubmitResp { @@ -624,7 +678,8 @@ async fn publish_topic( if !admin_ok(&headers, &st.admin_hashes) { return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); } - doc.validate(&st.pin, &supported_custom()) + let registered = st.registered_custom(); + doc.validate(&st.pin, &custom_ids_ref(®istered)) .map_err(|e| topic_err(&e))?; doc.verify_signature(&st.pin).map_err(|e| topic_err(&e))?; if doc.status == TopicStatus::Open && !doc.baseline.is_sealed() { @@ -811,6 +866,7 @@ mod tests { no_nvlink: true, no_nccl_fast_fabric: true, max_inter_node_gbps: Some(12.5), + ..Constraints::default() }, metric: MetricSpec { family: MetricFamily::Throughput, @@ -857,9 +913,14 @@ mod tests { } } - fn seal_topic( + fn seal_topic(pin: &ProofPin, topic: TopicDocument) -> (TopicDocument, BaselineMeasurement) { + seal_topic_with(pin, topic, &[]) + } + + fn seal_topic_with( pin: &ProofPin, mut topic: TopicDocument, + registered: &[&str], ) -> (TopicDocument, BaselineMeasurement) { let recs = synthetic_holdout(STRATUM_SIZE, 1); topic.holdout_commitment = holdout_commitment(&recs); @@ -876,7 +937,7 @@ mod tests { }; topic.baseline.metrics_commitment = meas.commitment(); topic.signature = topic.sign_with(&sk()).expect("sign"); - topic.validate(pin, &[]).expect("valid"); + topic.validate(pin, registered).expect("valid"); topic.verify_signature(pin).expect("sig"); (topic, meas) } @@ -1688,19 +1749,35 @@ mod tests { assert_eq!(created["id"], "adamw-beater-v0"); } + fn custom_metric(custom_id: &str) -> MetricSpec { + MetricSpec { + family: MetricFamily::Custom, + primary: "primary_value".into(), + direction: MetricDirection::Max, + unit: "rate".into(), + epsilon_rel: 0.05, + quality_floor_nll: 0.0, + wall_budget_s: 0, + custom_id: custom_id.into(), + } + } + + /// Custom ids are topic data. With no runner registered on the host an + /// open custom topic is a publish 400 (nobody can compute it); the same + /// document drafts fine, and a topic-minted id with a registered runner + /// opens. Nothing about the id is compiled in. #[tokio::test] - async fn unknown_custom_is_a_publish_400() { + async fn custom_topics_open_only_with_a_registered_runner() { let token = "op"; - let app = app(token); let recs = synthetic_holdout(STRATUM_SIZE, 1); let p = pin(""); let (mut doc, _) = seal_topic(&p, unsigned_topic(&recs)); - doc.id = "custom-unknown-v0".into(); - doc.metric.family = MetricFamily::Custom; - doc.metric.custom_id = "not-implemented".into(); + doc.id = "custom-topic-v0".into(); + doc.payout_mode = proof_task::PayoutMode::Discovery; + doc.metric = custom_metric("topic_minted_metric"); doc.signature = doc.sign_with(&sk()).expect("sign"); let (st, body) = json_req( - app, + app(token), "POST", "/v1/admin/proof/topics", serde_json::to_value(&doc).expect("json"), @@ -1711,37 +1788,12 @@ mod tests { assert!(body["error"] .as_str() .unwrap_or_default() - .contains("custom metric")); - } + .contains("no registered runner")); - #[tokio::test] - async fn harness_success_rate_is_a_listed_custom_and_publishes() { - let token = "op"; - let app = app(token); - let recs = synthetic_holdout(STRATUM_SIZE, 1); - let p = pin(""); - let (mut doc, _) = seal_topic(&p, unsigned_topic(&recs)); - doc.id = "agent-harness-improve-v0".into(); - doc.payout_mode = proof_task::PayoutMode::Discovery; - doc.validation = proof_task::ValidationSpec { - score_on: "Holdout harness success rate (and secondary latency) vs sealed baseline" - .into(), - accept_if: "Reproduced under FLOP/wall budget; no contamination; success rate >= baseline + epsilon".into(), - reject_if: "Unreproduced claim; eval short-circuit; FLOP over budget; near-duplicate of an accepted artifact".into(), - }; - doc.metric = MetricSpec { - family: MetricFamily::Custom, - primary: "success_rate".into(), - direction: MetricDirection::Max, - unit: "rate".into(), - epsilon_rel: 0.05, - quality_floor_nll: 0.0, - wall_budget_s: 0, - custom_id: proof_task::CUSTOM_HARNESS_SUCCESS_RATE.into(), - }; + doc.status = TopicStatus::Draft; doc.signature = doc.sign_with(&sk()).expect("sign"); let (st, body) = json_req( - app, + app(token), "POST", "/v1/admin/proof/topics", serde_json::to_value(&doc).expect("json"), @@ -1749,8 +1801,290 @@ mod tests { ) .await; assert_eq!(st, StatusCode::CREATED, "{body}"); - assert_eq!(body["id"], "agent-harness-improve-v0"); - assert_eq!(body["payout_mode"], "discovery"); + assert_eq!(body["status"], "draft"); + + let (st, body) = json_req( + app_with_custom(Arc::new(FamilyStub::win("topic_minted_metric"))), + "POST", + "/v1/admin/proof/topics", + serde_json::to_value(&{ + let mut open = doc.clone(); + open.status = TopicStatus::Open; + open.signature = open.sign_with(&sk()).expect("sign"); + open + }) + .expect("json"), + Some(token), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{body}"); + assert_eq!(body["metric"]["custom_id"], "topic_minted_metric"); + } + + /// A live scorer with one registered custom id. `wired: false` models a + /// registered runner whose backend (topic VM) is not configured. It + /// crowns every pass and records persist hooks, so the generic promotion + /// / artefact plumbing is exercised without the RLM crates. + struct FamilyStub { + inner: StubScorer, + custom_id: String, + wired: bool, + persisted: std::sync::Mutex>, + } + + impl FamilyStub { + fn win(custom_id: &str) -> Self { + Self { + inner: StubScorer::win(), + custom_id: custom_id.into(), + wired: true, + persisted: std::sync::Mutex::new(Vec::new()), + } + } + + fn unwired(custom_id: &str) -> Self { + Self { + wired: false, + ..Self::win(custom_id) + } + } + } + + #[async_trait] + impl LiveScorer for FamilyStub { + async fn score( + &self, + pin: &ProofPin, + topic: &TopicDocument, + offer: &InferenceOffer, + plan: &ExecutorPlan, + frozen: &str, + artifact: &str, + holdout: &[proof_task::HoldoutRecord], + claim: &str, + ) -> Result { + self.ready_for_topic(topic)?; + let mut doc = self + .inner + .score(pin, topic, offer, plan, frozen, artifact, holdout, claim) + .await?; + if topic.metric.family == MetricFamily::Custom { + doc.harness.custom_value = Some(0.7); + } + Ok(doc) + } + + fn ready_for_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { + if topic.metric.family != MetricFamily::Custom { + return Ok(()); + } + if topic.metric.custom_id != self.custom_id { + return Err(EvalError::RunnerUnwired { + custom_id: topic.metric.custom_id.clone(), + detail: "no registered runner".into(), + }); + } + if !self.wired { + return Err(EvalError::RunnerUnwired { + custom_id: topic.metric.custom_id.clone(), + detail: "topic-vm orchestrator not wired".into(), + }); + } + Ok(()) + } + + fn custom_ids(&self) -> Vec { + vec![self.custom_id.clone()] + } + + async fn auto_promote( + &self, + _topic: &TopicDocument, + _digest: &str, + pass: bool, + primary: Option, + bar: Option, + ) -> bool { + pass && primary.is_some() && bar.is_some() + } + + async fn on_persisted(&self, topic_id: &str, _digest: &str, id: &str, promoted: bool) { + self.persisted + .lock() + .expect("persisted") + .push((topic_id.into(), id.into(), promoted)); + } + } + + fn unsigned_custom_topic(recs: &[proof_task::HoldoutRecord], custom_id: &str) -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.optimizer = "reference-placeholder".into(); + baseline.lr = 1.0; + baseline.schedule = "n/a".into(); + baseline.dtype = "n/a".into(); + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: "custom-topic-v0".into(), + statement: "Placeholder problem scored by a topic-minted custom metric.".into(), + payout_mode: proof_task::PayoutMode::Discovery, + constraints: Constraints { + firecracker_required: true, + model_pin: Some("vendor/model-placeholder".into()), + task_slice: Some("slice-placeholder".into()), + ..Constraints::default() + }, + metric: custom_metric(custom_id), + checklist: vec![proof_task::ChecklistRule { + id: "rule_a".into(), + text: "placeholder rule".into(), + }], + baseline, + holdout_commitment: holdout_commitment(recs), + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } + } + + /// Live host: the throughput topic scores through the harvest stub, the + /// custom topic goes through `scorer` (registered id `topic_minted_metric`). + fn app_with_custom(scorer: Arc) -> Router { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let store = MemoryStore::new(); + let registered = scorer.custom_ids(); + for draft in [ + unsigned_topic(&[]), + unsigned_custom_topic(&[], "topic_minted_metric"), + ] { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, meas) = seal_topic_with(&p, draft, &custom_ids_ref(®istered)); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + let mut sealed = meas.into_sealed(); + if topic.metric.family == MetricFamily::Custom { + sealed.custom_value = Some(0.5); + } + store.set_baseline(&topic.id, sealed).expect("baseline"); + } + let executor = test_executor(&p); + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Lium, + live_scorer: Some(scorer), + offer: Some(offer()), + executor: executor_slot(Some(executor)), + judge_api_key: Some("test-judge-key".into()), + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + + /// A registered runner whose topic VM is not wired: the topic is open + /// but not scorable, and a submit is a 503 with no row. + #[tokio::test] + async fn an_unwired_custom_runner_is_503_with_no_row_and_not_scorable() { + let scorer = Arc::new(FamilyStub::unwired("topic_minted_metric")); + let app = app_with_custom(scorer.clone()); + let (st, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + let ids = |key: &str| -> Vec { + status[key] + .as_array() + .expect(key) + .iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }; + assert!( + ids("open_topics").contains(&"custom-topic-v0".to_owned()), + "{status}" + ); + assert_eq!(ids("scorable_topics"), ["dt-no-ib-v0"], "{status}"); + assert_eq!( + ids("registered_custom"), + ["topic_minted_metric"], + "{status}" + ); + assert_eq!(status["can_score"], true, "{status}"); + + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body( + "custom-artifact", + &serde_json::json!({ "topic_id": "custom-topic-v0" }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let msg = body["error"].as_str().unwrap_or_default(); + assert!(msg.contains("topic_minted_metric"), "{body}"); + assert!(msg.contains("not wired"), "{body}"); + assert_eq!(scorer.inner.hits.load(Ordering::SeqCst), 0); + let (st, list) = json_req(app, "GET", "/v1/submissions", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert!( + list["items"].as_array().is_some_and(Vec::is_empty), + "unwired runner banked a row: {list}" + ); + assert!(scorer.persisted.lock().expect("p").is_empty()); + } + + /// The generic promotion plumbing: a pass the family scorer crowns is + /// persisted as `champion`, and the persist hook fires with the row id. + #[tokio::test] + async fn a_family_scorer_can_crown_a_pass_and_sees_the_persisted_row() { + let scorer = Arc::new(FamilyStub::win("topic_minted_metric")); + let app = app_with_custom(scorer.clone()); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body( + "crowned", + &serde_json::json!({ "topic_id": "custom-topic-v0" }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + assert_eq!(created["state"], "champion", "{created}"); + let id = created["id"].as_str().expect("id").to_owned(); + let persisted = scorer.persisted.lock().expect("p").clone(); + assert_eq!( + persisted, + vec![("custom-topic-v0".to_owned(), id.clone(), true)] + ); + + // A reject is persisted too, never crowned. + let lose = Arc::new(FamilyStub { + inner: StubScorer::lose(), + ..FamilyStub::win("topic_minted_metric") + }); + let (st, created) = json_req( + app_with_custom(lose.clone()), + "POST", + "/v1/submissions", + submit_body("not-crowned", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "rejected", "{created}"); + let persisted = lose.persisted.lock().expect("p").clone(); + assert_eq!(persisted.len(), 1); + assert!(!persisted[0].2, "a reject must not be promoted"); } fn app_lium_missing_judge_key() -> Router { diff --git a/crates/proof-rlm-scorer/Cargo.toml b/crates/proof-rlm-scorer/Cargo.toml index 60c57165f..a4948224b 100644 --- a/crates/proof-rlm-scorer/Cargo.toml +++ b/crates/proof-rlm-scorer/Cargo.toml @@ -13,6 +13,7 @@ async-trait = "0.1" hex = "0.4" proof-canon = { path = "../proof-canon" } proof-eval = { path = "../proof-eval" } +proof-executor = { path = "../proof-executor" } proof-rlm = { path = "../proof-rlm" } proof-rlm-store = { path = "../proof-rlm-store" } proof-score = { path = "../proof-score" } diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index 93908e2af..3f539f58b 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -22,6 +22,7 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; use proof_eval::{EvalError, LiveScorer, ProofEvalDocument, PROOF_METRICS_SCHEMA}; +use proof_executor::ExecutorPlan; use proof_rlm::{ authorize_spend, decide_promote, ArtifactFile, Checklist, CustomRunReport, CustomRunRequest, Lifecycle, LogFile, PromoteDecision, RlmEvent, RlmState, RuleSet, RunnerError, RunnerRegistry, @@ -297,6 +298,7 @@ impl RlmScorer { pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, + plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, claim: &str, @@ -317,7 +319,8 @@ impl RlmScorer { None, claim, ) - .map_err(|e| map_runner(&custom_id, e))?; + .map_err(|e| map_runner(&custom_id, e))? + .with_executor_plan(plan.deadline_s, &plan.config_commitment); let inspected = runner .inspect(&req, &rules) .await @@ -489,6 +492,7 @@ impl LiveScorer for RlmScorer { pin: &ProofPin, topic: &TopicDocument, offer: &InferenceOffer, + plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, _holdout: &[HoldoutRecord], @@ -502,7 +506,15 @@ impl LiveScorer for RlmScorer { self.apply(topic, RlmEvent::SubmissionReceived, frozen_digest) .await?; let out = self - .evaluate(pin, topic, offer, frozen_digest, artifact_digest, claim) + .evaluate( + pin, + topic, + offer, + plan, + frozen_digest, + artifact_digest, + claim, + ) .await; if out.is_err() { // No row will follow a refusal, so the verdict phase is over now. diff --git a/crates/proof-rlm-scorer/src/setup.rs b/crates/proof-rlm-scorer/src/setup.rs index 4e238c6bf..70a42a05f 100644 --- a/crates/proof-rlm-scorer/src/setup.rs +++ b/crates/proof-rlm-scorer/src/setup.rs @@ -163,7 +163,7 @@ impl TopicSetup { deadline_s: topic .eval_executor .max_proof_deadline_s - .unwrap_or(pin.max_proof_deadline_s), + .unwrap_or(pin.max_proof_deadline_s_ceiling), }; let spec = TopicVmSpec::for_topic(&topic.id, self.template.clone(), sandbox); spec.validate()?; diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 4878cabc7..e7d063ef2 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -28,7 +28,8 @@ use axum::http::{Request, StatusCode}; use axum::Router; use http_body_util::BodyExt; use proof_eval::{EvalBackend, EvalError, FamilyMux, LiveScorer, ProofEvalDocument}; -use proof_http::{hash_admin_token, proof_router, AppState}; +use proof_executor::{EvalExecutorOffer, ExecutorPlan}; +use proof_http::{executor_slot, hash_admin_token, proof_router, AppState}; use proof_rlm::fixtures::{offer, pin, pinned_template, topic, FakeOrchestrator}; use proof_rlm::{ FileKeysProbe, OwnerDecision, RlmEvent, RlmState, RunnerRegistry, StaticOwnerHook, @@ -55,6 +56,7 @@ impl LiveScorer for IdleHarvest { _pin: &ProofPin, _topic: &TopicDocument, _offer: &InferenceOffer, + _plan: &ExecutorPlan, _frozen: &str, _artifact: &str, _holdout: &[HoldoutRecord], @@ -64,6 +66,23 @@ impl LiveScorer for IdleHarvest { } } +/// Open `1x` executor on the digest-scoped template of `pin` (host state the +/// live path requires; the RLM path only records its plan commitment). +fn test_executor(pin: &ProofPin) -> EvalExecutorOffer { + let hex = pin.eval_image_digest.trim_start_matches("sha256:"); + let mut o = EvalExecutorOffer { + offer_id: "executor-placeholder".into(), + lium_template_id: format!("proof-eval-{}", hex.get(..12).unwrap_or("unpinned")), + machine_shape: "1x".into(), + max_proof_deadline_s: 3_600, + eval_image_digest: pin.eval_image_digest.clone(), + config_commitment: String::new(), + status: proof_executor::OfferStatus::Open, + }; + o.config_commitment = o.expected_commitment(); + o +} + fn digest(label: &str) -> String { hex::encode(Sha256::digest(label.as_bytes())) } @@ -133,12 +152,14 @@ fn stack(register: bool) -> Stack { let scorer = RlmScorer::new(Arc::new(registry), rlm_store.clone()) .with_artefacts(Some(ArtefactStore::new(&root))); let mux = FamilyMux::new(Arc::new(IdleHarvest)).with_custom_family(Arc::new(scorer)); + let executor = test_executor(&pin); let app = proof_router(AppState { store, pin, backend: EvalBackend::Lium, live_scorer: Some(Arc::new(mux)), offer: Some(offer()), + executor: executor_slot(Some(executor)), judge_api_key: Some("test-judge-key".into()), admin_hashes: Arc::new(vec![hash_admin_token("op")]), epoch: 0, diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs index 89b0bf63d..867d15a0e 100644 --- a/crates/proof-rlm/src/runner.rs +++ b/crates/proof-rlm/src/runner.rs @@ -162,11 +162,27 @@ impl CustomRunRequest { deadline_s: topic .eval_executor .max_proof_deadline_s - .unwrap_or(pin.max_proof_deadline_s), + .unwrap_or(pin.max_proof_deadline_s_ceiling), }, executor_commitment: topic.eval_executor.require_offer_commitment.clone(), }) } + + /// Bind the resolved executor plan the host is running under: the run's + /// deadline is the tighter of the topic's and the plan's, and the + /// commitment of the configuration that actually runs replaces the + /// topic's pin as provenance. + #[must_use] + pub fn with_executor_plan(mut self, deadline_s: u64, config_commitment: &str) -> Self { + if deadline_s > 0 { + self.sandbox.deadline_s = self.sandbox.deadline_s.min(deadline_s); + } + let c = config_commitment.trim(); + if !c.is_empty() { + self.executor_commitment = Some(c.to_owned()); + } + self + } } /// Runner-authored measurement for one submission. @@ -413,7 +429,7 @@ mod tests { assert_eq!(req.rules_digest, rules().digest()); assert_eq!(req.seed, t.baseline.seed); assert!(req.sandbox.firecracker_required); - assert_eq!(req.sandbox.deadline_s, pin().max_proof_deadline_s); + assert_eq!(req.sandbox.deadline_s, pin().max_proof_deadline_s_ceiling); assert_eq!(req.judge.offer_id, offer().offer_id); let dump = serde_json::to_string(&req).expect("json"); assert!( @@ -428,6 +444,15 @@ mod tests { CustomRunRequest::from_topic(&plain, &pin(), &offer(), &rules(), "d", "a", None, ""), Err(RunnerError::NotCustom(_)) )); + let bound = request().with_executor_plan(900, "ab".repeat(32).as_str()); + assert_eq!(bound.sandbox.deadline_s, 900, "plan tightens the deadline"); + assert_eq!( + bound.executor_commitment.as_deref(), + Some("ab".repeat(32).as_str()) + ); + let looser = request().with_executor_plan(u64::MAX, ""); + assert_eq!(looser.sandbox.deadline_s, request().sandbox.deadline_s); + assert_eq!(looser.executor_commitment, request().executor_commitment); let mut tight = topic(); tight.eval_executor.max_proof_deadline_s = Some(600); let req = From 20b5f18b5171c06d453ee24db6deebccb81b6a54 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 13:10:09 +0000 Subject: [PATCH 07/14] docs(proof): dynamic agentic engine, generic bindings, env names PROOF.md: custom ids as topic data (draft freely, open with a registered runner, empty registry by default), rules as a topic-carried vector re-versioned by the RLM into the DB, the per-topic VM isolation boundary, the lifecycle, the registry, artefacts / best pointer / public events, scorable_topics + registered_custom on /v1/status, and the re-sign note. external-miner/proof.md: what a custom-family submission is, the topic-carried checklist a miner must pass, the new 503 / 201-rejected rows, champion state. COMPLETENESS.md: RLM engine row (generic / fail-closed). AGENTS.md: product rule (zero challenge content in git). deploy: PROOF_ARTEFACT_ROOT on the proof-artifacts volume plus the orchestrator / image / owner-key env var names (names only, no values). --- AGENTS.md | 2 +- crates/proof-rlm/src/lib.rs | 3 +- deploy/env/proof-challenge.env.example | 22 ++++ docker-compose.yml | 3 + docs/COMPLETENESS.md | 1 + docs/PROOF.md | 152 ++++++++++++++++++++++--- docs/external-miner/proof.md | 45 +++++++- 7 files changed, 207 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1c9a37f6f..3d3dbf675 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. -**Bounty product rules:** pay is precision x severity, an unpriced `valid` row is not creditable, and the triage-noise ratio stays off the visible score. **Proof product rules (do not weaken):** topics are operator-published signed documents, not a git catalog; a topic may tighten a floor never loosen it; a baseline must be sealed to open; each topic is `wta` (winner takes the topic mass) or `discovery` (pass floor + novelty); global miner score is the **sum** of per-topic masses, not a mean of binary lattices; empty open set / empty eval digest fails closed (`503`); `custom` unknown ids refuse at publish; `harness_success_rate` is listed and fail-closes until the real harness exists; the eval executor is exactly `1x` (pin `gpu_class`), a topic may only tighten `eval_executor.max_proof_deadline_s` / pin `require_offer_commitment`, and there is no per-topic `machine_id`. +**Bounty product rules:** pay is precision x severity, an unpriced `valid` row is not creditable, and the triage-noise ratio stays off the visible score. **Proof product rules (do not weaken):** topics are operator-published signed documents, not a git catalog; a topic may tighten a floor never loosen it; a baseline must be sealed to open; each topic is `wta` (winner takes the topic mass) or `discovery` (pass floor + novelty); global miner score is the **sum** of per-topic masses, not a mean of binary lattices; empty open set / empty eval digest fails closed (`503`); `custom` ids are **topic data** (any well-formed id drafts; a custom topic may **open** only when a runner is registered under its id, and the registry is **empty by default**); anti-cheat rules are a **vector carried by the signed topic** (re-versioned by the topic's RLM into the DB) and are ticked **before any paid inference** (one red item = persisted reject, no spend); the RLM runs **inside a per-topic VM** behind the `TopicVmOrchestrator` boundary (unwired stub = `503`), never on the control-plane host; the eval executor is exactly `1x` (pin `gpu_class`), a topic may only tighten `eval_executor.max_proof_deadline_s` / pin `require_offer_commitment`, and there is no per-topic `machine_id`. **Zero challenge content in git:** no benchmark, metric, model, rule list, repository, or topic catalog is compiled in — the first live topic is a signed document its RLM sets up (`crates/proof-rlm*`, `docs/PROOF.md` § Dynamic agentic engine). Local smoke automates the weights seal step via `weights-smoke` inside `./deploy/scripts/local-e2e.sh --smoke` (see [`deploy/AGENTS.md`](deploy/AGENTS.md) and [`docs/runbooks/local-testnet-e2e.md`](docs/runbooks/local-testnet-e2e.md)). diff --git a/crates/proof-rlm/src/lib.rs b/crates/proof-rlm/src/lib.rs index adbce4b26..2661e11c8 100644 --- a/crates/proof-rlm/src/lib.rs +++ b/crates/proof-rlm/src/lib.rs @@ -31,7 +31,8 @@ //! Persistence (`proof-rlm-store`), the artefact store, and the //! `LiveScorer` glue (`proof-rlm-scorer`) live next door. The topic schema //! (constraints, `eval_executor`, checklist vector, custom id shape) lives -//! in `proof-task`; the live `EvalExecutorOffer` is owned elsewhere. +//! in `proof-task`; the live `EvalExecutorOffer` lives in `proof-executor` +//! and a run request records the resolved plan's deadline and commitment. #![forbid(unsafe_code)] #![allow( diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 7e8eb0ef4..6cd11dd78 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -3,6 +3,9 @@ # Compose requires this file. # Must match deploy/env/postgres.env on the host (see postgres.env.example). +# Also backs the Proof RLM store (topic versions, RLM-written rule versions, +# checklists, lifecycle, artefact metadata, promotions; migration 0020). +# A configured-but-unreachable database is fatal; unset falls back to memory. BASE_DATABASE_URL=postgres://base:base_dev_only_change_me@postgres:5432/base BASE_NETUID=541 @@ -93,3 +96,22 @@ PROOF_SIM_STUB_WIN=false # PROOF_HARVEST_TEMPLATE_ID= # PROOF_HARVEST_GPU_COUNT=1 # PROOF_HARVEST_DEADLINE_SECS= +# RLM engine (custom metric family). Nothing about a challenge lives here: +# metrics, rules, models, and slices come from signed topics. +# Per-submission artefact zips: {root}/{topic_id}/{submission_id}.zip plus +# {topic_id}/best.json and events.jsonl. Compose points this at the +# persistent proof-artifacts volume. +# PROOF_ARTEFACT_ROOT=/var/lib/proof/artefacts +# +# Topic-VM orchestrator (the RLM runs inside a VM per topic; miner code in a +# Firecracker guest under it). Not implemented in this repo yet: with these +# unset every custom topic answers 503 (runner not wired / not registered) +# and nothing rents or spends. Names only — never a value in git, never +# logged, never on /v1/status. +# PROOF_VM_ORCHESTRATOR_URL= +# PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token +# sha256: digest of the RLM VM image the orchestrator boots (unpinned = never boots). +# PROOF_RLM_VM_IMAGE_DIGEST= +# Owner paid-inference key file probed (presence only) at awaiting_owner_keys +# before the baseline run; staged into the topic VM, never read by this host. +# PROOF_RLM_OWNER_INFERENCE_KEY_FILE=/run/base/proof/rlm_owner_inference_key diff --git a/docker-compose.yml b/docker-compose.yml index 406caf9bd..7afeb46bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -265,6 +265,9 @@ services: # Live 1x EvalExecutorOffer (Lium template + proof deadline). Operator # state, no secret; missing/closed → can_score=false / submit 503. PROOF_EVAL_EXECUTOR_OFFER_FILE: /run/base/proof/eval_executor_offer.json + # RLM artefact zips ({topic_id}/{submission_id}.zip, best.json, + # events.jsonl) on the persistent proof-artifacts volume. + PROOF_ARTEFACT_ROOT: /var/lib/proof/artefacts BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} env_file: - path: ./deploy/env/proof-challenge.env diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index cb7b13b1a..6218060d3 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -85,6 +85,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | +| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux`, artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver. **No live VM orchestrator, no registered runner, no challenge content:** every custom topic answers **503** until an operator / RLM registers a runner. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index c30d5cc3e..15c157780 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -132,10 +132,25 @@ baseline + an open topic are on the host. - Global miner proof score = **sum** of per-topic masses, not a mean of binary lattices. Skipped topic = 0 on that topic. Empty open set → `NoScore(ChallengeInternal)`, not a paid 0. -- `custom` metric family: unknown id is **400 at publish**, **503 at score**. - v0 `supported_custom()` lists `harness_success_rate` so the operator can - publish that topic; scoring fail-closes until the real harness fills - `custom_value`. +- `custom` metric family: the `custom_id` is **topic data** + (`[a-z0-9][a-z0-9_-]{1,63}`). There is no compiled-in list of metrics. + Any well-formed id may **draft**; a custom topic may **open** only when a + runner is registered under its id on the host (`400` otherwise), and the + runner registry is **empty by default** — an unregistered or unwired id is + **503 at score**, never a harvest fallback. No benchmark, model, rule + list, or repository is compiled into this repository; the first live + topic is a signed document its RLM sets up, not a code branch. +- Anti-cheat rules are a **vector carried by the signed topic** + (`checklist: [{id, text}]`), re-versioned by the topic's RLM into the + database. Every rule of the current version is ticked with evidence + **before any paid inference**; one red, missing, duplicated, or + evidence-less item is a persisted reject with no spend. +- The RLM runs **inside a VM attributed to its topic**, reached only through + the orchestrator boundary (`TopicVmOrchestrator`). Miner code runs in a + Firecracker guest under that VM when the topic says + `constraints.firecracker_required`. The control plane never runs RLM + logic and never hands the VM a host path or a secret; an unwired + orchestrator is a **503**, not a host-local fallback. - `PROOF_FORCE_SIM` is CI/local opt-in only. Never a fallback. Forbidden on droplet overlays. Under sim, a sealed topic scores with harness numbers relative to the seal (`sim_win_document`); skill-only `sim_document` @@ -179,7 +194,7 @@ Empty digest stays 503 (never invent a sha256). |--------|---------|-----| | `nll` | `holdout_nll` (min) | Beat sealed AdamW by `epsilon_nll >= 0.02`. Per-split NLL regress `<= epsilon_topic_max_regress >= 0.05` | | `throughput` | `tokens_per_sec` (max) or `step_latency_ms` (min) | Requires `flops_budget` **and** `wall_budget_s`. `epsilon_rel >= 0.05`. Quality floor: `holdout_nll <= sealed_nll + quality_floor_nll` (≤ pin 0.02). Eval image enforces comms (e.g. 12.5 Gbit/s); it does not trust the claim | -| `custom` | named inside `proof-eval` | Unknown id refuses. `harness_success_rate` is listed and fail-closes until the harness exists | +| `custom` | `custom_id` minted by the topic | `primary` (topic name, `max` or `min`), `epsilon_rel > 0` relative to the sealed value. Scored by the runner registered under `custom_id` (empty registry by default → **503**); the checklist gate runs first | Holdout: 120 records, stratified 24 each across `web_ood`, `code_ood`, `math_ood`, `longctx` (8k–32k), `multilingual_ood`. `canary_offpath` is @@ -194,10 +209,12 @@ Trust-root keygen is the throwaway owner path in ## HTTP - `GET /health`, `GET /v1/status` — `can_score`, `eval_backend`, `force_sim`, - `live_harvest_wired`, `baseline_sealed`, public pin `inference` judge - defaults (no origin), public `inference_offer` (RLM judge backend), public - `eval_executor` (live `1x` executor offer) and pin `executor` ceilings. - Never leak origins, keys, or holdout records. + `live_harvest_wired`, `baseline_sealed`, `open_topics`, `scorable_topics` + (open topics whose scorer is wired on this host; `can_score` is true when + it is non-empty), `registered_custom` (custom ids with a runner), public + pin `inference` judge defaults (no origin), public `inference_offer` (RLM + judge backend), public `eval_executor` (live `1x` executor offer) and pin + `executor` ceilings. Never leak origins, keys, or holdout records. - `GET /v1/proof/topics`, `GET /v1/proof/topics/{id}` - `GET /v1/proof/executor` — always **200**: `eval_executor` (public offer or `null`), `ready`, `reason` when not ready, and the pin ceilings. @@ -210,9 +227,13 @@ Trust-root keygen is the throwaway owner path in **400**. Miners do **not** bind the judge offer or the executor offer. Zero open / unsealed baseline / empty digest / missing or closed RLM judge backend / missing, closed, or non-`1x` executor / agent down / run cut at - the proof deadline → **503**. Refusals must **not** persist rows. Scored - rows stamp `executor_offer_id` + `executor_commitment` next to the judge + the proof deadline / no registered or wired runner for the topic's + `custom_id` → **503**. Refusals must **not** persist rows. Scored rows + stamp `executor_offer_id` + `executor_commitment` next to the judge `inference_offer_id` + `config_commitment`. +- A pass that the family scorer crowns (custom: green checklist and + `primary >= bar * (1 + epsilon_rel)` direction-aware, bar = sealed value or + reigning best) persists as `champion`; other passes stay `awaiting_admin`. - Submit fields miners must send: `claim` (what the recipe achieved), `declared_flops` (≤ topic budget), `artifact_digest` of a **reproducible train/eval recipe** (code under budget, not weights-only), plus `manifest`. @@ -277,8 +298,10 @@ to `dt-no-ib-v0`; miners still discover it from `GET /v1/proof/topics`. ### `agent-harness-improve-v0` — custom **discovery** -Operator POST, not in git. `custom_id = harness_success_rate` is listed so -this document publishes; scoring fail-closes until the harness exists. +Operator POST, not in git. The `custom_id` is the topic's own name for its +metric; nothing in this repository knows it. The document drafts as-is and +may open once a runner is registered under that id on the host (until then +an `open` publish is **400** and the registry is empty by default). ```json { @@ -291,6 +314,12 @@ this document publishes; scoring fail-closes until the harness exists. "reject_if": "Unreproduced claim; eval short-circuit; FLOP over budget; near-duplicate of an accepted artifact" }, "metric": { "family": "custom", "custom_id": "harness_success_rate", "primary": "success_rate", "direction": "max", "epsilon_rel": 0.05 }, + "constraints": { "firecracker_required": true, "model_pin": "vendor/model", "task_slice": "operator-label", "params": {} }, + "checklist": [ + { "id": "same_seed", "text": "every paid call uses the topic baseline seed" }, + { "id": "no_eval_short_circuit", "text": "the evaluator and the metric path are untouched" } + ], + "eval_executor": { "require_offer_commitment": null, "max_proof_deadline_s": 3600 }, "flops_budget": 2000000000000000000, "status": "draft" } @@ -301,7 +330,102 @@ commitment, a sealed baseline, a signed `inference{…}` that does not loosen pin **judge** defaults, and an sr25519 signature under the `proof` trust-root key. Omitted inference fields inherit the pin; `open` requires a complete resolved judge config (provider + model + mode + tokens). Empty pin -model with no topic `model` is **400** at publish. +model with no topic `model` is **400** at publish. The signing payload is +the whole document, but the generic `constraints.*` knobs, `checklist`, and +`eval_executor` are omitted from it when unset, so a topic that sets none of +them signs to the exact bytes it signed before they existed and older +signatures keep verifying. + +## Dynamic agentic engine (RLM) + +Proof is a **dynamic agentic challenge system**. Every research problem is a +signed topic; each topic's RLM (research lifecycle manager) runs **inside a +VM attributed to that topic**, where it writes the anti-cheat rules, runs the +baseline, inspects and runs miner submissions, and promotes the best +artefact. The binary is the orchestrator: schema, DB, isolation boundary, +artefact store, runner registry. Nothing about a challenge — no benchmark, +metric, model, rule list, or repository — is compiled in. Crates: +`proof-rlm` (core), `proof-rlm-store` (Postgres / memory), `proof-rlm-scorer` +(`LiveScorer` + artefacts + setup driver), `proof-canon` (canonical JSON + +id shapes shared with `proof-task`). + +### Topic-carried, generic bindings (signed) + +| Field | Meaning | +|-------|---------| +| `metric.custom_id` | Topic-minted metric id, `[a-z0-9][a-z0-9_-]{1,63}`. Draft with any; open needs a registered runner | +| `constraints.firecracker_required` | Miner code runs only inside a Firecracker guest under the topic VM | +| `constraints.model_pin` | `vendor/model[:tag]` every paid call must name (shape-checked only) | +| `constraints.task_slice` | Opaque label the runner interprets; the control plane does not | +| `constraints.params` | ≤32 opaque `slug → printable` runner params | +| `checklist` | ≤64 `{id, text}` anti-cheat rules (unique slug ids), version 1 of the rule set | +| `eval_executor.require_offer_commitment` | 64-hex pin against the live `1x` `EvalExecutorOffer` (`proof-executor`) | +| `eval_executor.max_proof_deadline_s` | Tighten-only against pin `max_proof_deadline_s_ceiling` (7200 s; the live offer may be shorter) | + +### Rules → DB, not logs + +Rule sets are versioned per topic in `proof_rule_version` (migration +`0020_proof_rlm.sql`): v1 is the signed vector, later versions are what the +RLM writes (`source = rlm`) or the operator edits. A checklist binds to a +rule version **and** its digest, so it cannot be replayed against edited +rules; it is green only when every rule of that version is ticked with +evidence and passes. Every checklist (red or green), every lifecycle +transition, the baseline measurement, artefact metadata, and every promotion +event land in the DB (`proof_checklist`, `proof_lifecycle_event`, +`proof_baseline_measurement`, `proof_artefact`, `proof_promotion_event`, +`proof_topic_version`). Tables are append-only for `base_app`; "current +best" is the newest promotion row. `BASE_DATABASE_URL` selects Postgres; a +configured-but-unreachable database is fatal, an unset one falls back to the +in-memory store with a warning. + +### Lifecycle + +`draft → owner_presend → awaiting_owner_keys → provisioning → baselining → +open ⇄ evaluating → promoting → open … → closed`. `owner_presend` is an +`askUser`-style hook (no hook = cannot advance; decline = back to draft); +`awaiting_owner_keys` probes `PROOF_RLM_OWNER_INFERENCE_KEY_FILE` for +presence only. `TopicSetup` drives the ceremony over the VM boundary +(provision → RLM `ProposeRules` → rules vN in DB → `Baseline` job → +measurement in DB) and `mark_sealed` moves `baselining → open` after the +operator seals `custom_value` and re-signs. A re-run resumes from the +persisted state. + +### Isolation boundary + +`TopicVmOrchestrator` (create / attach / run / teardown-or-retain) is the only +way RLM work happens. `VmJob`s carry public data (signed topic, digests, +rule set, request) — never a host path, a key, or a judge origin. The shipped +orchestrator is `UnwiredVmOrchestrator` (refuses, names +`PROOF_VM_ORCHESTRATOR_URL` / `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`; +`PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM VM image). The generic +`VmBackedRunner` turns inspect / evaluate into VM jobs; registering it under +a `custom_id` is an operator / RLM action. The Lium harvest for +`nll` / `throughput` and the live `1x` `EvalExecutorOffer` (`proof-executor`) +govern the harvest rent; on the custom path each run request records the +resolved executor plan's deadline (tighter of topic and plan) and +`config_commitment` as provenance, and the row stamps `executor_commitment` +like every other scored row. + +### Runner registry + +`custom_id → CustomRunner`, **empty by default**. `GET /v1/status` lists +`registered_custom`. An open custom topic whose id is not registered (or +whose runner reports its backend unwired) is open but not in +`scorable_topics`; a submit is **503** with the root cause and no row. +Publishing an `open` custom topic without a registered runner is **400**; +the same document drafts fine. + +### Artefacts and promotion + +Every scored row leaves `$PROOF_ARTEFACT_ROOT/{topic_id}/{submission_id}.zip` +(`manifest.json`, `artifact/`, `report.json`, `checklist.json`, +`baseline_ref.json`, `logs/`; a red-checklist reject ships no report), plus +`best.json` (current best pointer) and `events.jsonl` (public `scored` / +`promoted` events). Default root `/artefacts`; compose sets +`/var/lib/proof/artefacts` on the `proof-artifacts` volume. **Promote:** a +pass with a green checklist whose primary beats the bar (sealed value or +reigning best) by `epsilon_rel`, direction-aware, persists as `champion`, +gets a promotion row (with the displaced best), and moves the pointer. `GET /v1/status` exposes pin `inference` public judge defaults (`provider`, `model`, `mode`, token caps) and `inference_offer` **public fields only** diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index a82352993..d13788a40 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -88,6 +88,8 @@ rented. | `inference_offer` | Public RLM **judge** backend (id, kind, mode, model_ref, token caps, commitment, status). Missing/closed/misconfigured → **503**. You do not pass an offer id | | `eval_executor` | Public `1x` **executor**: the Lium machine class your recipe is re-run on (`lium_template_id`, `machine_shape`, `max_proof_deadline_s`, commitment, status). Your recipe must finish inside `max_proof_deadline_s` (≤ pin ceiling 7200 s; a topic may name a shorter one) on **one** GPU — the host never rents more. Missing/closed/any shape but `1x` → **503**. You do not pass or rent it | | `open_topics` empty | No currently `open` signed topic with a sealed baseline → **503** | +| `scorable_topics` | Open topics whose scorer is wired on this host. An open topic **not** listed here (a `custom` topic whose runner is not registered or not wired) answers **503** | +| `registered_custom` | Custom metric ids with a registered runner. Nothing is compiled in; ids come from signed topics | | `baseline_sealed: false` | An open topic without `script_sha256` + `metrics_commitment` → **503** | | `live_harvest_wired: false` | Live RLM harvest is not connected → **503** | @@ -109,8 +111,10 @@ Each topic is a signed document. Read at least: |-------|------------------------| | `id` | The `topic_id` you submit against | | `statement` | The research problem in English | -| `constraints` | Fabric / comms caps the eval image enforces (it never trusts the claim) | -| `metric.family` | `nll` \| `throughput` \| `custom` | +| `constraints` | Fabric / comms caps the eval image enforces (it never trusts the claim). Custom topics may add `firecracker_required`, `model_pin`, an opaque `task_slice`, and `params` | +| `checklist` | Anti-cheat rules `[{id, text}]` the topic's RLM ticks over your artefact **before any paid inference** | +| `eval_executor` | Executor commitment (`require_offer_commitment`, tighten-only `max_proof_deadline_s`) | +| `metric.family` | `nll` \| `throughput` \| `custom` (`metric.custom_id` names the metric; it is topic data) | | `flops_budget` | Hard cap. `declared_flops` must be `≤` this | | `epsilon_nll` / `epsilon_topic_max_regress` / throughput knobs | Pass-rule epsilons. A topic may **tighten** a pin floor, never loosen it | | `payout_mode` | `wta` or `discovery` | @@ -237,8 +241,8 @@ curl -sS https://network.cortex.foundation/challenge/proof/v1/submissions/ | `state` | Meaning | |---------|---------| | `awaiting_admin` | Clean pass; mass recorded. Operator audit is informational. | -| `rejected` | Gates failed (contamination, unreproduced claim, NLL miss, …). No rent on pre-eval rejects. | -| `champion` | Optional operator promote. Proof pays on pass, not on a crown. | +| `rejected` | Gates failed (contamination, unreproduced claim, NLL miss, red anti-cheat checklist, …). No rent and no paid inference on pre-eval rejects. | +| `champion` | Promoted: operator crown, or automatic on custom topics when a pass beats the current best by `epsilon_rel` with a green checklist. Proof pays on pass, not on a crown. | Poll `GET /challenge/proof/v1/submissions/{id}` for the verdict envelope below. While `can_score` is `false`, the POST itself answers **503** and @@ -263,11 +267,14 @@ Refusals (**400** / **503**) do **not** persist a submission row. | **503** missing / closed RLM judge backend | Live `InferenceOffer` not scoring | no | no | | **503** missing / closed / non-`1x` executor | Live `eval_executor` cannot rent the `1x` machine | no | no | | **503** `proof deadline … exceeded` | Your recipe did not finish inside `max_proof_deadline_s`; the body carries the run's `stdout_tail` | no | no (pod torn down) | +| **503** `custom metric … has no registered runner` / `not wired` | The topic's `custom_id` has no runner on this host, or its topic VM is not configured | no | no | | **201** `rejected` + `contamination_evidence_missing` | Empty manifest | **yes** (rejected) | **no** | | **201** `rejected` + contamination | Holdout shard / corpus id in `manifest` | **yes** (rejected) | **no** | +| **201** `rejected` + `anti-cheat checklist red` | A topic rule failed on your artefact | **yes** (rejected) | **no** (no paid inference) | Contamination (including empty evidence) is a **reject, no rent**. It is -not a 400 and not a 503. +not a 400 and not a 503. A red anti-cheat checklist is the same shape: a +persisted reject with no spend. ## Agent verdict (RLM judge) @@ -324,6 +331,34 @@ Quality floor: `holdout_nll <= sealed_nll + quality_floor_nll` (pin max 0.02). Speed is not free. The eval image enforces comms (for example **12.5 Gbit/s**); it does not trust the claim. +### `custom` family (topic-minted metrics) + +Primary: `metric.primary` (`max` or `min`, as the topic says). Win: +beat the sealed value by `metric.epsilon_rel` relative +(`primary >= sealed * (1 + epsilon_rel)` for `max`). The metric is computed +by the runner registered on the host under `metric.custom_id`; nothing +about it is compiled into the network. If the topic sets +`constraints.firecracker_required`, your code runs only inside a Firecracker +guest under the topic's own VM; if it sets `constraints.model_pin`, every +paid call must name exactly that model; `task_slice` / `params` are opaque +runner inputs the topic defines. + +**Anti-cheat checklist — every rule in the topic's `checklist` (current +version) must pass before a single paid inference call is made.** Read the +rule texts in `ctx proof topics`; they are the contract. One red, missing, +duplicated, or evidence-less item is a persisted `rejected` row with no +spend. The rules may be re-versioned by the topic's RLM; the version you were +ticked against is recorded with your row. + +A clean pass that beats the current best (sealed value or reigning best) by +`epsilon_rel` is promoted automatically: the row is `champion` and the +operator archive keeps your artefact, `report.json`, and `checklist.json` +under `{topic_id}/{submission_id}.zip`. + +If the topic's `custom_id` is not in `registered_custom`, the topic is +`open` but not in `scorable_topics`, and submits answer **503** (`no +registered runner`). Nothing is stored and nothing is spent. + ### Paid mass A clean pass is eligible. `wta` / `discovery` then assign that topic's share From 3211601b4d10a65fddee9716bcce5dfd73a46f66 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:26:55 +0000 Subject: [PATCH 08/14] fix(proof-rlm-scorer): pass the submitted artifact locator to the runner Greptile P1 on #243: the custom submission flow accepted `artifact_uri` but `RlmScorer` built `CustomRunRequest` with `None`, so the runner got the digest and no way to retrieve the bytes. - `LiveScorer::score`, `eval_after_freeze`, and `FamilyMux` take the miner's `artifact_uri`; `proof-http` forwards `body.artifact_uri`; the Lium harvest ignores it (it fetches by digest). - `RlmScorer::evaluate` threads it into `CustomRunRequest::from_topic`; the request normalises whitespace/empty to `None`. - e2e: the inspect and evaluate VM jobs carry the submitted locator with the digest; unit test for the normalisation. --- crates/proof-eval/src/lib.rs | 21 +++++++++++++++ crates/proof-harvest/src/lib.rs | 7 ++++- crates/proof-http/src/lib.rs | 15 ++++++++++- crates/proof-rlm-scorer/src/scorer.rs | 6 ++++- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 34 ++++++++++++++++++------ crates/proof-rlm/src/runner.rs | 26 ++++++++++++++++-- 6 files changed, 96 insertions(+), 13 deletions(-) diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 3cc40c2df..9429c6c30 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -293,6 +293,9 @@ pub trait LiveScorer: Send + Sync { /// /// `offer` is the RLM judge the image calls; `plan` is the resolved `1x` /// rent ([`Self::plan`]) the image is run under. Both are host state. + /// `artifact_uri` is the miner-supplied locator of the bytes behind + /// `artifact_digest` (the runner fetches and digest-checks them); the + /// digest alone is not enough to retrieve an artefact. #[allow(clippy::too_many_arguments)] async fn score( &self, @@ -302,6 +305,7 @@ pub trait LiveScorer: Send + Sync { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, + artifact_uri: Option<&str>, holdout: &[HoldoutRecord], claim: &str, ) -> Result; @@ -416,6 +420,7 @@ impl LiveScorer for FamilyMux { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, + artifact_uri: Option<&str>, holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -427,6 +432,7 @@ impl LiveScorer for FamilyMux { plan, frozen_digest, artifact_digest, + artifact_uri, holdout, claim, ) @@ -885,6 +891,9 @@ pub fn sim_win_document( } /// Score only after the submission digest is frozen and a topic is open. +/// +/// `artifact_uri` travels to the live scorer untouched: it is the miner's +/// locator for the bytes behind `artifact_digest`, never trusted beyond that. #[allow(clippy::too_many_arguments)] pub async fn eval_after_freeze( pin: &ProofPin, @@ -893,6 +902,7 @@ pub async fn eval_after_freeze( executor: Option<&EvalExecutorOffer>, frozen_digest: &str, artifact_digest: &str, + artifact_uri: Option<&str>, holdout: &[HoldoutRecord], claim: &str, backend: EvalBackend, @@ -961,6 +971,7 @@ pub async fn eval_after_freeze( &resolved, frozen_digest, artifact_digest, + artifact_uri, holdout, claim, ) @@ -1099,6 +1110,7 @@ mod tests { _plan: &ExecutorPlan, frozen: &str, artifact: &str, + _artifact_uri: Option<&str>, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -1130,6 +1142,7 @@ mod tests { Some(&executor(&pin(""))), "d", "art", + None, &recs, "claim", EvalBackend::Lium, @@ -1151,6 +1164,7 @@ mod tests { Some(&executor(&pin(&format!("sha256:{}", "ab".repeat(32))))), "d", "art", + None, &recs, "claim", EvalBackend::Lium, @@ -1178,6 +1192,7 @@ mod tests { Some(&executor(&p)), "digest-a", "art", + None, &recs, "claim", EvalBackend::Lium, @@ -1383,6 +1398,7 @@ mod tests { None, "digest-a", "art", + None, &recs, "claim", EvalBackend::Lium, @@ -1405,6 +1421,7 @@ mod tests { Some(&executor(&p)), "digest-a", "art", + None, &recs, "claim", EvalBackend::Lium, @@ -1464,6 +1481,7 @@ mod tests { _plan: &ExecutorPlan, _frozen: &str, _artifact: &str, + _artifact_uri: Option<&str>, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -1567,6 +1585,7 @@ mod tests { &plan, "d", "a", + None, &recs, "c", ) @@ -1687,6 +1706,7 @@ mod tests { Some(&executor(&p)), "digest-a", "art", + None, &recs, "claim", EvalBackend::Sim, @@ -1721,6 +1741,7 @@ mod tests { Some(&executor(&p)), "digest-a", "art", + None, &recs, "claim", EvalBackend::Lium, diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 06bfc28bb..62b7b04cd 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -661,6 +661,9 @@ impl LiveScorer for LiumProofHarvest { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, + // The digest-pinned image fetches by digest from the artifact store; + // the miner locator is a custom-family concern. + _artifact_uri: Option<&str>, holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -774,7 +777,9 @@ mod tests { ) -> Result { let plan = harvest.plan(pin, topic, executor)?; harvest - .score(pin, topic, offer, &plan, frozen, artifact, holdout, claim) + .score( + pin, topic, offer, &plan, frozen, artifact, None, holdout, claim, + ) .await } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 70db29e8a..f306c3aae 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -417,6 +417,7 @@ async fn submit( executor.as_ref(), &submission_digest, &artifact, + body.artifact_uri.as_deref(), &holdout, &body.claim, st.backend, @@ -975,6 +976,7 @@ mod tests { _plan: &ExecutorPlan, frozen: &str, artifact: &str, + _artifact_uri: Option<&str>, _holdout: &[proof_task::HoldoutRecord], _claim: &str, ) -> Result { @@ -1860,13 +1862,24 @@ mod tests { plan: &ExecutorPlan, frozen: &str, artifact: &str, + artifact_uri: Option<&str>, holdout: &[proof_task::HoldoutRecord], claim: &str, ) -> Result { self.ready_for_topic(topic)?; let mut doc = self .inner - .score(pin, topic, offer, plan, frozen, artifact, holdout, claim) + .score( + pin, + topic, + offer, + plan, + frozen, + artifact, + artifact_uri, + holdout, + claim, + ) .await?; if topic.metric.family == MetricFamily::Custom { doc.harness.custom_value = Some(0.7); diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index 3f539f58b..601422469 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -293,6 +293,7 @@ impl RlmScorer { ); } + #[allow(clippy::too_many_arguments)] async fn evaluate( &self, pin: &ProofPin, @@ -301,6 +302,7 @@ impl RlmScorer { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, + artifact_uri: Option<&str>, claim: &str, ) -> Result { let custom_id = topic.metric.custom_id.trim().to_owned(); @@ -316,7 +318,7 @@ impl RlmScorer { &rules, frozen_digest, artifact_digest, - None, + artifact_uri, claim, ) .map_err(|e| map_runner(&custom_id, e))? @@ -495,6 +497,7 @@ impl LiveScorer for RlmScorer { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, + artifact_uri: Option<&str>, _holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -513,6 +516,7 @@ impl LiveScorer for RlmScorer { plan, frozen_digest, artifact_digest, + artifact_uri, claim, ) .await; diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index e7d063ef2..aedd1f5d1 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -5,7 +5,8 @@ //! temp artefact root. //! //! Covers: an unregistered `custom_id` is a 503 with no row; a green -//! checklist scores and is crowned against the sealed value; a red checklist +//! checklist scores and is crowned against the sealed value, with the +//! miner's artefact locator reaching the runner; a red checklist //! is a persisted reject with **zero** paid runs; a later pass below the best //! stays `awaiting_admin`; every scored row leaves its zip, the crown leaves //! `best.json` + a promotion row, and the store holds rules v1, every @@ -59,6 +60,7 @@ impl LiveScorer for IdleHarvest { _plan: &ExecutorPlan, _frozen: &str, _artifact: &str, + _artifact_uri: Option<&str>, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -294,19 +296,34 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { ); // 1. Green checklist, primary 0.70 > 0.50 * 1.02: scored and crowned. - let (st, created) = json_req( - app.clone(), - "POST", - "/v1/submissions", - submit_body(&tid, "artifact-a"), - ) - .await; + // The miner's artefact locator travels to the runner with the digest. + let uri = "https://example.invalid/artefacts/artifact-a.zip"; + let mut body_a = submit_body(&tid, "artifact-a"); + body_a["artifact_uri"] = serde_json::json!(uri); + let (st, created) = json_req(app.clone(), "POST", "/v1/submissions", body_a).await; assert_eq!(st, StatusCode::CREATED, "{created}"); assert_eq!(created["eligible"], true, "{created}"); assert_eq!(created["state"], "champion", "{created}"); let id_a = created["id"].as_str().unwrap().to_owned(); assert_eq!(paid_runs(&orchestrator), 1); assert_eq!(orchestrator.created(), 1, "one topic, one vm"); + let requests: Vec = orchestrator + .jobs() + .into_iter() + .filter_map(|j| match j { + VmJob::Inspect { request, .. } | VmJob::Evaluate { request, .. } => Some(request), + _ => None, + }) + .collect(); + assert_eq!(requests.len(), 2, "one inspect, one evaluate"); + for req in &requests { + assert_eq!( + req.artifact_uri.as_deref(), + Some(uri), + "runner must receive the submitted locator" + ); + assert_eq!(req.artifact_digest, digest("artifact-a")); + } let (_, row) = json_req( app.clone(), @@ -317,6 +334,7 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { .await; assert_eq!(row["verdict"]["pass"], true, "{row}"); assert!((row["verdict"]["harness"]["custom_value"].as_f64().unwrap() - 0.7).abs() < 1e-12); + assert_eq!(row["artifact_uri"], uri, "{row}"); assert!(row["verdict"]["agent"]["rationale"] .as_str() .unwrap() diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs index 867d15a0e..21206828f 100644 --- a/crates/proof-rlm/src/runner.rs +++ b/crates/proof-rlm/src/runner.rs @@ -70,7 +70,9 @@ pub struct CustomRunRequest { pub submission_digest: String, /// Artefact digest (sha256 of the recipe bytes). pub artifact_digest: String, - /// Optional locator for the same bytes. + /// Miner-supplied locator for the same bytes. The runner fetches from it + /// inside the topic VM and checks the digest; it is never trusted beyond + /// that. Empty / whitespace is `None`. pub artifact_uri: Option, /// Miner claim (English). pub claim: String, @@ -146,7 +148,10 @@ impl CustomRunRequest { epsilon_rel: topic.metric.epsilon_rel, submission_digest: submission_digest.trim().to_owned(), artifact_digest: artifact_digest.trim().to_ascii_lowercase(), - artifact_uri: artifact_uri.map(|u| u.trim().to_owned()), + artifact_uri: artifact_uri + .map(str::trim) + .filter(|u| !u.is_empty()) + .map(str::to_owned), claim: claim.to_owned(), constraints: topic.constraints.clone(), rules_version: rules.version, @@ -428,6 +433,11 @@ mod tests { assert_eq!(req.rules_version, 1); assert_eq!(req.rules_digest, rules().digest()); assert_eq!(req.seed, t.baseline.seed); + assert_eq!( + req.artifact_uri.as_deref(), + Some("https://example.invalid/artifact.zip"), + "the miner locator reaches the runner" + ); assert!(req.sandbox.firecracker_required); assert_eq!(req.sandbox.deadline_s, pin().max_proof_deadline_s_ceiling); assert_eq!(req.judge.offer_id, offer().offer_id); @@ -459,6 +469,18 @@ mod tests { CustomRunRequest::from_topic(&tight, &pin(), &offer(), &rules(), "d", "a", None, "") .expect("request"); assert_eq!(req.sandbox.deadline_s, 600); + let blank = CustomRunRequest::from_topic( + &topic(), + &pin(), + &offer(), + &rules(), + "d", + "a", + Some(" "), + "", + ) + .expect("request"); + assert_eq!(blank.artifact_uri, None, "whitespace is no locator"); } #[test] From 40456c3d325e3ce009cfaa0a3b05f56605023a21 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:30:35 +0000 Subject: [PATCH 09/14] fix(proof-rlm): verdicts carry runner-measured flops, fail-closed Greptile P1 on #243: every custom-run verdict reported `flops_used: 0` while neither the run request nor the verified report carried a usage measurement, so the judge compared zero with the signed budget and an over-budget run could pass. - `CustomRunRequest` carries the topic's `flops_budget`; `CustomRunReport` carries the runner-measured `flops_used`. `flops_used_for` is the only way to read it: `None` against a budget is `ReportError::FlopsMissing` (report binding fails closed, 503, no row); zero is only a measurement. - `RlmScorer` puts that figure in the verdict. A measurement over the budget is `reject` + `flops_over_budget`, which the judge fails on; the pre-spend reject keeps `0` because nothing ran. - `TopicSetup::baseline` refuses a baseline that spent over the topic budget or measured nothing (`SetupError::BaselineOverBudget`). - Fixtures: `FakeOrchestrator::set_flops_used`; tests for the binding, the persisted over-budget reject, the 503, and the setup refusals. --- crates/proof-rlm-scorer/src/scorer.rs | 78 ++++++++---- crates/proof-rlm-scorer/src/setup.rs | 18 ++- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 152 +++++++++++++++++++++-- crates/proof-rlm/src/fixtures_tests.rs | 10 ++ crates/proof-rlm/src/runner.rs | 75 +++++++++++ 5 files changed, 302 insertions(+), 31 deletions(-) diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index 601422469..def17b08f 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -10,7 +10,9 @@ //! persisted red or green; //! 4. red → **reject document, no paid inference**; //! 5. green → [`SpendToken`] → **evaluate** (the only paid step) → report → -//! `custom_value = primary_value`; +//! `custom_value = primary_value`, `flops_used` = the runner's +//! measurement (a report without one is not evidence; over budget is a +//! reject the judge fails on); //! 6. on persist: artefact zip + metadata row + public event; on promotion: //! promotion row, `best.json`, lifecycle `promoting → open`. //! @@ -215,6 +217,8 @@ impl RlmScorer { kind: ProofKind, reproduced: bool, claim_holds: bool, + flops_used: u64, + cheat_codes: Vec, rationale: String, ) -> AgentVerdict { AgentVerdict { @@ -223,13 +227,9 @@ impl RlmScorer { claim_holds_public: claim_holds, contamination: false, canary_hit: false, - flops_used: 0, + flops_used, flops_budget: topic.flops_budget, - cheat_codes: if kind == ProofKind::Clean { - Vec::new() - } else { - vec![ProofCheatCode::Other] - }, + cheat_codes, rationale, topic_id: topic.id.clone(), family: topic.metric.family, @@ -293,6 +293,44 @@ impl RlmScorer { ); } + /// Verdict for a verified paid run: the runner's measured FLOPs are the + /// verdict's usage, and a measurement over the topic budget is a reject + /// (the judge's `FlopsOverBudget` gate then fails the row). + fn paid_verdict( + topic: &TopicDocument, + req: &CustomRunRequest, + report: &CustomRunReport, + rules_version: u32, + ) -> Result { + let flops_used = report + .flops_used_for(req) + .map_err(|e| EvalError::NoVerdict(e.to_string()))?; + let measured = format!( + "{}: {} = {:.6}; checklist green (rules v{rules_version}); sandboxed={}; flops_used={flops_used}", + req.custom_id, req.primary, report.primary_value, report.sandboxed + ); + if flops_used > req.flops_budget { + return Ok(Self::agent( + topic, + ProofKind::Reject, + true, + report.claim_holds, + flops_used, + vec![ProofCheatCode::FlopsOverBudget], + format!("{measured}; over the topic budget {}", req.flops_budget), + )); + } + Ok(Self::agent( + topic, + ProofKind::Clean, + true, + report.claim_holds, + flops_used, + Vec::new(), + measured, + )) + } + #[allow(clippy::too_many_arguments)] async fn evaluate( &self, @@ -335,7 +373,15 @@ impl RlmScorer { "anti-cheat {red} (rules v{}); no paid inference", rules.version ); - let agent = Self::agent(topic, ProofKind::Reject, false, false, rationale); + let agent = Self::agent( + topic, + ProofKind::Reject, + false, + false, + 0, + vec![ProofCheatCode::Other], + rationale, + ); self.stash( pin, topic, @@ -357,21 +403,7 @@ impl RlmScorer { run.report .verify(&req) .map_err(|e| EvalError::NoVerdict(e.to_string()))?; - let rationale = format!( - "{}: {} = {:.6}; checklist green (rules v{}); sandboxed={}", - req.custom_id, - req.primary, - run.report.primary_value, - rules.version, - run.report.sandboxed - ); - let agent = Self::agent( - topic, - ProofKind::Clean, - true, - run.report.claim_holds, - rationale, - ); + let agent = Self::paid_verdict(topic, &req, &run.report, rules.version)?; let doc = Self::document(pin, topic, &req, agent, Some(run.report.primary_value)); self.stash( pin, diff --git a/crates/proof-rlm-scorer/src/setup.rs b/crates/proof-rlm-scorer/src/setup.rs index 70a42a05f..132fa784e 100644 --- a/crates/proof-rlm-scorer/src/setup.rs +++ b/crates/proof-rlm-scorer/src/setup.rs @@ -4,7 +4,8 @@ //! ```text //! draft ──submit_for_review──▶ owner_presend ──(owner hook)──▶ awaiting_owner_keys //! ──(key probe)──▶ provisioning ──(orchestrator.create)──▶ baselining -//! ──(ProposeRules job → rules vN in store; Baseline job → measurement in store)──▶ +//! ──(ProposeRules job → rules vN in store; Baseline job → measurement in store, +//! refused when it spends over the topic budget or measures nothing)──▶ //! returns SetupOutcome; the operator seals custom_value, re-signs `open`, //! and calls `mark_sealed` (baselining → open). //! ``` @@ -47,6 +48,14 @@ pub enum SetupError { /// The baseline report did not bind to the request. #[error("baseline report: {0}")] Report(#[from] proof_rlm::ReportError), + /// The baseline run measured more FLOPs than the topic budget allows. + #[error("baseline spent {used} FLOPs over the topic budget {budget}")] + BaselineOverBudget { + /// Runner-measured usage. + used: u64, + /// Topic budget. + budget: u64, + }, /// The owner declined at presend; the topic is back at draft. #[error("owner declined; topic {0:?} returned to draft")] Declined(String), @@ -248,6 +257,13 @@ impl TopicSetup { } }; report.verify(&request)?; + let used = report.flops_used_for(&request)?; + if used > topic.flops_budget { + return Err(SetupError::BaselineOverBudget { + used, + budget: topic.flops_budget, + }); + } self.store .put_baseline(&BaselineRow { topic_id: topic.id.clone(), diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index aedd1f5d1..9d68f1b97 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -6,9 +6,11 @@ //! //! Covers: an unregistered `custom_id` is a 503 with no row; a green //! checklist scores and is crowned against the sealed value, with the -//! miner's artefact locator reaching the runner; a red checklist -//! is a persisted reject with **zero** paid runs; a later pass below the best -//! stays `awaiting_admin`; every scored row leaves its zip, the crown leaves +//! miner's artefact locator reaching the runner and the runner's measured +//! FLOPs in the verdict; a red checklist is a persisted reject with **zero** +//! paid runs; a later pass below the best stays `awaiting_admin`; an +//! over-budget measurement is a persisted reject and a missing one is a 503; +//! every scored row leaves its zip, the crown leaves //! `best.json` + a promotion row, and the store holds rules v1, every //! checklist, and the lifecycle. Then the setup driver walks //! `draft → … → baselining` with RLM-written rules and a baseline in the @@ -323,6 +325,7 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { "runner must receive the submitted locator" ); assert_eq!(req.artifact_digest, digest("artifact-a")); + assert_eq!(req.flops_budget, topic.flops_budget); } let (_, row) = json_req( @@ -334,6 +337,11 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { .await; assert_eq!(row["verdict"]["pass"], true, "{row}"); assert!((row["verdict"]["harness"]["custom_value"].as_f64().unwrap() - 0.7).abs() < 1e-12); + assert_eq!( + row["verdict"]["agent"]["flops_used"], 1u64, + "verdict carries the runner's measured usage, not zero" + ); + assert_eq!(row["verdict"]["agent"]["flops_budget"], topic.flops_budget); assert_eq!(row["artifact_uri"], uri, "{row}"); assert!(row["verdict"]["agent"]["rationale"] .as_str() @@ -403,6 +411,10 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { .unwrap() .contains("rule_b")); assert!(row_b["verdict"]["harness"]["custom_value"].is_null()); + assert_eq!( + row_b["verdict"]["agent"]["flops_used"], 0u64, + "nothing ran, nothing was spent" + ); let names_b = zip_names(&root.join(&tid).join(format!("{id_b}.zip"))); assert!(!names_b.iter().any(|n| n == "report.json"), "{names_b:?}"); let digest_b = row_b["submission_digest"].as_str().unwrap(); @@ -463,6 +475,104 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { let _ = std::fs::remove_dir_all(&root); } +/// The signed budget binds the runner's measurement, not the miner's +/// declaration: a run measured over budget is a persisted reject even with a +/// winning primary, and a report with no measurement is refused with no row. +#[tokio::test] +async fn an_over_budget_or_unmeasured_run_never_passes() { + let Stack { + app, + orchestrator, + rlm_store, + root, + topic, + } = stack(true); + let tid = topic.id.clone(); + let budget = topic.flops_budget; + + orchestrator.set_flops_used(Some(budget + 1)); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-over"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], false, "{created}"); + assert_eq!(created["state"], "rejected", "{created}"); + let id = created["id"].as_str().unwrap().to_owned(); + let (_, row) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + ) + .await; + assert_eq!(row["verdict"]["pass"], false, "{row}"); + assert_eq!(row["verdict"]["agent"]["flops_used"], budget + 1, "{row}"); + assert_eq!(row["verdict"]["agent"]["verdict"], "reject", "{row}"); + assert!((row["verdict"]["harness"]["custom_value"].as_f64().unwrap() - 0.7).abs() < 1e-12); + let failed = row["verdict"]["failed"].to_string(); + assert!(failed.contains("flops_over_budget"), "{failed}"); + assert!(row["verdict"]["agent"]["cheat_codes"] + .as_array() + .unwrap() + .iter() + .any(|c| c == "flops_over_budget")); + assert!(row["verdict"]["agent"]["rationale"] + .as_str() + .unwrap() + .contains("over the topic budget")); + assert!(ArtefactStore::new(&root).best(&tid).is_none()); + assert!(rlm_store.best(&tid).await.unwrap().is_none()); + assert!(root.join(&tid).join(format!("{id}.zip")).is_file()); + + // No measurement at all: not evidence, 503, no row. + orchestrator.set_flops_used(None); + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-unmeasured"), + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap() + .contains("no measured flops_used"), + "{body}" + ); + let (_, list) = json_req(app.clone(), "GET", "/v1/submissions", serde_json::json!({})).await; + assert_eq!(list["items"].as_array().unwrap().len(), 1, "{list}"); + assert_eq!( + paid_runs(&orchestrator), + 2, + "both runs were paid; neither passed" + ); + let lc = rlm_store.lifecycle(&tid).await.unwrap().unwrap(); + assert_eq!( + lc.state, + RlmState::Open, + "a refusal closes the verdict phase" + ); + + // Back within budget the same topic scores again. + orchestrator.set_flops_used(Some(budget)); + let (st, created) = json_req( + app, + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-at-budget"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "champion", "{created}"); + let _ = std::fs::remove_dir_all(&root); +} + /// The agentic setup: owner hook, key probe, VM provision, RLM-written /// rules and baseline persisted, then the operator seals and opens. #[tokio::test] @@ -511,17 +621,45 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { ); assert_eq!(orchestrator.created(), 0); - // Key present: provision, RLM rules v1 in store, baseline in store. + // Key present but the baseline run overspends: refused, nothing sealed. std::fs::write(&key, "not-a-real-secret\n").unwrap(); + orchestrator.set_flops_used(Some(draft.flops_budget + 1)); + let err = setup + .run(&draft, &pin(), &offer()) + .await + .expect_err("over budget"); + assert!( + matches!(err, SetupError::BaselineOverBudget { used, budget } if used == budget + 1), + "{err}" + ); + assert!(rlm_store.baseline(&draft.id).await.unwrap().is_none()); + orchestrator.set_flops_used(None); + let err = setup + .run(&draft, &pin(), &offer()) + .await + .expect_err("unmeasured"); + assert!(matches!(err, SetupError::Report(_)), "{err}"); + assert!(rlm_store.baseline(&draft.id).await.unwrap().is_none()); + + // Measured within budget: the attached VM, RLM rules in store (each pass + // through setup had the RLM write a version: v1, v2 for the two refused + // baselines, v3 now), baseline in store under the current version. + orchestrator.set_flops_used(Some(1)); let out = setup.run(&draft, &pin(), &offer()).await.expect("setup"); - assert_eq!(out.rules_version, 1); + assert_eq!(out.rules_version, 3); assert!((out.baseline_primary - 0.42).abs() < 1e-12); - assert_eq!(orchestrator.created(), 1); + assert_eq!( + orchestrator.created(), + 1, + "the vm is attached, not re-created" + ); let rules = rlm_store.current_rules(&draft.id).await.unwrap().unwrap(); + assert_eq!(rules.version, 3); assert_eq!(rules.source, proof_rlm::RuleSource::Rlm); assert_eq!(rules.rules[0].id, "rlm_rule"); let baseline = rlm_store.baseline(&draft.id).await.unwrap().unwrap(); - assert_eq!(baseline.rules_version, 1); + assert_eq!(baseline.rules_version, 3); + assert_eq!(baseline.report.flops_used, Some(1)); let lc = rlm_store.lifecycle(&draft.id).await.unwrap().unwrap(); assert_eq!(lc.state, RlmState::Baselining); assert!(lc.history.iter().any(|h| h.event == RlmEvent::Provisioned)); diff --git a/crates/proof-rlm/src/fixtures_tests.rs b/crates/proof-rlm/src/fixtures_tests.rs index 581541e44..0f45addca 100644 --- a/crates/proof-rlm/src/fixtures_tests.rs +++ b/crates/proof-rlm/src/fixtures_tests.rs @@ -147,6 +147,7 @@ pub fn report_for(req: &CustomRunRequest, primary_value: f64) -> CustomRunReport primary_value, claim_holds: true, sandboxed: true, + flops_used: Some(1), evidence, } } @@ -181,6 +182,7 @@ pub struct FakeOrchestrator { primary: Mutex, red: Mutex>, sandboxed: AtomicBool, + flops_used: Mutex>, created: AtomicUsize, vms: Mutex>, jobs: Mutex>, @@ -193,6 +195,7 @@ impl FakeOrchestrator { primary: Mutex::new(primary), red: Mutex::new(None), sandboxed: AtomicBool::new(true), + flops_used: Mutex::new(Some(1)), created: AtomicUsize::new(0), vms: Mutex::new(Vec::new()), jobs: Mutex::new(Vec::new()), @@ -207,6 +210,12 @@ impl FakeOrchestrator { *self.primary.lock().unwrap() = v; } + /// What every report measures as `flops_used` (`None` = the runner + /// forgot to measure, which the host must refuse). + pub fn set_flops_used(&self, v: Option) { + *self.flops_used.lock().unwrap() = v; + } + /// Make inspections fail this rule id (None = green). pub fn set_red(&self, id: Option<&str>) { *self.red.lock().unwrap() = id.map(str::to_owned); @@ -235,6 +244,7 @@ impl FakeOrchestrator { fn report(&self, req: &CustomRunRequest) -> CustomRunReport { let mut r = report_for(req, *self.primary.lock().unwrap()); r.sandboxed = self.sandboxed.load(Ordering::SeqCst); + r.flops_used = *self.flops_used.lock().unwrap(); r } } diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs index 21206828f..cd9dd4482 100644 --- a/crates/proof-rlm/src/runner.rs +++ b/crates/proof-rlm/src/runner.rs @@ -76,6 +76,9 @@ pub struct CustomRunRequest { pub artifact_uri: Option, /// Miner claim (English). pub claim: String, + /// FLOPs one run may spend (the topic's signed `flops_budget`). The report + /// must carry its measured usage against this figure. + pub flops_budget: u64, /// Topic constraints (sandbox flag, model pin, opaque slice / params). pub constraints: Constraints, /// Rule version the checklist must tick. @@ -153,6 +156,7 @@ impl CustomRunRequest { .filter(|u| !u.is_empty()) .map(str::to_owned), claim: claim.to_owned(), + flops_budget: topic.flops_budget, constraints: topic.constraints.clone(), rules_version: rules.version, rules_digest: rules.digest(), @@ -211,6 +215,12 @@ pub struct CustomRunReport { pub claim_holds: bool, /// Whether miner code ran inside the Firecracker guest. pub sandboxed: bool, + /// FLOPs the run consumed, as measured by the runner. This is the only + /// usage figure a verdict may carry — the miner's declaration never is. + /// Absent against a topic budget the report is not evidence + /// ([`ReportError::FlopsMissing`]); zero is accepted only as a measurement. + #[serde(default)] + pub flops_used: Option, /// Opaque runner evidence (per-task rows, timings). Shipped in the artefact. pub evidence: BTreeMap, } @@ -236,6 +246,12 @@ pub enum ReportError { /// The topic requires Firecracker and the run was not sandboxed. #[error("run report says miner code ran outside the Firecracker guest")] NotSandboxed, + /// The topic has a FLOP budget and the runner measured no usage. + #[error("run report carries no measured flops_used against a budget of {budget}")] + FlopsMissing { + /// The topic's budget the usage had to be measured against. + budget: u64, + }, } impl CustomRunReport { @@ -289,8 +305,26 @@ impl CustomRunReport { if req.sandbox.firecracker_required && !self.sandboxed { return Err(ReportError::NotSandboxed); } + self.flops_used_for(req)?; Ok(()) } + + /// The runner-measured FLOPs this run consumed: the authoritative usage + /// the verdict carries and the judge compares with the topic budget. + /// + /// # Errors + /// + /// [`ReportError::FlopsMissing`] when the topic has a budget and the + /// report carries no measurement. A missing figure is never read as zero. + pub fn flops_used_for(&self, req: &CustomRunRequest) -> Result { + match self.flops_used { + Some(used) => Ok(used), + None if req.flops_budget == 0 => Ok(0), + None => Err(ReportError::FlopsMissing { + budget: req.flops_budget, + }), + } + } } /// One file of the miner's artefact tree as inspected. @@ -433,6 +467,10 @@ mod tests { assert_eq!(req.rules_version, 1); assert_eq!(req.rules_digest, rules().digest()); assert_eq!(req.seed, t.baseline.seed); + assert_eq!( + req.flops_budget, t.flops_budget, + "budget travels to the runner" + ); assert_eq!( req.artifact_uri.as_deref(), Some("https://example.invalid/artifact.zip"), @@ -483,6 +521,43 @@ mod tests { assert_eq!(blank.artifact_uri, None, "whitespace is no locator"); } + /// The verdict's usage figure comes from the runner's measurement. A + /// report that carries none against a budget is not evidence, so a runner + /// cannot leave the budget unenforced by omitting the field. + #[test] + fn a_report_must_measure_flops_against_a_budget() { + let req = request(); + assert!(req.flops_budget > 0, "fixture topic carries a budget"); + let measured = report_for(&req, 0.7); + assert_eq!(measured.flops_used_for(&req), Ok(1)); + let mut none = report_for(&req, 0.7); + none.flops_used = None; + assert_eq!( + none.flops_used_for(&req), + Err(ReportError::FlopsMissing { + budget: req.flops_budget + }) + ); + assert_eq!( + none.verify(&req), + Err(ReportError::FlopsMissing { + budget: req.flops_budget + }), + "binding fails closed without a measurement" + ); + let mut over = report_for(&req, 0.7); + over.flops_used = Some(req.flops_budget + 1); + over.verify(&req) + .expect("over budget is a measurement, judged downstream"); + assert_eq!(over.flops_used_for(&req), Ok(req.flops_budget + 1)); + let mut unbudgeted = req.clone(); + unbudgeted.flops_budget = 0; + assert_eq!(none.flops_used_for(&unbudgeted), Ok(0)); + let legacy: CustomRunReport = + serde_json::from_str(&none.to_json()).expect("a report without the field parses"); + assert_eq!(legacy.flops_used, None); + } + #[test] fn a_report_must_echo_the_request_and_honour_the_sandbox() { let req = request(); From 58e1ade1c6d6d1833e49d90eec94d1104abd6405 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:34:01 +0000 Subject: [PATCH 10/14] fix(proof-rlm-scorer): hold the topic lease through promote and persist Greptile P1 on #243: the per-topic guard ended when `score` returned, so two completed runs could both promote against the same old bar and the persist path appended both promotions and moved the best pointer unconditionally; a later, worse run could replace a better champion. - A run now holds its topic lease from `score` until `on_persisted` (the guard moves into the pending entry). The next run of the topic cannot score, decide, or persist until the previous row landed. - `auto_promote` decides under that lease against the harder of the caller's bar and the store's current best, and records the best it was decided against. - `persist` runs a compare-and-swap before writing: the best pointer must still be the one the decision saw and the run must be strictly better than it, else the crown is refused (`promotion_refused`, manifest `promoted: false`, no promotion row, pointer untouched). The promotion row is written to the RLM store whether or not a filesystem artefact store is configured. - A scored run whose row never lands releases its lease after `DEFAULT_LEASE_TTL` (5 min, `with_lease_ttl` for tests) so a failed insert cannot block a topic; the lifecycle is recovered on next score. - e2e: a second run blocks until the first persists and then loses against the new best with the stale bar; an externally moved best refuses a decided crown; an abandoned lease is reaped. --- crates/proof-rlm-scorer/Cargo.toml | 4 +- crates/proof-rlm-scorer/src/lib.rs | 11 +- crates/proof-rlm-scorer/src/scorer.rs | 397 +++++++++++++++++------ crates/proof-rlm-scorer/tests/rlm_e2e.rs | 242 +++++++++++++- 4 files changed, 540 insertions(+), 114 deletions(-) diff --git a/crates/proof-rlm-scorer/Cargo.toml b/crates/proof-rlm-scorer/Cargo.toml index a4948224b..081b312e6 100644 --- a/crates/proof-rlm-scorer/Cargo.toml +++ b/crates/proof-rlm-scorer/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" thiserror = "2" -tokio = { version = "1", features = ["sync", "rt"] } +tokio = { version = "1", features = ["sync", "rt", "time"] } tracing = "0.1" zip = { version = "2", default-features = false, features = ["deflate"] } @@ -34,7 +34,7 @@ proof-http = { path = "../proof-http" } proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } proof-store = { path = "../proof-store" } sha2 = "0.10" -tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "time"] } tower = { version = "0.5", features = ["util"] } [lints] diff --git a/crates/proof-rlm-scorer/src/lib.rs b/crates/proof-rlm-scorer/src/lib.rs index 76868b4eb..aec057ebc 100644 --- a/crates/proof-rlm-scorer/src/lib.rs +++ b/crates/proof-rlm-scorer/src/lib.rs @@ -4,9 +4,12 @@ //! resolve the topic's `custom_id` in the runner registry (unknown → //! `RunnerUnwired`, 503, no row), load the topic's current rule version //! from the store, inspect → checklist (persisted), red → reject without -//! paid inference, green → spend token → evaluate → `custom_value`. It is -//! wired through `proof_eval::FamilyMux` so no custom topic ever falls -//! back to the digest-pinned harvest. +//! paid inference, green → spend token → evaluate → `custom_value` with +//! the runner-measured `flops_used` in the verdict. It is wired through +//! `proof_eval::FamilyMux` so no custom topic ever falls back to the +//! digest-pinned harvest. Runs hold a per-topic lease from `score` until +//! the row is persisted, so promotion is decided and written against the +//! store's current best and a worse run can never displace a champion. //! - [`ArtefactStore`] writes `{root}/{topic_id}/{submission_id}.zip` //! (`manifest.json`, `artifact/`, `report.json`, `checklist.json`, //! `baseline_ref.json`, `logs/`) once the row is persisted, `best.json` @@ -37,5 +40,5 @@ pub use artefact::{ ARTEFACT_ROOT_ENV, ARTIFACT_DIR, BASELINE_REF_FILE, BEST_FILE, CHECKLIST_FILE, DEFAULT_ARTEFACT_ROOT, EVENTS_FILE, LOGS_DIR, MANIFEST_FILE, MAX_ENTRY_PATH, REPORT_FILE, }; -pub use scorer::RlmScorer; +pub use scorer::{RlmScorer, DEFAULT_LEASE_TTL}; pub use setup::{SetupError, SetupOutcome, TopicSetup}; diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index def17b08f..f7eecdf7c 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -16,11 +16,19 @@ //! 6. on persist: artefact zip + metadata row + public event; on promotion: //! promotion row, `best.json`, lifecycle `promoting → open`. //! -//! Evaluations are serialised per topic so the lifecycle mirror is exact -//! and every transition is written to the store. +//! Runs are serialised per topic by a **lease** the run holds from `score` +//! until the host reports the row persisted (`on_persisted`). Promotion is +//! decided under that lease against the store's current best and persisted +//! under the same lease with a compare-and-swap on the best pointer, so two +//! runs can never both promote against one stale bar and a later, worse run +//! can never displace a better champion. A run whose row never lands +//! releases its lease after [`DEFAULT_LEASE_TTL`]. +//! +//! [`SpendToken`]: proof_rlm::SpendToken use std::collections::BTreeMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; use async_trait::async_trait; use proof_eval::{EvalError, LiveScorer, ProofEvalDocument, PROOF_METRICS_SCHEMA}; @@ -31,14 +39,38 @@ use proof_rlm::{ }; use proof_rlm_store::{ArtefactRow, ChecklistRow, PromotionRow, RlmStore, TransitionRow}; use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind}; -use proof_task::{HoldoutRecord, InferenceOffer, MetricFamily, ProofPin, TopicDocument}; +use proof_task::{ + HoldoutRecord, InferenceOffer, MetricDirection, MetricFamily, ProofPin, TopicDocument, +}; +use tokio::sync::OwnedMutexGuard; use crate::artefact::{ArtefactBundle, ArtefactStore, BaselineRef, BestRef, PublicEvent}; +/// How long a scored run may hold its topic lease waiting for the host to +/// report its row persisted. Past this the persist is treated as abandoned +/// (the row never landed) and the next run of the topic proceeds. +pub const DEFAULT_LEASE_TTL: Duration = Duration::from_mins(5); + +/// How often a run waiting for a topic lease re-checks for abandoned holders. +const LEASE_POLL: Duration = Duration::from_secs(1); + +/// A promotion decision and the world it was taken in. +struct Decided { + outcome: PromoteDecision, + /// The store's best when the decision was taken; persist refuses when it + /// has moved (another writer crowned something in between). + previous_best: Option, + direction: MetricDirection, +} + /// Scored-but-not-yet-persisted state for one submission. struct Pending { bundle: ArtefactBundle, - decision: Option, + decided: Option, + /// Topic lease held since `score` returned; dropped when the row is + /// persisted (end of `on_persisted`) or the entry is reaped. + lease: Option>, + since: Instant, } /// Family scorer over the runner registry, the RLM store, and the artefact store. @@ -48,6 +80,7 @@ pub struct RlmScorer { artefacts: Option, pending: Mutex>, locks: Mutex>>>, + lease_ttl: Duration, } fn unwired(custom_id: &str, detail: String) -> EvalError { @@ -70,6 +103,26 @@ fn store_err(e: E) -> EvalError { EvalError::Backend(format!("rlm store: {e}")) } +/// The harder of two bars, direction-aware (`None` when neither exists). +fn tighter_bar(a: Option, b: Option, direction: MetricDirection) -> Option { + match (a.filter(|v| v.is_finite()), b.filter(|v| v.is_finite())) { + (None, None) => None, + (Some(v), None) | (None, Some(v)) => Some(v), + (Some(x), Some(y)) => Some(match direction { + MetricDirection::Max => x.max(y), + MetricDirection::Min => x.min(y), + }), + } +} + +/// Strictly better, direction-aware. +fn strictly_better(candidate: f64, incumbent: f64, direction: MetricDirection) -> bool { + match direction { + MetricDirection::Max => candidate > incumbent, + MetricDirection::Min => candidate < incumbent, + } +} + impl RlmScorer { /// Scorer over `registry` and `store`, no artefact store. #[must_use] @@ -80,6 +133,7 @@ impl RlmScorer { artefacts: None, pending: Mutex::new(BTreeMap::new()), locks: Mutex::new(BTreeMap::new()), + lease_ttl: DEFAULT_LEASE_TTL, } } @@ -90,6 +144,14 @@ impl RlmScorer { self } + /// How long a scored run may wait for its row before its topic lease is + /// released (default [`DEFAULT_LEASE_TTL`]). + #[must_use] + pub fn with_lease_ttl(mut self, ttl: Duration) -> Self { + self.lease_ttl = ttl; + self + } + /// Bundles scored but not yet persisted. #[must_use] pub fn pending_len(&self) -> usize { @@ -99,12 +161,58 @@ impl RlmScorer { fn topic_lock(&self, topic_id: &str) -> Arc> { self.locks .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) + .unwrap_or_else(PoisonError::into_inner) .entry(topic_id.to_owned()) .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) .clone() } + /// Drop pending runs of `topic_id` whose row never landed within the TTL, + /// releasing the lease they hold. + fn reap_abandoned(&self, topic_id: &str) { + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + let stale: Vec = pending + .iter() + .filter(|(_, p)| p.bundle.topic_id == topic_id && p.since.elapsed() >= self.lease_ttl) + .map(|(digest, _)| digest.clone()) + .collect(); + for digest in stale { + pending.remove(&digest); + tracing::warn!( + topic_id, + submission_digest = %digest, + "scored run never persisted within the lease ttl; lease released" + ); + } + } + + /// Take the topic lease, reaping abandoned holders while waiting. + async fn lease(&self, topic_id: &str) -> OwnedMutexGuard<()> { + let lock = self.topic_lock(topic_id); + loop { + self.reap_abandoned(topic_id); + if let Ok(guard) = tokio::time::timeout(LEASE_POLL, lock.clone().lock_owned()).await { + return guard; + } + } + } + + /// Hand the lease to the pending run so it outlives `score`. + fn hold(&self, submission_digest: &str, lease: OwnedMutexGuard<()>) { + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + match pending.get_mut(submission_digest.trim()) { + Some(p) => p.lease = Some(lease), + None => drop(lease), + } + } + + fn take(&self, submission_digest: &str) -> Option { + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(submission_digest) + } + /// Make sure the signed document this run is scored under is in the store /// (a new signature is a new topic version). async fn ensure_topic(&self, topic: &TopicDocument) -> Result<(), EvalError> { @@ -165,7 +273,7 @@ impl RlmScorer { } } - /// Called with the topic lock held: a persisted `evaluating` / + /// Called with the topic lease held: a persisted `evaluating` / /// `promoting` means the previous run's row never landed. Close that /// phase rather than refusing the topic forever. async fn recover_stale(&self, topic: &TopicDocument) -> Result<(), EvalError> { @@ -283,12 +391,14 @@ impl RlmScorer { }; self.pending .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) + .unwrap_or_else(PoisonError::into_inner) .insert( req.submission_digest.clone(), Pending { bundle, - decision: None, + decided: None, + lease: None, + since: Instant::now(), }, ); } @@ -418,104 +528,144 @@ impl RlmScorer { Ok(doc) } - async fn persist( + /// Whether a decided promotion still stands at persist time: the best + /// pointer must be the one the decision was taken against (compare-and- + /// swap) and this run must be strictly better than it. Returns the + /// promotion row to append, or `None` when the crown is refused. + async fn crown( &self, topic_id: &str, submission_digest: &str, submission_id: &str, - promoted: bool, - ) { - let Some(pending) = self - .pending - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(submission_digest) - else { - return; + pending: &Pending, + ) -> Option { + let primary = pending.bundle.report.as_ref().map(|r| r.primary_value); + let (Some(primary_value), Some(decided)) = (primary, pending.decided.as_ref()) else { + tracing::error!( + topic_id, + submission_id, + "promoted without a primary or a decision" + ); + return None; }; - let bundle = pending.bundle; - let primary = bundle.report.as_ref().map(|r| r.primary_value); - let Some(store) = &self.artefacts else { - return; + let PromoteDecision::Promote { bar, .. } = decided.outcome else { + tracing::error!(topic_id, submission_id, "promoted against a keep decision"); + return None; }; - match store.write(&bundle, submission_id, promoted) { - Ok(written) => { - let row = ArtefactRow { - topic_id: topic_id.to_owned(), - submission_id: submission_id.to_owned(), - submission_digest: submission_digest.to_owned(), - path: written.path.display().to_string(), - sha256: written.sha256, - bytes: written.bytes, - primary_value: primary, - checklist_green: bundle.checklist_green, - promoted, - }; - if let Err(e) = self.store.put_artefact(&row).await { - tracing::error!(topic_id, submission_id, %e, "artefact metadata not persisted"); - } - let _ = store.append_event( - topic_id, - &PublicEvent::Scored { - submission_id: submission_id.to_owned(), - primary_value: primary, - checklist_green: bundle.checklist_green, - }, - ); - tracing::info!(topic_id, submission_id, path = %written.path.display(), "artefact written"); + let current = match self.store.best(topic_id).await { + Ok(b) => b, + Err(e) => { + tracing::error!(topic_id, submission_id, %e, "best unreadable; promotion refused"); + return None; } - Err(e) => tracing::error!(topic_id, submission_id, %e, "artefact not written"), - } - if !promoted { - return; - } - let (Some(primary_value), Some(PromoteDecision::Promote { bar, .. })) = - (primary, pending.decision) - else { + }; + let expected = decided + .previous_best + .as_ref() + .map(|b| b.submission_id.as_str()); + let unchanged = current.as_ref().map(|b| b.submission_id.as_str()) == expected; + let better = current + .as_ref() + .is_none_or(|b| strictly_better(primary_value, b.primary_value, decided.direction)); + if !unchanged || !better { tracing::error!( topic_id, submission_id, - "promoted without a primary or a decision" + primary_value, + best = ?current.as_ref().map(|b| (&b.submission_id, b.primary_value)), + "stale promotion refused: best moved since the decision" ); - return; - }; - let previous_best = self - .store - .best(topic_id) - .await - .ok() - .flatten() - .map(|b| b.submission_id); - let row = PromotionRow { + return None; + } + Some(PromotionRow { topic_id: topic_id.to_owned(), submission_id: submission_id.to_owned(), submission_digest: submission_digest.to_owned(), primary_value, bar: Some(bar), - previous_best: previous_best.clone(), + previous_best: current.map(|b| b.submission_id), + }) + } + + /// Write the artefact and, when the crown stands ([`Self::crown`]), the + /// promotion row, best pointer, and public event. Returns whether the + /// promotion landed; the manifest records that, not the caller's flag. + async fn persist( + &self, + topic_id: &str, + submission_digest: &str, + submission_id: &str, + promoted: bool, + pending: &Pending, + ) -> bool { + let crown = if promoted { + self.crown(topic_id, submission_digest, submission_id, pending) + .await + } else { + None + }; + let landed = crown.is_some(); + let bundle = &pending.bundle; + let primary = bundle.report.as_ref().map(|r| r.primary_value); + if let Some(store) = &self.artefacts { + match store.write(bundle, submission_id, landed) { + Ok(written) => { + let row = ArtefactRow { + topic_id: topic_id.to_owned(), + submission_id: submission_id.to_owned(), + submission_digest: submission_digest.to_owned(), + path: written.path.display().to_string(), + sha256: written.sha256, + bytes: written.bytes, + primary_value: primary, + checklist_green: bundle.checklist_green, + promoted: landed, + }; + if let Err(e) = self.store.put_artefact(&row).await { + tracing::error!(topic_id, submission_id, %e, "artefact metadata not persisted"); + } + let _ = store.append_event( + topic_id, + &PublicEvent::Scored { + submission_id: submission_id.to_owned(), + primary_value: primary, + checklist_green: bundle.checklist_green, + }, + ); + tracing::info!(topic_id, submission_id, path = %written.path.display(), "artefact written"); + } + Err(e) => tracing::error!(topic_id, submission_id, %e, "artefact not written"), + } + } + let Some(row) = crown else { + return false; }; if let Err(e) = self.store.record_promotion(&row).await { tracing::error!(topic_id, submission_id, %e, "promotion not persisted"); + return false; } - if let Err(e) = store.mark_best(&BestRef { - topic_id: topic_id.to_owned(), - submission_id: submission_id.to_owned(), - submission_digest: submission_digest.to_owned(), - primary_value, - bar: Some(bar), - artefact: format!("{submission_id}.zip"), - }) { - tracing::error!(topic_id, submission_id, %e, "best pointer not written"); - } - let _ = store.append_event( - topic_id, - &PublicEvent::Promoted { + if let Some(store) = &self.artefacts { + if let Err(e) = store.mark_best(&BestRef { + topic_id: topic_id.to_owned(), submission_id: submission_id.to_owned(), - primary_value, - bar: Some(bar), - previous_best, - }, - ); + submission_digest: submission_digest.to_owned(), + primary_value: row.primary_value, + bar: row.bar, + artefact: format!("{submission_id}.zip"), + }) { + tracing::error!(topic_id, submission_id, %e, "best pointer not written"); + } + let _ = store.append_event( + topic_id, + &PublicEvent::Promoted { + submission_id: submission_id.to_owned(), + primary_value: row.primary_value, + bar: row.bar, + previous_best: row.previous_best, + }, + ); + } + true } } @@ -534,8 +684,7 @@ impl LiveScorer for RlmScorer { claim: &str, ) -> Result { self.ready_for_topic(topic)?; - let lock = self.topic_lock(&topic.id); - let _serial = lock.lock().await; + let lease = self.lease(&topic.id).await; self.ensure_topic(topic).await?; self.recover_stale(topic).await?; self.apply(topic, RlmEvent::SubmissionReceived, frozen_digest) @@ -552,10 +701,15 @@ impl LiveScorer for RlmScorer { claim, ) .await; - if out.is_err() { + if out.is_ok() { + // The lease now belongs to the pending run: promotion is decided + // and persisted under it, then it is released in `on_persisted`. + self.hold(frozen_digest, lease); + } else { // No row will follow a refusal, so the verdict phase is over now. self.apply_logged(topic, RlmEvent::VerdictRecorded, "refused; no row") .await; + drop(lease); } out } @@ -583,6 +737,10 @@ impl LiveScorer for RlmScorer { self.registry.ids() } + /// Decided under the topic lease this run has held since `score` + /// returned, against the harder of the caller's bar and the store's + /// current best: no other run of this topic can be between score and + /// persist, and a bar computed before an earlier crown cannot be reused. async fn auto_promote( &self, topic: &TopicDocument, @@ -591,17 +749,34 @@ impl LiveScorer for RlmScorer { primary: Option, bar: Option, ) -> bool { - let mut pending = self + let held = self .pending .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner) + .contains_key(submission_digest); + if !held { + return false; + } + let current = match self.store.best(&topic.id).await { + Ok(b) => b, + Err(e) => { + tracing::warn!(topic_id = %topic.id, %e, "best unreadable; no promotion"); + return false; + } + }; + let bar = tighter_bar( + bar, + current.as_ref().map(|b| b.primary_value), + topic.metric.direction, + ); + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); let Some(p) = pending.get_mut(submission_digest) else { return false; }; let reported = p.bundle.report.as_ref().map(|r| r.primary_value); // The payout primary and the report must agree, or neither is evidence. let agree = matches!((primary, reported), (Some(a), Some(b)) if (a - b).abs() < 1e-9); - let decision = decide_promote( + let outcome = decide_promote( pass && agree, p.bundle.checklist_green, &p.bundle.checklist.failed_ids(), @@ -610,8 +785,12 @@ impl LiveScorer for RlmScorer { topic.metric.direction, topic.metric.epsilon_rel, ); - let promote = decision.is_promote(); - p.decision = Some(decision); + let promote = outcome.is_promote(); + p.decided = Some(Decided { + outcome, + previous_best: current, + direction: topic.metric.direction, + }); promote } @@ -622,14 +801,12 @@ impl LiveScorer for RlmScorer { submission_id: &str, promoted: bool, ) { - let owned = self - .pending - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .contains_key(submission_digest); - if !owned { + // `pending` (and the topic lease inside it) lives to the end of this + // function: the artefact, the promotion row, and the best pointer + // land under the guard the decision was taken under. + let Some(pending) = self.take(submission_digest) else { return; - } + }; let topic = self .store .latest_topic(topic_id) @@ -643,15 +820,23 @@ impl LiveScorer for RlmScorer { .await; } } - self.persist(topic_id, submission_digest, submission_id, promoted) + let landed = self + .persist( + topic_id, + submission_digest, + submission_id, + promoted, + &pending, + ) .await; if let Some(t) = &topic { - let event = if promoted { - RlmEvent::Promoted - } else { - RlmEvent::VerdictRecorded + let event = match (promoted, landed) { + (true, true) => RlmEvent::Promoted, + (true, false) => RlmEvent::PromotionRefused, + (false, _) => RlmEvent::VerdictRecorded, }; self.apply_logged(t, event, submission_id).await; } + drop(pending); } } diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 9d68f1b97..60262f558 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -12,7 +12,10 @@ //! over-budget measurement is a persisted reject and a missing one is a 503; //! every scored row leaves its zip, the crown leaves //! `best.json` + a promotion row, and the store holds rules v1, every -//! checklist, and the lifecycle. Then the setup driver walks +//! checklist, and the lifecycle. Runs hold their topic lease until +//! persisted, so a worse run decided against a stale bar can never displace +//! the champion, a moved best pointer refuses a stale crown, and an +//! abandoned run releases its lease after the TTL. Then the setup driver walks //! `draft → … → baselining` with RLM-written rules and a baseline in the //! store, and `mark_sealed` opens the topic. Every id here is a placeholder. @@ -25,6 +28,7 @@ )] use std::sync::Arc; +use std::time::Duration; use axum::body::Body; use axum::http::{Request, StatusCode}; @@ -39,7 +43,7 @@ use proof_rlm::{ VmBackedRunner, VmJob, }; use proof_rlm_scorer::{ArtefactStore, RlmScorer, SetupError, TopicSetup}; -use proof_rlm_store::{MemoryRlmStore, RlmStore}; +use proof_rlm_store::{MemoryRlmStore, PromotionRow, RlmStore}; use proof_score::SealedBaseline; use proof_store::MemoryStore; use proof_task::{ @@ -217,6 +221,15 @@ fn zip_names(path: &std::path::Path) -> Vec { names } +fn manifest_of(path: &std::path::Path) -> serde_json::Value { + let bytes = std::fs::read(path).unwrap(); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut file = archive.by_name("manifest.json").unwrap(); + let mut body = String::new(); + std::io::Read::read_to_string(&mut file, &mut body).unwrap(); + serde_json::from_str(&body).unwrap() +} + fn paid_runs(orchestrator: &FakeOrchestrator) -> usize { orchestrator .jobs() @@ -365,6 +378,7 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { "report.json", ] ); + assert_eq!(manifest_of(&zip_a)["promoted"], true); let best = ArtefactStore::new(&root).best(&tid).expect("best.json"); assert_eq!(best.submission_id, id_a); let promo = rlm_store.best(&tid).await.unwrap().expect("promotion row"); @@ -573,6 +587,230 @@ async fn an_over_budget_or_unmeasured_run_never_passes() { let _ = std::fs::remove_dir_all(&root); } +struct Direct { + scorer: Arc, + orchestrator: Arc, + rlm_store: Arc, + root: std::path::PathBuf, + topic: TopicDocument, + pin: ProofPin, + plan: ExecutorPlan, +} + +/// The scorer alone (no router), for lease and persist ordering tests. +fn direct(tag: &str, lease_ttl: Option) -> Direct { + let pin = pin(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let mut t = topic(); + t.holdout_commitment = holdout_commitment(&recs); + let orchestrator = FakeOrchestrator::new(0.7); + let registry = RunnerRegistry::new().with( + &t.metric.custom_id, + Arc::new(VmBackedRunner::new(orchestrator.clone(), pinned_template())), + ); + let rlm_store = Arc::new(MemoryRlmStore::new()); + let root = tmp_root(tag); + let mut scorer = RlmScorer::new(Arc::new(registry), rlm_store.clone()) + .with_artefacts(Some(ArtefactStore::new(&root))); + if let Some(ttl) = lease_ttl { + scorer = scorer.with_lease_ttl(ttl); + } + let plan = scorer.plan(&pin, &t, &test_executor(&pin)).expect("plan"); + Direct { + scorer: Arc::new(scorer), + orchestrator, + rlm_store, + root, + topic: t, + pin, + plan, + } +} + +async fn score(d: &Direct, label: &str) -> Result { + d.scorer + .score( + &d.pin, + &d.topic, + &offer(), + &d.plan, + &format!("digest-{label}"), + &digest(label), + None, + &[], + "placeholder claim", + ) + .await +} + +/// Two runs decided against the same old bar: the second cannot score until +/// the first is persisted, its decision then sees the new best, and a moved +/// best pointer refuses a crown that was decided before it moved. +#[tokio::test] +async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { + let d = direct("lease", None); + let tid = d.topic.id.clone(); + + // A scores 0.70 and now holds the topic lease until its row lands. + let doc_a = score(&d, "a").await.expect("a scores"); + assert!((doc_a.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); + assert_eq!(d.scorer.pending_len(), 1); + + // B (0.60) is blocked on the lease, not scored against the stale world. + d.orchestrator.set_primary(0.6); + let scorer_b = d.scorer.clone(); + let (pin_b, topic_b, plan_b) = (d.pin.clone(), d.topic.clone(), d.plan.clone()); + let mut task_b = tokio::spawn(async move { + scorer_b + .score( + &pin_b, + &topic_b, + &offer(), + &plan_b, + "digest-b", + &digest("b"), + None, + &[], + "placeholder claim", + ) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(300), &mut task_b) + .await + .is_err(), + "b must wait for a's row" + ); + assert_eq!(paid_runs(&d.orchestrator), 1, "b has not run"); + + // A is decided against bar 0.50 and persisted under the lease. + assert!( + d.scorer + .auto_promote(&d.topic, "digest-a", true, Some(0.7), Some(0.5)) + .await + ); + d.scorer + .on_persisted(&tid, "digest-a", "pf_0000000000000001", true) + .await; + assert_eq!(d.scorer.pending_len(), 0); + + // The lease is free: b scores, and its decision — even handed the stale + // bar 0.50 — is taken against the store's best 0.70. + let doc_b = task_b.await.unwrap().expect("b scores after a persisted"); + assert!((doc_b.harness.custom_value.unwrap() - 0.6).abs() < 1e-12); + assert!( + !d.scorer + .auto_promote(&d.topic, "digest-b", true, Some(0.6), Some(0.5)) + .await, + "0.60 does not beat the reigning 0.70" + ); + d.scorer + .on_persisted(&tid, "digest-b", "pf_0000000000000002", false) + .await; + let best = d.rlm_store.best(&tid).await.unwrap().expect("best"); + assert_eq!(best.submission_id, "pf_0000000000000001"); + assert!((best.primary_value - 0.7).abs() < 1e-12); + assert_eq!(d.rlm_store.promotions(&tid).await.unwrap().len(), 1); + assert_eq!( + ArtefactStore::new(&d.root) + .best(&tid) + .unwrap() + .submission_id, + "pf_0000000000000001" + ); + + // C (0.90) is decided to promote; another writer crowns 0.95 before C's + // row lands. The compare-and-swap on the best pointer refuses C. + d.orchestrator.set_primary(0.9); + score(&d, "c").await.expect("c scores"); + assert!( + d.scorer + .auto_promote(&d.topic, "digest-c", true, Some(0.9), Some(0.7)) + .await + ); + d.rlm_store + .record_promotion(&PromotionRow { + topic_id: tid.clone(), + submission_id: "pf_00000000000000ff".into(), + submission_digest: "digest-elsewhere".into(), + primary_value: 0.95, + bar: Some(0.7), + previous_best: Some("pf_0000000000000001".into()), + }) + .await + .unwrap(); + d.scorer + .on_persisted(&tid, "digest-c", "pf_0000000000000003", true) + .await; + let best = d.rlm_store.best(&tid).await.unwrap().unwrap(); + assert_eq!( + best.submission_id, "pf_00000000000000ff", + "stale crown refused" + ); + assert_eq!(d.rlm_store.promotions(&tid).await.unwrap().len(), 2); + assert_eq!( + ArtefactStore::new(&d.root) + .best(&tid) + .unwrap() + .submission_id, + "pf_0000000000000001", + "best pointer untouched by the refused crown" + ); + let zip_c = d.root.join(&tid).join("pf_0000000000000003.zip"); + assert_eq!(manifest_of(&zip_c)["promoted"], false); + let artefacts = d.rlm_store.artefacts(&tid).await.unwrap(); + assert!( + !artefacts + .iter() + .find(|a| a.submission_id == "pf_0000000000000003") + .unwrap() + .promoted + ); + let lc = d.rlm_store.lifecycle(&tid).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Open); + let events: Vec = lc.history.iter().map(|h| h.event).collect(); + assert_eq!( + &events[events.len() - 3..], + [ + RlmEvent::SubmissionReceived, + RlmEvent::PromotionCandidate, + RlmEvent::PromotionRefused, + ] + ); + let _ = std::fs::remove_dir_all(&d.root); +} + +/// A run whose row never lands must not hold its topic hostage: past the +/// lease TTL the next run reaps it, recovers the lifecycle, and proceeds. +#[tokio::test] +async fn an_abandoned_run_releases_its_topic_lease_after_the_ttl() { + let d = direct("ttl", Some(Duration::ZERO)); + let tid = d.topic.id.clone(); + score(&d, "abandoned").await.expect("scores"); + assert_eq!(d.scorer.pending_len(), 1); + let next = tokio::time::timeout(Duration::from_secs(10), score(&d, "next")) + .await + .expect("the abandoned lease is reaped, not waited on") + .expect("scores"); + assert!((next.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); + assert_eq!(d.scorer.pending_len(), 1, "only the live run is pending"); + assert!( + !d.scorer + .auto_promote(&d.topic, "digest-abandoned", true, Some(0.7), Some(0.5)) + .await, + "a reaped run cannot be promoted" + ); + let lc = d.rlm_store.lifecycle(&tid).await.unwrap().unwrap(); + assert!( + lc.history + .iter() + .any(|h| h.event == RlmEvent::VerdictRecorded && h.note.contains("recovered")), + "{lc:?}" + ); + assert_eq!(lc.state, RlmState::Evaluating, "the live run is still open"); + let _ = std::fs::remove_dir_all(&d.root); +} + /// The agentic setup: owner hook, key probe, VM provision, RLM-written /// rules and baseline persisted, then the operator seals and opens. #[tokio::test] From 8296cf799ccab8116f5464b85375b98a6a13609b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:35:38 +0000 Subject: [PATCH 11/14] fix(proof-rlm-scorer): mark_sealed opens only a signed valid sealed doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 on #243: `mark_sealed` advanced `baselining → open` and stored the supplied document as the new version without checking that it was open, sealed, structurally valid, or signed, so an invalid draft could become the version behind an open lifecycle. `TopicSetup::mark_sealed(topic, pin, registered_custom, sealed)` now refuses, in order and before anything is written or moved: a document that is not `status: open` (`SetupError::NotOpen`); one that does not validate as an open topic on this host — sealed baseline, registered `custom_id`, tighten-only floors (`SetupError::Topic`); one whose operator signature does not verify under the pin's topic key; a sealed `BaselineMeasurement` that does not bind to the document (commitment, holdout, image digest) or whose `custom_value` is not the primary the RLM measured in the topic VM (`SetupError::Seal`); and a lifecycle not at `baselining`. The version is stored, then the transition recorded. e2e: draft, unsigned, unregistered, mis-sealed, and mismatched documents all leave the lifecycle at `baselining` with version 1; the signed open document sealing the measured value opens as version 2; a second seal is illegal. Test-only `crypto` dev-dependency for the topic key. --- Cargo.lock | 1 + crates/proof-rlm-scorer/Cargo.toml | 1 + crates/proof-rlm-scorer/src/lib.rs | 4 +- crates/proof-rlm-scorer/src/setup.rs | 86 +++++++++-- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 174 +++++++++++++++++++---- 5 files changed, 225 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5254cb9e..a0bc953df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3571,6 +3571,7 @@ version = "0.1.0" dependencies = [ "async-trait", "axum", + "crypto", "hex", "http-body-util", "proof-canon", diff --git a/crates/proof-rlm-scorer/Cargo.toml b/crates/proof-rlm-scorer/Cargo.toml index 081b312e6..6bd102384 100644 --- a/crates/proof-rlm-scorer/Cargo.toml +++ b/crates/proof-rlm-scorer/Cargo.toml @@ -28,6 +28,7 @@ zip = { version = "2", default-features = false, features = ["deflate"] } [dev-dependencies] axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +crypto = { path = "../crypto" } hex = "0.4" http-body-util = "0.1" proof-http = { path = "../proof-http" } diff --git a/crates/proof-rlm-scorer/src/lib.rs b/crates/proof-rlm-scorer/src/lib.rs index aec057ebc..63c249518 100644 --- a/crates/proof-rlm-scorer/src/lib.rs +++ b/crates/proof-rlm-scorer/src/lib.rs @@ -17,7 +17,9 @@ //! the promotion continuum mirrored into the RLM store. //! - [`TopicSetup`] drives `draft → … → baselining` over the topic-VM //! boundary (owner hook, key probe, provision, RLM rule proposal → -//! store, baseline → store), and `mark_sealed` closes the loop to `open`. +//! store, baseline → store), and `mark_sealed` closes the loop to `open` +//! only for a signed, valid, open document whose sealed measurement is the +//! one the RLM produced. //! //! Core types live in `proof-rlm`; persistence in `proof-rlm-store`. diff --git a/crates/proof-rlm-scorer/src/setup.rs b/crates/proof-rlm-scorer/src/setup.rs index 132fa784e..b17301441 100644 --- a/crates/proof-rlm-scorer/src/setup.rs +++ b/crates/proof-rlm-scorer/src/setup.rs @@ -4,10 +4,10 @@ //! ```text //! draft ──submit_for_review──▶ owner_presend ──(owner hook)──▶ awaiting_owner_keys //! ──(key probe)──▶ provisioning ──(orchestrator.create)──▶ baselining -//! ──(ProposeRules job → rules vN in store; Baseline job → measurement in store, -//! refused when it spends over the topic budget or measures nothing)──▶ +//! ──(ProposeRules job → rules vN in store; Baseline job → measurement in store)──▶ //! returns SetupOutcome; the operator seals custom_value, re-signs `open`, -//! and calls `mark_sealed` (baselining → open). +//! and calls `mark_sealed` (baselining → open) with the signed open +//! document and the sealed measurement — both are checked before the move. //! ``` //! //! The control plane never runs RLM logic: it forwards jobs through @@ -18,6 +18,7 @@ use std::sync::Arc; +use proof_eval::BaselineMeasurement; use proof_rlm::{ await_owner_keys, owner_presend, CustomRunReport, CustomRunRequest, Lifecycle, OwnerHook, OwnerKeysProbe, OwnerPrompt, RlmEvent, RlmState, RuleSet, RuleSource, SandboxPolicy, @@ -25,7 +26,7 @@ use proof_rlm::{ VmTemplate, }; use proof_rlm_store::{BaselineRow, RlmStore, StoreError, TransitionRow}; -use proof_task::{InferenceOffer, MetricFamily, ProofPin, TopicDocument}; +use proof_task::{InferenceOffer, MetricFamily, ProofPin, TopicDocument, TopicError, TopicStatus}; /// Why setup stopped. #[derive(Debug, thiserror::Error)] @@ -59,6 +60,17 @@ pub enum SetupError { /// The owner declined at presend; the topic is back at draft. #[error("owner declined; topic {0:?} returned to draft")] Declined(String), + /// `mark_sealed` was handed a document that is not `status: open`. + #[error("topic {0:?} is not an open document; nothing to open")] + NotOpen(String), + /// The document does not validate as an open topic or does not verify + /// under the pin's topic key. + #[error("topic document: {0}")] + Topic(#[from] TopicError), + /// The sealed measurement does not bind to the document, or is not what + /// the RLM measured in the topic VM. + #[error("seal: {0}")] + Seal(String), } /// What setup produced for the operator to seal. @@ -315,22 +327,66 @@ impl TopicSetup { }) } - /// The operator sealed the baseline and re-signed `status: open`: + /// The operator sealed the RLM's baseline and re-signed `status: open`: /// `baselining → open`, recorded, and the new document version stored. /// + /// Nothing moves or is written unless, in this order: the document is + /// `status: open`; it validates as an open topic on this host + /// (`registered_custom` are the custom ids with a runner here — an open + /// custom topic needs one, a sealed baseline, tighten-only floors); its + /// operator signature verifies under the pin's topic key; `sealed` binds + /// to it (`BaselineMeasurement::verify`: commitment, holdout, image + /// digest) and its `custom_value` is the primary the RLM measured in the + /// topic VM; and the lifecycle is at `baselining`. + /// /// # Errors /// - /// [`SetupError::State`] when the topic is not at `baselining`. - pub async fn mark_sealed(&self, topic: &TopicDocument) -> Result { + /// [`SetupError::NotOpen`], [`SetupError::Topic`], [`SetupError::Seal`], + /// or [`SetupError::State`] when the topic is not at `baselining`. An + /// invalid draft never becomes the open version. + pub async fn mark_sealed( + &self, + topic: &TopicDocument, + pin: &ProofPin, + registered_custom: &[&str], + sealed: &BaselineMeasurement, + ) -> Result { + if topic.status != TopicStatus::Open { + return Err(SetupError::NotOpen(topic.id.clone())); + } + topic.validate(pin, registered_custom)?; + topic.verify_signature(pin)?; + sealed + .verify(pin, topic) + .map_err(|e| SetupError::Seal(e.to_string()))?; + let measured = + self.store.baseline(&topic.id).await?.ok_or_else(|| { + SetupError::Seal(format!("no baseline measured for {:?}", topic.id)) + })?; + let sealed_primary = sealed + .custom_value + .filter(|v| v.is_finite()) + .ok_or_else(|| SetupError::Seal("sealed measurement has no custom_value".into()))?; + if (sealed_primary - measured.primary_value).abs() > 1e-9 { + return Err(SetupError::Seal(format!( + "sealed custom_value {sealed_primary} is not the measured baseline {}", + measured.primary_value + ))); + } let mut lc = self.lifecycle(topic).await?; - let to = self - .step( - &mut lc, - RlmEvent::BaselineSealed, - "operator sealed the baseline", - ) - .await?; + if lc.state != RlmState::Baselining { + return Err(StateError::Illegal { + from: lc.state, + event: RlmEvent::BaselineSealed, + } + .into()); + } self.store.put_topic_version(topic).await?; - Ok(to) + self.step( + &mut lc, + RlmEvent::BaselineSealed, + "operator sealed the baseline", + ) + .await } } diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 60262f558..d3d36a239 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -10,14 +10,15 @@ //! FLOPs in the verdict; a red checklist is a persisted reject with **zero** //! paid runs; a later pass below the best stays `awaiting_admin`; an //! over-budget measurement is a persisted reject and a missing one is a 503; -//! every scored row leaves its zip, the crown leaves -//! `best.json` + a promotion row, and the store holds rules v1, every -//! checklist, and the lifecycle. Runs hold their topic lease until -//! persisted, so a worse run decided against a stale bar can never displace -//! the champion, a moved best pointer refuses a stale crown, and an -//! abandoned run releases its lease after the TTL. Then the setup driver walks -//! `draft → … → baselining` with RLM-written rules and a baseline in the -//! store, and `mark_sealed` opens the topic. Every id here is a placeholder. +//! every scored row leaves its zip, the crown leaves `best.json` + a +//! promotion row, and the store holds rules v1, every checklist, and the +//! lifecycle. Runs hold their topic lease until persisted, so a worse run +//! decided against a stale bar can never displace the champion, a moved +//! best pointer refuses a stale crown, and an abandoned run releases its +//! lease after the TTL. Then the setup driver walks `draft → … → +//! baselining` with RLM-written rules and a baseline in the store, and +//! `mark_sealed` opens the topic only for the signed, valid, open document +//! whose sealed value is the RLM's. Every id here is a placeholder. #![allow( clippy::unwrap_used, @@ -27,6 +28,7 @@ clippy::too_many_lines )] +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -34,7 +36,9 @@ use axum::body::Body; use axum::http::{Request, StatusCode}; use axum::Router; use http_body_util::BodyExt; -use proof_eval::{EvalBackend, EvalError, FamilyMux, LiveScorer, ProofEvalDocument}; +use proof_eval::{ + BaselineMeasurement, EvalBackend, EvalError, FamilyMux, LiveScorer, ProofEvalDocument, +}; use proof_executor::{EvalExecutorOffer, ExecutorPlan}; use proof_http::{executor_slot, hash_admin_token, proof_router, AppState}; use proof_rlm::fixtures::{offer, pin, pinned_template, topic, FakeOrchestrator}; @@ -47,8 +51,8 @@ use proof_rlm_store::{MemoryRlmStore, PromotionRow, RlmStore}; use proof_score::SealedBaseline; use proof_store::MemoryStore; use proof_task::{ - holdout_commitment, synthetic_holdout, HoldoutRecord, InferenceOffer, ProofPin, TopicDocument, - TopicStatus, STRATUM_SIZE, + holdout_commitment, synthetic_holdout, HoldoutRecord, HoldoutSplit, InferenceOffer, ProofPin, + TopicDocument, TopicError, TopicStatus, STRATUM_SIZE, }; use sha2::{Digest, Sha256}; use tower::ServiceExt; @@ -811,18 +815,62 @@ async fn an_abandoned_run_releases_its_topic_lease_after_the_ttl() { let _ = std::fs::remove_dir_all(&d.root); } +fn sk() -> [u8; 32] { + let mut s = [7u8; 32]; + s[0] = 42; + s +} + +fn pin_with_topic_key() -> ProofPin { + let mut p = pin(); + p.topic_pubkey = hex::encode(crypto::public_key_from_mini_secret(&sk()).unwrap()); + p +} + +/// A sealed measurement for a custom topic: every scored split present, the +/// custom value the operator seals. +fn sealed_measurement(pin: &ProofPin, topic: &TopicDocument, custom: f64) -> BaselineMeasurement { + let split_nll: BTreeMap = HoldoutSplit::SCORED + .iter() + .map(|s| (s.as_str().to_owned(), 0.0)) + .collect(); + BaselineMeasurement { + eval_image_digest: pin.eval_image_digest.clone(), + topic_id: topic.id.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + holdout_nll: 0.0, + split_nll, + tokens_per_sec: None, + step_latency_ms: None, + custom_value: Some(custom), + } +} + +/// The open document sealing `meas`, signed with the test topic key. +fn signed_open(draft: &TopicDocument, meas: &BaselineMeasurement) -> TopicDocument { + let mut open = draft.clone(); + open.status = TopicStatus::Open; + open.baseline.metrics_commitment = meas.commitment(); + open.signature = open.sign_with(&sk()).unwrap(); + open +} + /// The agentic setup: owner hook, key probe, VM provision, RLM-written -/// rules and baseline persisted, then the operator seals and opens. +/// rules and baseline persisted, then the operator seals and opens — and +/// only a signed, valid, open document sealing the RLM's value opens. #[tokio::test] async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { let root = tmp_root("setup"); let key = root.join("owner_key"); + let pin = pin_with_topic_key(); let orchestrator = FakeOrchestrator::new(0.42); let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); let mut draft = topic(); draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); draft.baseline.script_sha256 = "11".repeat(32); draft.baseline.metrics_commitment.clear(); + let registered = [draft.metric.custom_id.as_str()]; let mut setup = TopicSetup { orchestrator: orchestrator.clone(), store: rlm_store.clone(), @@ -836,7 +884,7 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { // A decline returns to draft; nothing is provisioned. let err = setup - .run(&draft, &pin(), &offer()) + .run(&draft, &pin, &offer()) .await .expect_err("declined"); assert!(matches!(err, SetupError::Declined(_)), "{err}"); @@ -848,10 +896,7 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { // Approved but no key file: stops at awaiting_owner_keys, nothing provisioned. setup.owner = Arc::new(StaticOwnerHook(OwnerDecision::Approve)); - let err = setup - .run(&draft, &pin(), &offer()) - .await - .expect_err("no key"); + let err = setup.run(&draft, &pin, &offer()).await.expect_err("no key"); assert!(err.to_string().contains("owner keys not present"), "{err}"); assert_eq!( rlm_store.lifecycle(&draft.id).await.unwrap().unwrap().state, @@ -863,7 +908,7 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { std::fs::write(&key, "not-a-real-secret\n").unwrap(); orchestrator.set_flops_used(Some(draft.flops_budget + 1)); let err = setup - .run(&draft, &pin(), &offer()) + .run(&draft, &pin, &offer()) .await .expect_err("over budget"); assert!( @@ -873,7 +918,7 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { assert!(rlm_store.baseline(&draft.id).await.unwrap().is_none()); orchestrator.set_flops_used(None); let err = setup - .run(&draft, &pin(), &offer()) + .run(&draft, &pin, &offer()) .await .expect_err("unmeasured"); assert!(matches!(err, SetupError::Report(_)), "{err}"); @@ -883,7 +928,7 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { // through setup had the RLM write a version: v1, v2 for the two refused // baselines, v3 now), baseline in store under the current version. orchestrator.set_flops_used(Some(1)); - let out = setup.run(&draft, &pin(), &offer()).await.expect("setup"); + let out = setup.run(&draft, &pin, &offer()).await.expect("setup"); assert_eq!(out.rules_version, 3); assert!((out.baseline_primary - 0.42).abs() < 1e-12); assert_eq!( @@ -914,13 +959,92 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { } ))); - // The operator seals custom_value (0.42) and re-signs open. - let mut open = draft.clone(); - open.status = TopicStatus::Open; - open.baseline.metrics_commitment = "22".repeat(32); - assert_eq!(setup.mark_sealed(&open).await.unwrap(), RlmState::Open); + // Sealing refuses everything that is not the signed, valid, open + // document sealing the RLM's 0.42 — and nothing moves or is stored. + let meas = sealed_measurement(&pin, &draft, 0.42); + let still_baselining = |store: &Arc| { + let store = store.clone(); + let id = draft.id.clone(); + async move { + let lc = store.lifecycle(&id).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Baselining); + let (version, latest) = store.latest_topic(&id).await.unwrap().unwrap(); + assert_eq!(version, 1, "no new version was stored"); + assert_eq!(latest.status, TopicStatus::Draft); + } + }; + + // Still a draft. + let err = setup + .mark_sealed(&draft, &pin, ®istered, &meas) + .await + .expect_err("draft"); + assert!(matches!(err, SetupError::NotOpen(_)), "{err}"); + still_baselining(&rlm_store).await; + + // Open but not signed by the operator key. + let mut unsigned = signed_open(&draft, &meas); + unsigned.signature = "00".repeat(64); + let err = setup + .mark_sealed(&unsigned, &pin, ®istered, &meas) + .await + .expect_err("unsigned"); + assert!( + matches!(err, SetupError::Topic(TopicError::SignatureInvalid)), + "{err}" + ); + still_baselining(&rlm_store).await; + + // Open and signed, but not valid as an open topic on this host (no runner). + let open = signed_open(&draft, &meas); + let err = setup + .mark_sealed(&open, &pin, &[], &meas) + .await + .expect_err("unregistered custom id cannot open"); + assert!( + matches!(err, SetupError::Topic(TopicError::UnknownCustomMetric(_))), + "{err}" + ); + still_baselining(&rlm_store).await; + + // Open, signed, valid, but the sealed value is not what the RLM measured. + let wrong = sealed_measurement(&pin, &draft, 0.99); + let wrong_doc = signed_open(&draft, &wrong); + let err = setup + .mark_sealed(&wrong_doc, &pin, ®istered, &wrong) + .await + .expect_err("sealed a value the rlm never measured"); + assert!( + matches!(err, SetupError::Seal(ref m) if m.contains("not the measured baseline")), + "{err}" + ); + still_baselining(&rlm_store).await; + + // Open, signed, valid, but the measurement does not bind to the document. + let err = setup + .mark_sealed(&open, &pin, ®istered, &wrong) + .await + .expect_err("commitment mismatch"); + assert!(matches!(err, SetupError::Seal(_)), "{err}"); + still_baselining(&rlm_store).await; + + // The real thing: baselining → open, version 2 stored. + assert_eq!( + setup + .mark_sealed(&open, &pin, ®istered, &meas) + .await + .unwrap(), + RlmState::Open + ); let (v, latest) = rlm_store.latest_topic(&draft.id).await.unwrap().unwrap(); assert_eq!(v, 2); assert_eq!(latest.status, TopicStatus::Open); + assert_eq!(latest.signature, open.signature); + // Sealing twice is illegal from open. + let err = setup + .mark_sealed(&open, &pin, ®istered, &meas) + .await + .expect_err("already open"); + assert!(matches!(err, SetupError::State(_)), "{err}"); let _ = std::fs::remove_dir_all(&root); } From 9ec78a38373b939154c21db509f689fcc4bd2589 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:35:59 +0000 Subject: [PATCH 12/14] docs(proof): lease, measured flops, locator, and sealed-open guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe what the Greptile P1 fixes on #243 changed for operators and miners: the run request carries `artifact_uri` and `flops_budget`, the verdict carries the runner-measured `flops_used` (missing → 503, over budget → reject; `declared_flops` is never the enforced figure); promotion is decided and persisted under a per-topic lease with a compare-and-swap on the best pointer; `mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value. --- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 28 ++++++++++++++++++++++++---- docs/external-miner/proof.md | 12 ++++++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 6218060d3..34285cf95 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -85,7 +85,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Live harvest | **partial** | `crates/proof-harvest` over `harvest-pod` stages `request.json`, `teacher.env`, `PROOF_PROXY_MODEL_DIR`, and `PROOF_HOLDOUT_STORE`. `PROOF_FORCE_SIM` is local-only. Live rent still needs a republished proof-eval digest (current pin still has the invalid HF default) plus operator-staged proxy dir + holdout shards. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | -| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux`, artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver. **No live VM orchestrator, no registered runner, no challenge content:** every custom topic answers **503** until an operator / RLM registers a runner. | +| RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No live VM orchestrator, no registered runner, no challenge content:** every custom topic answers **503** until an operator / RLM registers a runner. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index 15c157780..0779f83bc 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -386,9 +386,15 @@ open ⇄ evaluating → promoting → open … → closed`. `owner_presend` is a `awaiting_owner_keys` probes `PROOF_RLM_OWNER_INFERENCE_KEY_FILE` for presence only. `TopicSetup` drives the ceremony over the VM boundary (provision → RLM `ProposeRules` → rules vN in DB → `Baseline` job → -measurement in DB) and `mark_sealed` moves `baselining → open` after the -operator seals `custom_value` and re-signs. A re-run resumes from the -persisted state. +measurement in DB; a baseline measured over the topic `flops_budget` or +without a measurement is refused) and `mark_sealed` moves `baselining → +open` after the operator seals `custom_value` and re-signs. `mark_sealed` +is fail-closed: the document must be `status: open`, validate as an open +topic on this host (sealed baseline, registered `custom_id`, tighten-only +floors), verify under the pin's topic key, and the sealed +`BaselineMeasurement` must bind to it **and** carry the `custom_value` the +RLM measured — otherwise nothing moves and no version is stored. A re-run +resumes from the persisted state. ### Isolation boundary @@ -404,7 +410,13 @@ a `custom_id` is an operator / RLM action. The Lium harvest for govern the harvest rent; on the custom path each run request records the resolved executor plan's deadline (tighter of topic and plan) and `config_commitment` as provenance, and the row stamps `executor_commitment` -like every other scored row. +like every other scored row. The run request also carries the miner's +`artifact_uri` (the runner fetches it inside the VM and checks +`artifact_digest`) and the topic's `flops_budget`; the runner's report must +carry its measured `flops_used`, which becomes the verdict's usage — a +report without one is not evidence (**503**, no row), and a measurement over +budget is a persisted reject (`flops_over_budget`). The miner's +`declared_flops` is never the enforced figure. ### Runner registry @@ -426,6 +438,14 @@ Every scored row leaves `$PROOF_ARTEFACT_ROOT/{topic_id}/{submission_id}.zip` pass with a green checklist whose primary beats the bar (sealed value or reigning best) by `epsilon_rel`, direction-aware, persists as `champion`, gets a promotion row (with the displaced best), and moves the pointer. +Runs of one topic are serialised by a **lease** held from scoring until the +row is persisted, so the promotion is decided against the store's current +best (never a bar computed before an earlier crown) and written under the +same lease with a compare-and-swap on the best pointer: a crown whose +previous best moved, or that is not strictly better than the incumbent, is +refused (`promotion_refused` in the lifecycle, manifest `promoted: false`). +A run whose row never lands releases its lease after +`DEFAULT_LEASE_TTL` (5 min). `GET /v1/status` exposes pin `inference` public judge defaults (`provider`, `model`, `mode`, token caps) and `inference_offer` **public fields only** diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index d13788a40..8cb4e9f21 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -289,7 +289,7 @@ and the constraints, and must emit: | `claim_holds_public` | bool | Public-split numbers match the claim | | `contamination` | bool | Holdout fingerprints in the recipe / data | | `canary_hit` | bool | Off-score. Recorded, never a fail by itself | -| `flops_used` / `flops_budget` | u64 | Observed vs the topic budget | +| `flops_used` / `flops_budget` | u64 | Measured by the judge / runner vs the topic budget. Your `declared_flops` is a declaration, never the enforced figure; on custom topics the runner's measurement is the verdict's usage, and over budget is `flops_over_budget` | | `cheat_codes` | list | See below | | `rationale` | string | Audit text (truncated) | | `topic_id` / `family` | echo | Must match the submission | @@ -350,10 +350,18 @@ duplicated, or evidence-less item is a persisted `rejected` row with no spend. The rules may be re-versioned by the topic's RLM; the version you were ticked against is recorded with your row. +Send `artifact_uri` with your `artifact_digest`: the runner fetches the +bytes from it inside the topic VM and checks the digest, so a submission the +runner cannot retrieve cannot be inspected. The runner also measures your +run's FLOPs; that measurement (not `declared_flops`) is what the verdict +carries against the topic budget. + A clean pass that beats the current best (sealed value or reigning best) by `epsilon_rel` is promoted automatically: the row is `champion` and the operator archive keeps your artefact, `report.json`, and `checklist.json` -under `{topic_id}/{submission_id}.zip`. +under `{topic_id}/{submission_id}.zip`. Runs on one topic are scored and +crowned one at a time against the best at that moment, so a run that is not +strictly better than the reigning champion never replaces it. If the topic's `custom_id` is not in `registered_custom`, the topic is `open` but not in `scorable_topics`, and submits answer **503** (`no From 9f6e5256d883a548bafa0180c7612e14f7839df6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 15:01:02 +0000 Subject: [PATCH 13/14] fix(proof): a custom submission must carry its artifact locator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the Greptile P1 "artifact location discarded" on #243: the locator now reaches the runner, but a custom submission without one gave the runner nothing to retrieve at all — the request would still carry `None` for a real artefact. - `proof-http` intake: on a custom-family topic a missing / blank `artifact_uri` is a 400 (`artifact_uri is required for custom topics`) with no row and no rent; `nll` / `throughput` keep it optional (the digest-pinned image fetches by digest). - `RlmScorer::evaluate` refuses a request without a locator rather than hand the runner one it cannot act on; `CustomRunRequest.artifact_uri` is never `None` for a miner submission. - Tests: http 400 with no row and no scorer call (null and blank), the harvest path unaffected; e2e 400 before any VM job, every custom body carries a locator. Docs: required on custom topics. --- AGENTS.md | 2 +- crates/proof-http/src/lib.rs | 89 ++++++++++++++++++++++-- crates/proof-rlm-scorer/src/scorer.rs | 11 ++- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 38 +++++++--- docs/PROOF.md | 7 +- docs/external-miner/proof.md | 19 ++--- 6 files changed, 141 insertions(+), 25 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3d3dbf675..d08d5920a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 2. Edge / failure probes: bad harness, sanitize reject, quota, wrong routes/auth. 3. **Bounty — pair + report:** `ctx bounty pair --hotkey --account-id --accept-terms`, then `POST /v1/pair` (terms + signature) and `POST /v1/reports`. Operator bearer `POST /v1/admin/adjudicate` (`valid` / `already_fixed_not_prod` / `invalid_malicious` / `duplicate`). Scoring **reads** CortexLM/backend public JSON (`BOUNTY_BACKEND_PUBLIC_URL`); do not serve `/v1/public/*` from this repo. 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). -5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent. `GET /v1/proof/topics` must never leak holdout records. +5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent. `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. 7. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index f306c3aae..5c34cd741 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -43,8 +43,8 @@ use proof_store::{ freeze_submission_digest, ArtifactManifest, MemoryStore, Submission, SubmissionState, }; use proof_task::{ - resolve_inference, InferenceOffer, OfferError, ProofPin, TopicDocument, TopicError, - TopicStatus, CHALLENGE_ID, SCORE_MAX, SCORING_VERSION, + resolve_inference, InferenceOffer, MetricFamily, OfferError, ProofPin, TopicDocument, + TopicError, TopicStatus, CHALLENGE_ID, SCORE_MAX, SCORING_VERSION, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -327,6 +327,20 @@ async fn submit( "declared_flops exceeds the topic budget", )); } + // A custom-family runner retrieves the artefact from the miner's locator + // inside the topic VM; with none there is nothing to inspect, so the + // submission is refused here, before any row or rent. + let artifact_uri = body + .artifact_uri + .as_deref() + .map(str::trim) + .filter(|u| !u.is_empty()); + if topic.metric.family == MetricFamily::Custom && artifact_uri.is_none() { + return Err(err( + StatusCode::BAD_REQUEST, + "artifact_uri is required for custom topics", + )); + } let nonce = nonce_from(&hotkey, &topic_id, &artifact); let submission_digest = freeze_submission_digest(&hotkey, &topic_id, &artifact, &nonce); @@ -417,7 +431,7 @@ async fn submit( executor.as_ref(), &submission_digest, &artifact, - body.artifact_uri.as_deref(), + artifact_uri, &holdout, &body.claim, st.backend, @@ -1867,6 +1881,12 @@ mod tests { claim: &str, ) -> Result { self.ready_for_topic(topic)?; + if topic.metric.family == MetricFamily::Custom { + assert!( + artifact_uri.is_some_and(|u| !u.trim().is_empty()), + "intake must never hand a custom run to the scorer without a locator" + ); + } let mut doc = self .inner .score( @@ -2034,7 +2054,10 @@ mod tests { "/v1/submissions", submit_body( "custom-artifact", - &serde_json::json!({ "topic_id": "custom-topic-v0" }), + &serde_json::json!({ + "topic_id": "custom-topic-v0", + "artifact_uri": "https://example.invalid/custom-artifact.zip", + }), ), None, ) @@ -2053,6 +2076,59 @@ mod tests { assert!(scorer.persisted.lock().expect("p").is_empty()); } + /// A custom-family runner retrieves the artefact from the miner's + /// locator; a custom submission without one is refused at intake with + /// no row and no scorer call. `nll` / `throughput` keep it optional. + #[tokio::test] + async fn a_custom_submission_without_a_locator_is_400_with_no_row() { + let scorer = Arc::new(FamilyStub::win("topic_minted_metric")); + let app = app_with_custom(scorer.clone()); + for missing in [serde_json::Value::Null, serde_json::json!(" ")] { + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body( + "no-locator", + &serde_json::json!({ + "topic_id": "custom-topic-v0", + "artifact_uri": missing, + }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "artifact_uri is required for custom topics"); + } + assert_eq!(scorer.inner.hits.load(Ordering::SeqCst), 0, "no run"); + let (_, list) = json_req( + app.clone(), + "GET", + "/v1/submissions", + serde_json::json!({}), + None, + ) + .await; + assert!( + list["items"].as_array().is_some_and(Vec::is_empty), + "{list}" + ); + let (st, created) = json_req( + app, + "POST", + "/v1/submissions", + submit_body("harvest-topic", &serde_json::json!({})), + None, + ) + .await; + assert_eq!( + st, + StatusCode::CREATED, + "the harvest fetches by digest: {created}" + ); + } + /// The generic promotion plumbing: a pass the family scorer crowns is /// persisted as `champion`, and the persist hook fires with the row id. #[tokio::test] @@ -2065,7 +2141,10 @@ mod tests { "/v1/submissions", submit_body( "crowned", - &serde_json::json!({ "topic_id": "custom-topic-v0" }), + &serde_json::json!({ + "topic_id": "custom-topic-v0", + "artifact_uri": "https://example.invalid/crowned.zip", + }), ), None, ) diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index f7eecdf7c..fa667b778 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -458,6 +458,15 @@ impl RlmScorer { .registry .resolve(&custom_id) .map_err(|e| map_runner(&custom_id, e))?; + // The runner can only retrieve the artefact from the miner's locator; + // intake refuses a custom submission without one, and so does this + // path rather than hand the runner a request it cannot act on. + let Some(artifact_uri) = artifact_uri.map(str::trim).filter(|u| !u.is_empty()) else { + return Err(EvalError::Backend( + "custom submission carries no artifact_uri; the runner cannot retrieve the artefact" + .into(), + )); + }; let rules = self.rules_for(topic).await?; let req = CustomRunRequest::from_topic( topic, @@ -466,7 +475,7 @@ impl RlmScorer { &rules, frozen_digest, artifact_digest, - artifact_uri, + Some(artifact_uri), claim, ) .map_err(|e| map_runner(&custom_id, e))? diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index d3d36a239..920b4bba6 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -4,7 +4,8 @@ //! `VmBackedRunner` → fake orchestrator, with the memory RLM store and a //! temp artefact root. //! -//! Covers: an unregistered `custom_id` is a 503 with no row; a green +//! Covers: an unregistered `custom_id` is a 503 with no row; a custom +//! submission without an artefact locator is a 400 with no row; a green //! checklist scores and is crowned against the sealed value, with the //! miner's artefact locator reaching the runner and the runner's measured //! FLOPs in the verdict; a red checklist is a persisted reject with **zero** @@ -204,10 +205,18 @@ async fn json_req( (status, v) } +/// Where a test miner says the bytes behind `label` live. +fn locator(label: &str) -> String { + format!("https://example.invalid/artefacts/{label}.zip") +} + +/// A custom-topic submission; the locator is required on custom topics, so +/// every body carries one. fn submit_body(topic_id: &str, label: &str) -> serde_json::Value { serde_json::json!({ "miner_hotkey": digest("miner"), "artifact_digest": digest(label), + "artifact_uri": locator(label), "claim": "placeholder claim", "declared_flops": 1u64, "topic_id": topic_id, @@ -314,12 +323,25 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { "holdout leak" ); + // 0b. A custom submission without a locator is refused at intake: no + // row, no VM job, no rent — the runner would have nothing to fetch. + let mut no_locator = submit_body(&tid, "artifact-a"); + no_locator["artifact_uri"] = serde_json::Value::Null; + let (st, body) = json_req(app.clone(), "POST", "/v1/submissions", no_locator).await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "artifact_uri is required for custom topics"); + assert!(orchestrator.jobs().is_empty(), "no job without a locator"); + // 1. Green checklist, primary 0.70 > 0.50 * 1.02: scored and crowned. // The miner's artefact locator travels to the runner with the digest. - let uri = "https://example.invalid/artefacts/artifact-a.zip"; - let mut body_a = submit_body(&tid, "artifact-a"); - body_a["artifact_uri"] = serde_json::json!(uri); - let (st, created) = json_req(app.clone(), "POST", "/v1/submissions", body_a).await; + let uri = locator("artifact-a"); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-a"), + ) + .await; assert_eq!(st, StatusCode::CREATED, "{created}"); assert_eq!(created["eligible"], true, "{created}"); assert_eq!(created["state"], "champion", "{created}"); @@ -338,7 +360,7 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { for req in &requests { assert_eq!( req.artifact_uri.as_deref(), - Some(uri), + Some(uri.as_str()), "runner must receive the submitted locator" ); assert_eq!(req.artifact_digest, digest("artifact-a")); @@ -640,7 +662,7 @@ async fn score(d: &Direct, label: &str) -> Result &d.plan, &format!("digest-{label}"), &digest(label), - None, + Some(&locator(label)), &[], "placeholder claim", ) @@ -673,7 +695,7 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { &plan_b, "digest-b", &digest("b"), - None, + Some(&locator("b")), &[], "placeholder claim", ) diff --git a/docs/PROOF.md b/docs/PROOF.md index 0779f83bc..49d473b36 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -236,7 +236,8 @@ Trust-root keygen is the throwaway owner path in reigning best) persists as `champion`; other passes stay `awaiting_admin`. - Submit fields miners must send: `claim` (what the recipe achieved), `declared_flops` (≤ topic budget), `artifact_digest` of a **reproducible - train/eval recipe** (code under budget, not weights-only), plus `manifest`. + train/eval recipe** (code under budget, not weights-only), plus `manifest`; + on custom topics also `artifact_uri` (the runner fetches from it). The agent verdict (`reproduced`, `claim_holds_public`, cheat codes) is filled by the eval image, not the miner. - Contamination / empty manifest: persist **rejected** without renting. @@ -412,7 +413,9 @@ resolved executor plan's deadline (tighter of topic and plan) and `config_commitment` as provenance, and the row stamps `executor_commitment` like every other scored row. The run request also carries the miner's `artifact_uri` (the runner fetches it inside the VM and checks -`artifact_digest`) and the topic's `flops_budget`; the runner's report must +`artifact_digest`; a custom submission without one is a **400** at intake, +no row, and the scorer refuses a request without it) and the topic's +`flops_budget`; the runner's report must carry its measured `flops_used`, which becomes the verdict's usage — a report without one is not evidence (**503**, no row), and a measurement over budget is a persisted reject (`flops_over_budget`). The miner's diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 8cb4e9f21..f63bd023f 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -162,8 +162,10 @@ are paid on**. You never see the records. Build a recipe the judge can re-run: code, lockfile, and entrypoint, under the topic's FLOP (and for throughput, wall) budget. Hash that tree. That hash -is `artifact_digest`. Optional `artifact_uri` is a locator (git URL, object -URL) so the image can fetch the same bytes. +is `artifact_digest`. `artifact_uri` is a locator (git URL, object URL) for +the same bytes: optional on `nll` / `throughput` (the image fetches by +digest), **required on custom topics** (the topic's runner fetches from it +inside the topic VM and checks the digest; without one the submit is a 400). The **claim** is one English sentence of what improved. The RLM re-runs the code against the public split and checks the claim against those public @@ -224,7 +226,7 @@ judge config, no open sealed topic), submissions answer **503**. | `declared_flops` | yes | `u64`, must be `≤ topic.flops_budget` | | `manifest.train_content_hashes` | yes (array) | Shard hashes you trained on (may be `[]` if you declare dataset ids) | | `manifest.train_dataset_ids` | yes (array) | Corpus ids you trained on (may be `[]` if you declare hashes) | -| `artifact_uri` | no | Locator for the same bytes as `artifact_digest` | +| `artifact_uri` | custom topics: yes | Locator for the same bytes as `artifact_digest`; optional on `nll` / `throughput` | An empty `manifest` (both arrays empty / omitted) is **not** a clean contamination check. It is `contamination_evidence_missing`: the row is @@ -259,6 +261,7 @@ Refusals (**400** / **503**) do **not** persist a submission row. | **400** `unknown topic` | `topic_id` not published | no | no | | **400** `topic is not open` | Draft / closed / outside epoch window | no | no | | **400** `declared_flops exceeds the topic budget` | `declared_flops > topic.flops_budget` | no | no | +| **400** `artifact_uri is required for custom topics` | Custom topic, no locator | no | no | | **400** invalid `miner_hotkey` / `artifact_digest` | Not 64 hex | no | no | | **503** empty `eval_image_digest` | Digest not pinned | no | no | | **503** zero open sealed topics | Nothing to score against | no | no | @@ -350,11 +353,11 @@ duplicated, or evidence-less item is a persisted `rejected` row with no spend. The rules may be re-versioned by the topic's RLM; the version you were ticked against is recorded with your row. -Send `artifact_uri` with your `artifact_digest`: the runner fetches the -bytes from it inside the topic VM and checks the digest, so a submission the -runner cannot retrieve cannot be inspected. The runner also measures your -run's FLOPs; that measurement (not `declared_flops`) is what the verdict -carries against the topic budget. +`artifact_uri` is required: the runner fetches the bytes from it inside the +topic VM and checks the digest, so a submission the runner cannot retrieve is +a **400** with no row. The runner also measures your run's FLOPs; that +measurement (not `declared_flops`) is what the verdict carries against the +topic budget. A clean pass that beats the current best (sealed value or reigning best) by `epsilon_rel` is promoted automatically: the row is `champion` and the From ad3c6855a9f1b11c5a83db85806bef935b5f8b27 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 15:01:43 +0000 Subject: [PATCH 14/14] fix(proof-rlm): hold the measured flops to the miner's declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the Greptile P1 "FLOP budget bypass" on #243: the verdict now carries the runner's measurement against the signed budget, but the miner's `declared_flops` (checked `<=` budget at intake) had no teeth — a run could spend far more than it declared and still pass. - `LiveScorer::score` / `eval_after_freeze` / `FamilyMux` carry `declared_flops`; `proof-http` forwards the body's value; the Lium harvest ignores it (its agent observes usage against the budget). - `CustomRunRequest.declared_flops`: the runner may enforce it as a hard cap inside the topic VM. The operator baseline declares the budget. - `RlmScorer::paid_verdict`: measured usage over the budget is `flops_over_budget`, over the declaration is the new `ProofCheatCode::FlopsUnderDeclared`; either is a `reject` the judge fails the row on. Missing usage stays a 503 (not evidence). - e2e: declared 1, measured 2 → persisted reject with `flops_under_declared` only; declared the budget, measured one over → `flops_over_budget`; measured exactly the declared budget → crowned. Docs: declare what you will use, up to the budget. --- AGENTS.md | 2 +- crates/proof-eval/src/lib.rs | 24 +++++++- crates/proof-harvest/src/lib.rs | 8 ++- crates/proof-http/src/lib.rs | 4 ++ crates/proof-rlm-scorer/src/scorer.rs | 47 +++++++++------ crates/proof-rlm-scorer/src/setup.rs | 1 + crates/proof-rlm-scorer/tests/rlm_e2e.rs | 77 +++++++++++++++++++----- crates/proof-rlm/src/fixtures_tests.rs | 1 + crates/proof-rlm/src/runner.rs | 12 +++- crates/proof-score/src/lib.rs | 2 + docs/PROOF.md | 14 +++-- docs/external-miner/proof.md | 9 ++- 12 files changed, 150 insertions(+), 51 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d08d5920a..926036d7e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 2. Edge / failure probes: bad harness, sanitize reject, quota, wrong routes/auth. 3. **Bounty — pair + report:** `ctx bounty pair --hotkey --account-id --accept-terms`, then `POST /v1/pair` (terms + signature) and `POST /v1/reports`. Operator bearer `POST /v1/admin/adjudicate` (`valid` / `already_fixed_not_prod` / `invalid_malicious` / `duplicate`). Scoring **reads** CortexLM/backend public JSON (`BOUNTY_BACKEND_PUBLIC_URL`); do not serve `/v1/public/*` from this repo. 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). -5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent. `GET /v1/proof/topics` must never leak holdout records. +5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. 7. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 9429c6c30..461880cab 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -295,7 +295,9 @@ pub trait LiveScorer: Send + Sync { /// rent ([`Self::plan`]) the image is run under. Both are host state. /// `artifact_uri` is the miner-supplied locator of the bytes behind /// `artifact_digest` (the runner fetches and digest-checks them); the - /// digest alone is not enough to retrieve an artefact. + /// digest alone is not enough to retrieve an artefact. `declared_flops` + /// is the miner's declaration (already `<=` the topic budget at intake): + /// a scorer that measures usage must fail a run that exceeds it. #[allow(clippy::too_many_arguments)] async fn score( &self, @@ -306,6 +308,7 @@ pub trait LiveScorer: Send + Sync { frozen_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, holdout: &[HoldoutRecord], claim: &str, ) -> Result; @@ -421,6 +424,7 @@ impl LiveScorer for FamilyMux { frozen_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -433,6 +437,7 @@ impl LiveScorer for FamilyMux { frozen_digest, artifact_digest, artifact_uri, + declared_flops, holdout, claim, ) @@ -892,8 +897,9 @@ pub fn sim_win_document( /// Score only after the submission digest is frozen and a topic is open. /// -/// `artifact_uri` travels to the live scorer untouched: it is the miner's -/// locator for the bytes behind `artifact_digest`, never trusted beyond that. +/// `artifact_uri` and `declared_flops` travel to the live scorer untouched: +/// the miner's locator for the bytes behind `artifact_digest` (never trusted +/// beyond that) and the miner's FLOP declaration the measured run is held to. #[allow(clippy::too_many_arguments)] pub async fn eval_after_freeze( pin: &ProofPin, @@ -903,6 +909,7 @@ pub async fn eval_after_freeze( frozen_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, holdout: &[HoldoutRecord], claim: &str, backend: EvalBackend, @@ -972,6 +979,7 @@ pub async fn eval_after_freeze( frozen_digest, artifact_digest, artifact_uri, + declared_flops, holdout, claim, ) @@ -1111,6 +1119,7 @@ mod tests { frozen: &str, artifact: &str, _artifact_uri: Option<&str>, + _declared_flops: u64, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -1143,6 +1152,7 @@ mod tests { "d", "art", None, + 1, &recs, "claim", EvalBackend::Lium, @@ -1165,6 +1175,7 @@ mod tests { "d", "art", None, + 1, &recs, "claim", EvalBackend::Lium, @@ -1193,6 +1204,7 @@ mod tests { "digest-a", "art", None, + 1, &recs, "claim", EvalBackend::Lium, @@ -1399,6 +1411,7 @@ mod tests { "digest-a", "art", None, + 1, &recs, "claim", EvalBackend::Lium, @@ -1422,6 +1435,7 @@ mod tests { "digest-a", "art", None, + 1, &recs, "claim", EvalBackend::Lium, @@ -1482,6 +1496,7 @@ mod tests { _frozen: &str, _artifact: &str, _artifact_uri: Option<&str>, + _declared_flops: u64, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -1586,6 +1601,7 @@ mod tests { "d", "a", None, + 1, &recs, "c", ) @@ -1707,6 +1723,7 @@ mod tests { "digest-a", "art", None, + 1, &recs, "claim", EvalBackend::Sim, @@ -1742,6 +1759,7 @@ mod tests { "digest-a", "art", None, + 1, &recs, "claim", EvalBackend::Lium, diff --git a/crates/proof-harvest/src/lib.rs b/crates/proof-harvest/src/lib.rs index 62b7b04cd..e83a79355 100644 --- a/crates/proof-harvest/src/lib.rs +++ b/crates/proof-harvest/src/lib.rs @@ -661,9 +661,11 @@ impl LiveScorer for LiumProofHarvest { plan: &ExecutorPlan, frozen_digest: &str, artifact_digest: &str, - // The digest-pinned image fetches by digest from the artifact store; - // the miner locator is a custom-family concern. + // The digest-pinned image fetches by digest from the artifact store + // and its agent observes usage against the topic budget; the miner + // locator and declaration are custom-family concerns. _artifact_uri: Option<&str>, + _declared_flops: u64, holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -778,7 +780,7 @@ mod tests { let plan = harvest.plan(pin, topic, executor)?; harvest .score( - pin, topic, offer, &plan, frozen, artifact, None, holdout, claim, + pin, topic, offer, &plan, frozen, artifact, None, 1, holdout, claim, ) .await } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 5c34cd741..d6fb428e8 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -432,6 +432,7 @@ async fn submit( &submission_digest, &artifact, artifact_uri, + body.declared_flops, &holdout, &body.claim, st.backend, @@ -991,6 +992,7 @@ mod tests { frozen: &str, artifact: &str, _artifact_uri: Option<&str>, + _declared_flops: u64, _holdout: &[proof_task::HoldoutRecord], _claim: &str, ) -> Result { @@ -1877,6 +1879,7 @@ mod tests { frozen: &str, artifact: &str, artifact_uri: Option<&str>, + declared_flops: u64, holdout: &[proof_task::HoldoutRecord], claim: &str, ) -> Result { @@ -1897,6 +1900,7 @@ mod tests { frozen, artifact, artifact_uri, + declared_flops, holdout, claim, ) diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index fa667b778..ec0a6ece7 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -11,8 +11,8 @@ //! 4. red → **reject document, no paid inference**; //! 5. green → [`SpendToken`] → **evaluate** (the only paid step) → report → //! `custom_value = primary_value`, `flops_used` = the runner's -//! measurement (a report without one is not evidence; over budget is a -//! reject the judge fails on); +//! measurement (a report without one is not evidence; over the topic +//! budget or over the miner's declaration is a reject the judge fails on); //! 6. on persist: artefact zip + metadata row + public event; on promotion: //! promotion row, `best.json`, lifecycle `promoting → open`. //! @@ -27,6 +27,7 @@ //! [`SpendToken`]: proof_rlm::SpendToken use std::collections::BTreeMap; +use std::fmt::Write as _; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant}; @@ -404,8 +405,9 @@ impl RlmScorer { } /// Verdict for a verified paid run: the runner's measured FLOPs are the - /// verdict's usage, and a measurement over the topic budget is a reject - /// (the judge's `FlopsOverBudget` gate then fails the row). + /// verdict's usage. A measurement over the topic budget + /// (`FlopsOverBudget`) or over what the miner declared + /// (`FlopsUnderDeclared`) is a reject the judge then fails the row on. fn paid_verdict( topic: &TopicDocument, req: &CustomRunRequest, @@ -415,29 +417,36 @@ impl RlmScorer { let flops_used = report .flops_used_for(req) .map_err(|e| EvalError::NoVerdict(e.to_string()))?; - let measured = format!( + let mut rationale = format!( "{}: {} = {:.6}; checklist green (rules v{rules_version}); sandboxed={}; flops_used={flops_used}", req.custom_id, req.primary, report.primary_value, report.sandboxed ); + let mut cheats = Vec::new(); if flops_used > req.flops_budget { - return Ok(Self::agent( - topic, - ProofKind::Reject, - true, - report.claim_holds, - flops_used, - vec![ProofCheatCode::FlopsOverBudget], - format!("{measured}; over the topic budget {}", req.flops_budget), - )); + cheats.push(ProofCheatCode::FlopsOverBudget); + let _ = write!(rationale, "; over the topic budget {}", req.flops_budget); } + if flops_used > req.declared_flops { + cheats.push(ProofCheatCode::FlopsUnderDeclared); + let _ = write!( + rationale, + "; over the miner's declared_flops {}", + req.declared_flops + ); + } + let kind = if cheats.is_empty() { + ProofKind::Clean + } else { + ProofKind::Reject + }; Ok(Self::agent( topic, - ProofKind::Clean, + kind, true, report.claim_holds, flops_used, - Vec::new(), - measured, + cheats, + rationale, )) } @@ -451,6 +460,7 @@ impl RlmScorer { frozen_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, claim: &str, ) -> Result { let custom_id = topic.metric.custom_id.trim().to_owned(); @@ -476,6 +486,7 @@ impl RlmScorer { frozen_digest, artifact_digest, Some(artifact_uri), + declared_flops, claim, ) .map_err(|e| map_runner(&custom_id, e))? @@ -689,6 +700,7 @@ impl LiveScorer for RlmScorer { frozen_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, _holdout: &[HoldoutRecord], claim: &str, ) -> Result { @@ -707,6 +719,7 @@ impl LiveScorer for RlmScorer { frozen_digest, artifact_digest, artifact_uri, + declared_flops, claim, ) .await; diff --git a/crates/proof-rlm-scorer/src/setup.rs b/crates/proof-rlm-scorer/src/setup.rs index b17301441..a3afb90b9 100644 --- a/crates/proof-rlm-scorer/src/setup.rs +++ b/crates/proof-rlm-scorer/src/setup.rs @@ -253,6 +253,7 @@ impl TopicSetup { &format!("baseline-{}", rules.digest()), &topic.baseline.script_sha256, None, + topic.flops_budget, "operator baseline", ) .map_err(|e| SetupError::Vm(VmError::Backend(e.to_string())))?; diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 920b4bba6..8c3472cc5 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -7,10 +7,11 @@ //! Covers: an unregistered `custom_id` is a 503 with no row; a custom //! submission without an artefact locator is a 400 with no row; a green //! checklist scores and is crowned against the sealed value, with the -//! miner's artefact locator reaching the runner and the runner's measured -//! FLOPs in the verdict; a red checklist is a persisted reject with **zero** -//! paid runs; a later pass below the best stays `awaiting_admin`; an -//! over-budget measurement is a persisted reject and a missing one is a 503; +//! miner's artefact locator and declaration reaching the runner and the +//! runner's measured FLOPs in the verdict; a red checklist is a persisted +//! reject with **zero** paid runs; a later pass below the best stays +//! `awaiting_admin`; a measurement over the budget or over the miner's +//! declaration is a persisted reject and a missing one is a 503; //! every scored row leaves its zip, the crown leaves `best.json` + a //! promotion row, and the store holds rules v1, every checklist, and the //! lifecycle. Runs hold their topic lease until persisted, so a worse run @@ -72,6 +73,7 @@ impl LiveScorer for IdleHarvest { _frozen: &str, _artifact: &str, _artifact_uri: Option<&str>, + _declared_flops: u64, _holdout: &[HoldoutRecord], _claim: &str, ) -> Result { @@ -210,20 +212,24 @@ fn locator(label: &str) -> String { format!("https://example.invalid/artefacts/{label}.zip") } -/// A custom-topic submission; the locator is required on custom topics, so -/// every body carries one. -fn submit_body(topic_id: &str, label: &str) -> serde_json::Value { +/// A custom-topic submission declaring `declared_flops`; the locator is +/// required on custom topics, so every body carries one. +fn submit_declaring(topic_id: &str, label: &str, declared_flops: u64) -> serde_json::Value { serde_json::json!({ "miner_hotkey": digest("miner"), "artifact_digest": digest(label), "artifact_uri": locator(label), "claim": "placeholder claim", - "declared_flops": 1u64, + "declared_flops": declared_flops, "topic_id": topic_id, "manifest": { "train_dataset_ids": ["placeholder-corpus"] }, }) } +fn submit_body(topic_id: &str, label: &str) -> serde_json::Value { + submit_declaring(topic_id, label, 1) +} + fn zip_names(path: &std::path::Path) -> Vec { let bytes = std::fs::read(path).unwrap(); let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); @@ -333,7 +339,8 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { assert!(orchestrator.jobs().is_empty(), "no job without a locator"); // 1. Green checklist, primary 0.70 > 0.50 * 1.02: scored and crowned. - // The miner's artefact locator travels to the runner with the digest. + // The miner's artefact locator and declaration travel to the runner + // with the digest. let uri = locator("artifact-a"); let (st, created) = json_req( app.clone(), @@ -365,6 +372,7 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { ); assert_eq!(req.artifact_digest, digest("artifact-a")); assert_eq!(req.flops_budget, topic.flops_budget); + assert_eq!(req.declared_flops, 1, "the miner's declaration is the cap"); } let (_, row) = json_req( @@ -517,7 +525,8 @@ async fn submit_scores_rejects_and_promotes_through_the_registry_end_to_end() { /// The signed budget binds the runner's measurement, not the miner's /// declaration: a run measured over budget is a persisted reject even with a -/// winning primary, and a report with no measurement is refused with no row. +/// winning primary, so is one measured over what the miner declared, and a +/// report with no measurement is refused with no row. #[tokio::test] async fn an_over_budget_or_unmeasured_run_never_passes() { let Stack { @@ -530,12 +539,13 @@ async fn an_over_budget_or_unmeasured_run_never_passes() { let tid = topic.id.clone(); let budget = topic.flops_budget; + // Declared the whole budget, measured one over it. orchestrator.set_flops_used(Some(budget + 1)); let (st, created) = json_req( app.clone(), "POST", "/v1/submissions", - submit_body(&tid, "artifact-over"), + submit_declaring(&tid, "artifact-over", budget), ) .await; assert_eq!(st, StatusCode::CREATED, "{created}"); @@ -568,6 +578,38 @@ async fn an_over_budget_or_unmeasured_run_never_passes() { assert!(rlm_store.best(&tid).await.unwrap().is_none()); assert!(root.join(&tid).join(format!("{id}.zip")).is_file()); + // Within budget but over what the miner declared (1): under-declared, + // persisted reject, the declaration is the cap the miner committed to. + orchestrator.set_flops_used(Some(2)); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body(&tid, "artifact-under-declared"), + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "rejected", "{created}"); + let id = created["id"].as_str().unwrap().to_owned(); + let (_, row) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + ) + .await; + assert_eq!(row["verdict"]["pass"], false, "{row}"); + assert_eq!(row["verdict"]["agent"]["flops_used"], 2u64, "{row}"); + assert_eq!(row["declared_flops"], 1u64, "{row}"); + let codes = row["verdict"]["agent"]["cheat_codes"].to_string(); + assert!(codes.contains("flops_under_declared"), "{codes}"); + assert!(!codes.contains("flops_over_budget"), "{codes}"); + assert!(row["verdict"]["agent"]["rationale"] + .as_str() + .unwrap() + .contains("over the miner's declared_flops 1")); + assert!(rlm_store.best(&tid).await.unwrap().is_none()); + // No measurement at all: not evidence, 503, no row. orchestrator.set_flops_used(None); let (st, body) = json_req( @@ -586,11 +628,11 @@ async fn an_over_budget_or_unmeasured_run_never_passes() { "{body}" ); let (_, list) = json_req(app.clone(), "GET", "/v1/submissions", serde_json::json!({})).await; - assert_eq!(list["items"].as_array().unwrap().len(), 1, "{list}"); + assert_eq!(list["items"].as_array().unwrap().len(), 2, "{list}"); assert_eq!( paid_runs(&orchestrator), - 2, - "both runs were paid; neither passed" + 3, + "every run was paid; none passed" ); let lc = rlm_store.lifecycle(&tid).await.unwrap().unwrap(); assert_eq!( @@ -599,13 +641,13 @@ async fn an_over_budget_or_unmeasured_run_never_passes() { "a refusal closes the verdict phase" ); - // Back within budget the same topic scores again. + // Measured exactly the budget the miner declared: scores and is crowned. orchestrator.set_flops_used(Some(budget)); let (st, created) = json_req( app, "POST", "/v1/submissions", - submit_body(&tid, "artifact-at-budget"), + submit_declaring(&tid, "artifact-at-budget", budget), ) .await; assert_eq!(st, StatusCode::CREATED, "{created}"); @@ -663,6 +705,7 @@ async fn score(d: &Direct, label: &str) -> Result &format!("digest-{label}"), &digest(label), Some(&locator(label)), + d.topic.flops_budget, &[], "placeholder claim", ) @@ -687,6 +730,7 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { let scorer_b = d.scorer.clone(); let (pin_b, topic_b, plan_b) = (d.pin.clone(), d.topic.clone(), d.plan.clone()); let mut task_b = tokio::spawn(async move { + let budget = topic_b.flops_budget; scorer_b .score( &pin_b, @@ -696,6 +740,7 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { "digest-b", &digest("b"), Some(&locator("b")), + budget, &[], "placeholder claim", ) diff --git a/crates/proof-rlm/src/fixtures_tests.rs b/crates/proof-rlm/src/fixtures_tests.rs index 0f45addca..20b67921d 100644 --- a/crates/proof-rlm/src/fixtures_tests.rs +++ b/crates/proof-rlm/src/fixtures_tests.rs @@ -125,6 +125,7 @@ pub fn request() -> CustomRunRequest { "digest-a", &"ab".repeat(32), Some("https://example.invalid/artifact.zip"), + 1, "placeholder claim", ) .expect("request") diff --git a/crates/proof-rlm/src/runner.rs b/crates/proof-rlm/src/runner.rs index cd9dd4482..7fcd84798 100644 --- a/crates/proof-rlm/src/runner.rs +++ b/crates/proof-rlm/src/runner.rs @@ -79,6 +79,10 @@ pub struct CustomRunRequest { /// FLOPs one run may spend (the topic's signed `flops_budget`). The report /// must carry its measured usage against this figure. pub flops_budget: u64, + /// FLOPs the miner declared for this run (`<= flops_budget` at intake). + /// The runner may enforce it as a hard cap inside the VM; a measurement + /// above it is an under-declaration the host rejects. + pub declared_flops: u64, /// Topic constraints (sandbox flag, model pin, opaque slice / params). pub constraints: Constraints, /// Rule version the checklist must tick. @@ -137,6 +141,7 @@ impl CustomRunRequest { submission_digest: &str, artifact_digest: &str, artifact_uri: Option<&str>, + declared_flops: u64, claim: &str, ) -> Result { if topic.metric.family != MetricFamily::Custom { @@ -157,6 +162,7 @@ impl CustomRunRequest { .map(str::to_owned), claim: claim.to_owned(), flops_budget: topic.flops_budget, + declared_flops, constraints: topic.constraints.clone(), rules_version: rules.version, rules_digest: rules.digest(), @@ -471,6 +477,7 @@ mod tests { req.flops_budget, t.flops_budget, "budget travels to the runner" ); + assert_eq!(req.declared_flops, 1, "the miner's declaration travels too"); assert_eq!( req.artifact_uri.as_deref(), Some("https://example.invalid/artifact.zip"), @@ -489,7 +496,7 @@ mod tests { let mut plain = t; plain.metric.family = MetricFamily::Nll; assert!(matches!( - CustomRunRequest::from_topic(&plain, &pin(), &offer(), &rules(), "d", "a", None, ""), + CustomRunRequest::from_topic(&plain, &pin(), &offer(), &rules(), "d", "a", None, 1, ""), Err(RunnerError::NotCustom(_)) )); let bound = request().with_executor_plan(900, "ab".repeat(32).as_str()); @@ -504,7 +511,7 @@ mod tests { let mut tight = topic(); tight.eval_executor.max_proof_deadline_s = Some(600); let req = - CustomRunRequest::from_topic(&tight, &pin(), &offer(), &rules(), "d", "a", None, "") + CustomRunRequest::from_topic(&tight, &pin(), &offer(), &rules(), "d", "a", None, 1, "") .expect("request"); assert_eq!(req.sandbox.deadline_s, 600); let blank = CustomRunRequest::from_topic( @@ -515,6 +522,7 @@ mod tests { "d", "a", Some(" "), + 1, "", ) .expect("request"); diff --git a/crates/proof-score/src/lib.rs b/crates/proof-score/src/lib.rs index 6c0fee16b..ed18a6d32 100644 --- a/crates/proof-score/src/lib.rs +++ b/crates/proof-score/src/lib.rs @@ -50,6 +50,8 @@ pub enum ProofCheatCode { UnreproducedClaim, /// The run spent more FLOPs than the topic budget. FlopsOverBudget, + /// The run spent more FLOPs than the miner declared for it. + FlopsUnderDeclared, /// Compared against a weaker/different AdamW than the sealed recipe. StrawmanAdamw, /// Optimizer named Muon/TSP (etc.) but the code is AdamW. diff --git a/docs/PROOF.md b/docs/PROOF.md index 49d473b36..699f15841 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -414,12 +414,14 @@ resolved executor plan's deadline (tighter of topic and plan) and like every other scored row. The run request also carries the miner's `artifact_uri` (the runner fetches it inside the VM and checks `artifact_digest`; a custom submission without one is a **400** at intake, -no row, and the scorer refuses a request without it) and the topic's -`flops_budget`; the runner's report must -carry its measured `flops_used`, which becomes the verdict's usage — a -report without one is not evidence (**503**, no row), and a measurement over -budget is a persisted reject (`flops_over_budget`). The miner's -`declared_flops` is never the enforced figure. +no row, and the scorer refuses a request without it), the topic's +`flops_budget`, and the miner's `declared_flops` (the runner may enforce it +as a hard cap). The runner's report must carry its measured `flops_used`, +which becomes the verdict's usage — a report without one is not evidence +(**503**, no row); a measurement over the budget (`flops_over_budget`) or +over the miner's declaration (`flops_under_declared`) is a persisted reject. +The miner's `declared_flops` is never the enforced usage figure; it is the +cap the measurement is held to. ### Runner registry diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index f63bd023f..3257c681c 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -292,7 +292,7 @@ and the constraints, and must emit: | `claim_holds_public` | bool | Public-split numbers match the claim | | `contamination` | bool | Holdout fingerprints in the recipe / data | | `canary_hit` | bool | Off-score. Recorded, never a fail by itself | -| `flops_used` / `flops_budget` | u64 | Measured by the judge / runner vs the topic budget. Your `declared_flops` is a declaration, never the enforced figure; on custom topics the runner's measurement is the verdict's usage, and over budget is `flops_over_budget` | +| `flops_used` / `flops_budget` | u64 | Measured by the judge / runner vs the topic budget. Your `declared_flops` is never the enforced usage figure; on custom topics the runner's measurement is the verdict's usage, over budget is `flops_over_budget`, and over your own declaration is `flops_under_declared` | | `cheat_codes` | list | See below | | `rationale` | string | Audit text (truncated) | | `topic_id` / `family` | echo | Must match the submission | @@ -308,6 +308,7 @@ a win: |------|---------| | `unreproduced_claim` | Could not re-run the claimed recipe to the claimed result | | `flops_over_budget` | Run spent more FLOPs than the topic budget | +| `flops_under_declared` | Run spent more FLOPs than your `declared_flops` (custom topics: the runner's measurement is held to your declaration) | | `strawman_adamw` | Compared against a weaker / different AdamW than the sealed recipe | | `fake_optimizer` | Optimizer named Muon / TSP (etc.) but the code is AdamW | | `contamination` | Training data overlapped the holdout | @@ -356,8 +357,10 @@ ticked against is recorded with your row. `artifact_uri` is required: the runner fetches the bytes from it inside the topic VM and checks the digest, so a submission the runner cannot retrieve is a **400** with no row. The runner also measures your run's FLOPs; that -measurement (not `declared_flops`) is what the verdict carries against the -topic budget. +measurement (not `declared_flops`) is what the verdict carries, and it must +stay within both the topic budget (`flops_over_budget`) and your own +`declared_flops` (`flops_under_declared`) — declare what you will use, up to +the budget. The runner may enforce your declaration as a hard cap. A clean pass that beats the current best (sealed value or reigning best) by `epsilon_rel` is promoted automatically: the row is `champion` and the