From 1ffd25998444f805d48a03c185ef8390ad2ddd0c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 22:09:16 +0000 Subject: [PATCH 1/2] feat(proof): snapshot journal payouts and listed-op flop traces Emit paths now call snapshot_durable and propagate journal errors instead of empty maps. Training success requires a retained compute trace; artifact fingerprints hash file bytes. Co-authored-by: Mathis --- Cargo.lock | 3 + bins/proof-challenge/src/main.rs | 9 +- .../0032_proof_submission_digest.sql | 9 + crates/proof-challenge/Cargo.toml | 3 + crates/proof-challenge/src/lib.rs | 225 +++++++++++++++-- crates/proof-research/src/flops.rs | 231 ++++++++++++++++++ crates/proof-research/src/lib.rs | 2 + crates/proof-store/src/durable.rs | 77 +++++- crates/proof-store/tests/durability.rs | 9 +- docs/COMPLETENESS.md | 8 +- docs/PROOF.md | 28 ++- docs/external-miner/proof.md | 11 +- eval/README.md | 17 +- eval/baselines/adamw.py | 3 +- eval/src/proof_eval/agent.py | 10 +- eval/src/proof_eval/cli.py | 32 ++- eval/src/proof_eval/compute_trace.py | 230 +++++++++++++++++ eval/src/proof_eval/harness.py | 43 +++- eval/src/proof_eval/training_evidence.py | 65 +++++ eval/tests/test_compute_trace.py | 96 ++++++++ eval/tests/test_harness.py | 18 +- 21 files changed, 1064 insertions(+), 65 deletions(-) create mode 100644 crates/db/migrations/0032_proof_submission_digest.sql create mode 100644 crates/proof-research/src/flops.rs create mode 100644 eval/src/proof_eval/compute_trace.py create mode 100644 eval/src/proof_eval/training_evidence.py create mode 100644 eval/tests/test_compute_trace.py diff --git a/Cargo.lock b/Cargo.lock index 901480e71..30efa1280 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3595,6 +3595,7 @@ dependencies = [ "bundle", "challenge-common", "crypto", + "db", "hex", "proof-eval", "proof-http", @@ -3602,6 +3603,8 @@ dependencies = [ "proof-store", "proof-task", "serde_json", + "thiserror 2.0.19", + "tokio", ] [[package]] diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 7bb0a3dc6..52903a1d5 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -210,8 +210,13 @@ async fn open_store(path: Option<&Path>) -> Result { .with_journal(proof_store::durable::DurableJournal::new(pool)) .await .map_err(|_| "store database is not migrated or lacks restricted privileges")?; - let reloaded = store.list().map_or(0, |rows| rows.len()); - tracing::info!(submissions = reloaded, "durable store journal attached"); + let reloaded = store.list_durable().await.map_err(|_| { + "store journal snapshot failed; refuse to boot on a silent empty reload" + })?; + tracing::info!( + submissions = reloaded.len(), + "durable store journal attached" + ); store } }) diff --git a/crates/db/migrations/0032_proof_submission_digest.sql b/crates/db/migrations/0032_proof_submission_digest.sql new file mode 100644 index 000000000..9e2e96292 --- /dev/null +++ b/crates/db/migrations/0032_proof_submission_digest.sql @@ -0,0 +1,9 @@ +-- Frozen submission digest is the retry identity. +-- +-- 0031 on checkpoint/proof-production-readiness-20260907 already creates +-- `proof_submission_id_seq`. Do not replay or edit 0031 on an already-migrated +-- env: sqlx records it as applied. A pre-squash 0031 that lacked the sequence +-- needs a manual `CREATE SEQUENCE IF NOT EXISTS` repair, not a 0031 rewrite. +CREATE UNIQUE INDEX proof_submission_freeze + ON proof_submission ((document->>'submission_digest')) + WHERE coalesce(document->>'submission_digest', '') <> ''; diff --git a/crates/proof-challenge/Cargo.toml b/crates/proof-challenge/Cargo.toml index 028f1fbd1..dc5028da0 100644 --- a/crates/proof-challenge/Cargo.toml +++ b/crates/proof-challenge/Cargo.toml @@ -18,9 +18,12 @@ proof-score = { path = "../proof-score" } proof-store = { path = "../proof-store" } proof-task = { path = "../proof-task" } serde_json = "1" +thiserror = "2" [dev-dependencies] crypto = { path = "../crypto" } +db = { path = "../db", features = ["testing"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints] workspace = true diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index d411c48dc..a07a40590 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -20,7 +20,7 @@ pub use proof_eval::{ EvalBackend, LiveScorer, }; pub use proof_http::{hash_admin_token, proof_router, AppState}; -pub use proof_store::{ArtifactManifest, MemoryStore}; +pub use proof_store::{ArtifactManifest, MemoryStore, StoreError}; pub use proof_task::{ HoldoutRecord, InferenceOffer, OfferError, ProofPin, TopicDocument, BASE_MODEL_FAMILY, CHALLENGE_ID, CHALLENGE_ID_BYTES as PROOF_ID_BYTES, SCORE_MAX as PROOF_SCORE_MAX, @@ -90,30 +90,110 @@ pub fn emit_epoch( emit_signed_leaf_set(secret, CHALLENGE_ID_BYTES, epoch, expected, &scores) } +/// Why a journaled emission could not sign a leaf set. +#[derive(Debug, thiserror::Error)] +pub enum EmitStoreError { + /// Durable snapshot or sync reader failed; never treat this as "nobody scored". + #[error("store: {0}")] + Store(#[from] StoreError), + /// D24 leaf signing failed. + #[error("leaf: {0}")] + Leaf(#[from] LeafEmitError), +} + /// Per-miner topic attempts currently in the store. -pub fn store_runs(store: &MemoryStore) -> BTreeMap> { +/// +/// Journaled stores refuse these sync readers. Call [`store_runs_durable`] +/// or read a [`MemoryStore::snapshot_durable`] first. Errors are not empty maps. +pub fn store_runs( + store: &MemoryStore, +) -> Result>, StoreError> { let mut out = BTreeMap::new(); - if let Ok(keys) = store.scored_hotkeys() { - for k in keys { - if let Ok(m) = store.miner_runs(&k) { - out.insert(k, m); - } - } + for k in store.scored_hotkeys()? { + out.insert(k.clone(), store.miner_runs(&k)?); } - out + Ok(out) } /// Per-miner topic lattices currently in the store (binary pass map). -pub fn store_scores(store: &MemoryStore) -> BTreeMap> { +/// +/// Journaled stores refuse these sync readers. Errors are not empty maps. +pub fn store_scores( + store: &MemoryStore, +) -> Result>, StoreError> { let mut out = BTreeMap::new(); - if let Ok(keys) = store.scored_hotkeys() { - for k in keys { - if let Ok(m) = store.miner_scores(&k) { - out.insert(k, m); - } + for k in store.scored_hotkeys()? { + out.insert(k.clone(), store.miner_scores(&k)?); + } + Ok(out) +} + +/// Sealed baselines currently in the store. +pub fn store_baselines( + store: &MemoryStore, +) -> Result, StoreError> { + let mut out = BTreeMap::new(); + for topic in store.topics()? { + if let Some(metrics) = store.baseline(&topic.id)? { + out.insert(topic.id, metrics); } } - out + Ok(out) +} + +/// Operator-crowned champion primaries currently in the store. +pub fn store_champions(store: &MemoryStore) -> Result, StoreError> { + let mut out = BTreeMap::new(); + for topic in store.topics()? { + if let Some(primary) = store.champion_primary(&topic)? { + out.insert(topic.id, primary); + } + } + Ok(out) +} + +/// Decode stored hex hotkeys. An invalid key is a store fault, not a skip. +pub fn runs_by_hotkey( + hex_runs: &BTreeMap>, +) -> Result>, StoreError> { + let mut out = BTreeMap::new(); + for (key, runs) in hex_runs { + let hotkey = parse_hotkey(key).ok_or_else(|| { + StoreError::Illegal(format!("stored miner hotkey is not 32 bytes: {key}")) + })?; + out.insert(hotkey, runs.clone()); + } + Ok(out) +} + +/// Refresh the journal snapshot, then read payout runs. Propagates backend errors. +pub async fn store_runs_durable( + store: &MemoryStore, +) -> Result>, StoreError> { + store_runs(&store.snapshot_durable().await?) +} + +/// Refresh the journal snapshot once, then sign the exact-E leaf set. +/// +/// A journal failure is an error, never an empty score map. Sync readers on +/// the live journaled store still refuse; this path always snapshots first. +pub async fn emit_epoch_from_store( + store: &MemoryStore, + secret: &[u8; 32], + epoch: u64, + expected: &BTreeSet, +) -> Result, EmitStoreError> { + let snap = store.snapshot_durable().await?; + let mut topics = Vec::new(); + for id in snap.open_ids(epoch)? { + topics.push(snap.topic(&id)?); + } + let sealed = store_baselines(&snap)?; + let champions = store_champions(&snap)?; + let per_miner = runs_by_hotkey(&store_runs(&snap)?)?; + Ok(emit_epoch( + secret, epoch, expected, &topics, &sealed, &champions, &per_miner, + )?) } /// Parse a 32-byte hex hotkey. @@ -296,4 +376,117 @@ mod tests { assert_ne!(PROOF_ID_BYTES, other); } } + + #[test] + fn store_readers_propagate_errors_instead_of_empty_maps() { + let store = MemoryStore::new(); + store + .put_topic(discovery_topic("adamw-beater-v0")) + .expect("topic"); + store + .record_topic_run(&hex::encode([1u8; 32]), "adamw-beater-v0", pass(2.5, "d2")) + .expect("run"); + assert_eq!(store_runs(&store).expect("runs").len(), 1); + assert_eq!(store_scores(&store).expect("scores").len(), 1); + assert!( + runs_by_hotkey(&BTreeMap::from([("not-a-hotkey".into(), BTreeMap::new())])).is_err() + ); + } + + #[tokio::test] + async fn emit_epoch_from_store_snapshots_memory_and_covers_e() { + let store = MemoryStore::new(); + let topic = discovery_topic("adamw-beater-v0"); + store.put_topic(topic.clone()).expect("topic"); + store + .set_baseline("adamw-beater-v0", proof_score::flat_nll(3.0)) + .expect("sealed"); + let a = [1u8; 32]; + let b = [2u8; 32]; + store + .record_topic_run(&hex::encode(a), "adamw-beater-v0", pass(2.5, "d2")) + .expect("run"); + let expected: BTreeSet = [a, b].into_iter().collect(); + let leaves = emit_epoch_from_store(&store, &sk(), 1, &expected) + .await + .expect("emit"); + assert_eq!(leaves.len(), 2); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { .. } + )); + assert!(matches!( + leaves[&b].score_or_absence, + ScoreOrAbsence::NoScore { .. } + )); + } + + #[tokio::test] + #[ignore = "requires disposable DATABASE_URL"] + #[allow(clippy::too_many_lines)] + async fn emit_epoch_from_store_reads_the_journal_snapshot() { + let database = db::test_pool().await.expect("disposable postgres"); + let writer_pool = database.app_pool().await.expect("writer"); + let reader_pool = database.app_pool().await.expect("reader"); + let writer = MemoryStore::new() + .with_journal(proof_store::durable::DurableJournal::new(writer_pool)) + .await + .expect("journal"); + let reader = MemoryStore::new() + .with_journal(proof_store::durable::DurableJournal::new( + reader_pool.clone(), + )) + .await + .expect("journal"); + assert!( + store_runs(&writer).is_err(), + "journaled sync readers must refuse, not return empty" + ); + let topic = discovery_topic("adamw-beater-v0"); + writer.put_topic(topic.clone()).expect("topic"); + reader.put_topic(topic).expect("topic"); + writer + .set_baseline("adamw-beater-v0", proof_score::flat_nll(3.0)) + .expect("sealed"); + reader + .set_baseline("adamw-beater-v0", proof_score::flat_nll(3.0)) + .expect("sealed"); + let a = [1u8; 32]; + let row = proof_store::Submission { + id: String::new(), + topic_id: "adamw-beater-v0".into(), + miner_hotkey: hex::encode(a), + artifact_digest: "ab".repeat(32), + artifact_uri: None, + claim: "durable emit".into(), + declared_flops: 1, + architecture: String::new(), + inference_offer_id: String::new(), + config_commitment: String::new(), + manifest: proof_store::ArtifactManifest::default(), + nonce: "n".into(), + submission_digest: "cd".repeat(32), + state: proof_store::SubmissionState::AwaitingAdmin, + receipt_json: None, + verdict: None, + detail: None, + }; + writer + .finish_durable(row, pass(2.5, &"ab".repeat(32))) + .await + .expect("finish"); + let expected: BTreeSet = [a].into_iter().collect(); + let leaves = emit_epoch_from_store(&reader, &sk(), 1, &expected) + .await + .expect("reader must snapshot the writer's commit"); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { .. } + )); + reader_pool.close().await; + assert!(emit_epoch_from_store(&reader, &sk(), 1, &expected) + .await + .is_err()); + database.drop_schema().await.expect("drop"); + } } diff --git a/crates/proof-research/src/flops.rs b/crates/proof-research/src/flops.rs new file mode 100644 index 000000000..293076d2b --- /dev/null +++ b/crates/proof-research/src/flops.rs @@ -0,0 +1,231 @@ +//! Independent recomputation of a retained compute trace. +//! +//! This is the controller-side twin of `eval/src/proof_eval/compute_trace.py`. +//! A supplied `flops_used` is never trusted; only the op list is. Unlisted +//! compute-shaped ops refuse. A one-matmul fixture does not attest every recipe. + +use serde::Deserialize; + +use crate::ResearchError; + +const TRACE_SCHEMA: u32 = 1; + +/// One compacted op from a retained trace. +#[derive(Debug, Clone, Deserialize)] +pub struct ComputeOp { + pub op: String, + pub shapes: Vec>, + pub dtype: String, + pub count: u64, +} + +/// Retained op list. The `flops_used` field, if present, is ignored. +#[derive(Debug, Clone, Deserialize)] +pub struct ComputeTrace { + pub schema_version: u32, + pub ops: Vec, +} + +/// Recompute FLOPs from a retained trace. Never invents a number. +/// +/// # Errors +/// Empty/malformed trace, unlisted compute op, overflow, or no listed work. +pub fn flop_total(trace: &ComputeTrace) -> Result { + if trace.schema_version != TRACE_SCHEMA || trace.ops.is_empty() { + return Err(ResearchError::Evidence); + } + let mut total: u64 = 0; + let mut saw_compute = false; + for op in &trace.ops { + if op.dtype.trim().is_empty() || op.count == 0 { + return Err(ResearchError::Evidence); + } + let name = canonical_op(&op.op); + if looks_compute(&name) && !listed(&name) { + return Err(ResearchError::Evidence); + } + let flops = op_flops(&name, &op.shapes)? + .checked_mul(op.count) + .ok_or(ResearchError::Evidence)?; + if flops > 0 { + saw_compute = true; + } + total = total.checked_add(flops).ok_or(ResearchError::Evidence)?; + } + if !saw_compute { + return Err(ResearchError::Evidence); + } + Ok(total) +} + +fn canonical_op(name: &str) -> String { + let mut raw = name.trim().to_owned(); + if let Some(rest) = raw.strip_prefix("aten.") { + raw = format!("aten::{rest}"); + } + if let Some(stripped) = raw + .strip_suffix(".default") + .or_else(|| raw.strip_suffix(".out")) + { + stripped.to_owned() + } else { + raw + } +} + +fn listed(name: &str) -> bool { + matches!( + name, + "aten::mm" + | "aten::addmm" + | "aten::bmm" + | "aten::baddbmm" + | "aten::scaled_dot_product_attention" + | "aten::_scaled_dot_product_attention_math" + ) +} + +fn looks_compute(name: &str) -> bool { + listed(name) + || name.ends_with("::mm") + || name + .rsplit_once('.') + .is_some_and(|(_, suffix)| suffix == "mm") + || [ + "matmul", + "addmm", + "baddbmm", + "addbmm", + "convolution", + "conv2d", + "conv1d", + "conv3d", + "conv_transpose", + "scaled_dot_product", + "flash_attention", + "cudnn_convolution", + "miopen", + ] + .iter() + .any(|marker| name.contains(marker)) +} + +fn op_flops(name: &str, shapes: &[Vec]) -> Result { + match name { + "aten::mm" | "aten::bmm" => { + let pair = shapes.iter().rev().take(2).collect::>(); + if pair.len() < 2 { + return Err(ResearchError::Evidence); + } + matmul(pair[1], pair[0]) + } + "aten::addmm" | "aten::baddbmm" => { + let pair = shapes.iter().rev().take(2).collect::>(); + if pair.len() < 2 || shapes.len() < 3 { + return Err(ResearchError::Evidence); + } + matmul(pair[1], pair[0]) + } + "aten::scaled_dot_product_attention" | "aten::_scaled_dot_product_attention_math" => { + if shapes.len() < 3 { + return Err(ResearchError::Evidence); + } + let q = &shapes[0]; + let k = &shapes[1]; + let v = &shapes[2]; + if k.len() < 2 || q.len() < 2 { + return Err(ResearchError::Evidence); + } + let mut kt = k[..k.len() - 2].to_vec(); + kt.push(k[k.len() - 1]); + kt.push(k[k.len() - 2]); + let mut scores = q[..q.len() - 2].to_vec(); + scores.push(q[q.len() - 2]); + scores.push(k[k.len() - 2]); + let qk = matmul(q, &kt)?; + let av = matmul(&scores, v)?; + qk.checked_add(av).ok_or(ResearchError::Evidence) + } + _ if looks_compute(name) => Err(ResearchError::Evidence), + _ => Ok(0), + } +} + +fn matmul(left: &[i64], right: &[i64]) -> Result { + if left.len() < 2 || right.len() < 2 || left[left.len() - 1] != right[right.len() - 2] { + return Err(ResearchError::Evidence); + } + let left_batch = &left[..left.len() - 2]; + let right_batch = &right[..right.len() - 2]; + let width = left_batch.len().max(right_batch.len()); + let pad = |dims: &[i64], width: usize| -> Vec { + let mut out = vec![1; width.saturating_sub(dims.len())]; + out.extend_from_slice(dims); + out + }; + let padded_left = pad(left_batch, width); + let padded_right = pad(right_batch, width); + let mut batch: u64 = 1; + for (lhs, rhs) in padded_left.iter().zip(padded_right.iter()) { + if *lhs != *rhs && *lhs != 1 && *rhs != 1 { + return Err(ResearchError::Evidence); + } + let dim = u64::try_from((*lhs).max(*rhs)).map_err(|_| ResearchError::Evidence)?; + batch = batch.checked_mul(dim).ok_or(ResearchError::Evidence)?; + } + let rows = u64::try_from(left[left.len() - 2]).map_err(|_| ResearchError::Evidence)?; + let inner = u64::try_from(left[left.len() - 1]).map_err(|_| ResearchError::Evidence)?; + let cols = u64::try_from(right[right.len() - 1]).map_err(|_| ResearchError::Evidence)?; + batch + .checked_mul(2) + .and_then(|v| v.checked_mul(rows)) + .and_then(|v| v.checked_mul(inner)) + .and_then(|v| v.checked_mul(cols)) + .ok_or(ResearchError::Evidence) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn op(name: &str, shapes: &[&[i64]], count: u64) -> ComputeOp { + ComputeOp { + op: name.into(), + shapes: shapes.iter().map(|s| s.to_vec()).collect(), + dtype: "float32".into(), + count, + } + } + + #[test] + fn independent_recompute_matches_the_published_mm_cost() { + let trace = ComputeTrace { + schema_version: 1, + ops: vec![op("aten.mm.default", &[&[2, 3], &[3, 4]], 1)], + }; + assert_eq!(flop_total(&trace).expect("mm"), 48); + } + + #[test] + fn unlisted_compute_refuses_instead_of_undercounting() { + let trace = ComputeTrace { + schema_version: 1, + ops: vec![op("aten::convolution", &[&[1, 1, 3, 3], &[1, 1, 1, 1]], 1)], + }; + assert!(flop_total(&trace).is_err()); + } + + #[test] + fn empty_or_zero_work_refuses() { + assert!(flop_total(&ComputeTrace { + schema_version: 1, + ops: vec![], + }) + .is_err()); + let view = ComputeTrace { + schema_version: 1, + ops: vec![op("aten::view", &[&[2, 3]], 1)], + }; + assert!(flop_total(&view).is_err()); + } +} diff --git a/crates/proof-research/src/lib.rs b/crates/proof-research/src/lib.rs index a13daa2a1..f84bb553f 100644 --- a/crates/proof-research/src/lib.rs +++ b/crates/proof-research/src/lib.rs @@ -4,11 +4,13 @@ #![forbid(unsafe_code)] mod corpus; +mod flops; mod publication; mod science; mod store; pub use corpus::*; +pub use flops::{flop_total, ComputeOp, ComputeTrace}; pub use publication::*; pub use science::*; pub use store::*; diff --git a/crates/proof-store/src/durable.rs b/crates/proof-store/src/durable.rs index 5a128321d..b969485df 100644 --- a/crates/proof-store/src/durable.rs +++ b/crates/proof-store/src/durable.rs @@ -28,6 +28,7 @@ impl DurableJournal { /// Create a submission, or update only its mutable terminal fields, and /// record its payout run in the same transaction. + #[allow(clippy::too_many_lines)] pub(crate) async fn commit( &self, mut row: Submission, @@ -61,6 +62,10 @@ impl DurableJournal { if decoded.verdict != row.verdict { return Err(StoreError::Illegal("non-finite submission metric".into())); } + sqlx::query("SAVEPOINT proof_sub_insert") + .execute(&mut *tx) + .await + .map_err(|_| StoreError::Backend)?; let result = if update { sqlx::query( "UPDATE proof_submission SET document = $5, updated_at = now() \ @@ -76,14 +81,40 @@ impl DurableJournal { ) }.bind(&row.id).bind(&row.topic_id).bind(&row.miner_hotkey) .bind(&row.artifact_digest).bind(document) - .execute(&mut *tx).await.map_err(|_| StoreError::Backend)?; - if result.rows_affected() == 1 { - break; - } - if !generated { - return Err(StoreError::Illegal( - "duplicate id or immutable identity mismatch".into(), - )); + .execute(&mut *tx).await; + match result { + Ok(done) if done.rows_affected() == 1 => { + sqlx::query("RELEASE SAVEPOINT proof_sub_insert") + .execute(&mut *tx) + .await + .map_err(|_| StoreError::Backend)?; + break; + } + Ok(_) => { + sqlx::query("ROLLBACK TO SAVEPOINT proof_sub_insert") + .execute(&mut *tx) + .await + .map_err(|_| StoreError::Backend)?; + if !generated { + return Err(StoreError::Illegal( + "duplicate id or immutable identity mismatch".into(), + )); + } + } + Err(err) if unique_violation(&err) => { + sqlx::query("ROLLBACK TO SAVEPOINT proof_sub_insert") + .execute(&mut *tx) + .await + .map_err(|_| StoreError::Backend)?; + let existing = + Self::existing_by_digest(&mut tx, &row.submission_digest).await?; + if same_freeze(&existing, &row) { + row.id = existing.id; + break; + } + return Err(StoreError::Illegal("duplicate submission digest".into())); + } + Err(_) => return Err(StoreError::Backend), } } { @@ -109,6 +140,23 @@ impl DurableJournal { Ok(row) } + async fn existing_by_digest( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + digest: &str, + ) -> Result { + let row = sqlx::query( + "SELECT document FROM proof_submission \ + WHERE document->>'submission_digest' = $1", + ) + .bind(digest) + .fetch_optional(&mut **tx) + .await + .map_err(|_| StoreError::Backend)? + .ok_or(StoreError::Backend)?; + let value: serde_json::Value = row.try_get("document").map_err(|_| StoreError::Backend)?; + serde_json::from_value(value).map_err(|_| StoreError::Backend) + } + /// Read submissions and payouts from one MVCC snapshot. pub async fn snapshot( &self, @@ -181,3 +229,16 @@ impl DurableJournal { .collect() } } + +fn unique_violation(err: &sqlx::Error) -> bool { + err.as_database_error() + .and_then(sqlx::error::DatabaseError::code) + .is_some_and(|code| code == "23505") +} + +fn same_freeze(existing: &Submission, row: &Submission) -> bool { + existing.miner_hotkey == row.miner_hotkey + && existing.topic_id == row.topic_id + && existing.artifact_digest == row.artifact_digest + && existing.submission_digest == row.submission_digest +} diff --git a/crates/proof-store/tests/durability.rs b/crates/proof-store/tests/durability.rs index ab067a1bd..10bc21fd6 100644 --- a/crates/proof-store/tests/durability.rs +++ b/crates/proof-store/tests/durability.rs @@ -5,6 +5,7 @@ use proof_score::MinerTopicRun; use proof_store::{durable::DurableJournal, MemoryStore, Submission, SubmissionState}; +use sha2::{Digest, Sha256}; fn submission(id: &str, hotkey: &str, topic: &str) -> Submission { Submission { @@ -20,7 +21,7 @@ fn submission(id: &str, hotkey: &str, topic: &str) -> Submission { config_commitment: "c".repeat(64), manifest: proof_store::ArtifactManifest::default(), nonce: "n".into(), - submission_digest: "d".repeat(64), + submission_digest: hex::encode(Sha256::digest(format!("{id}|{hotkey}|{topic}").as_bytes())), state: SubmissionState::AwaitingAdmin, receipt_json: None, verdict: None, @@ -180,6 +181,12 @@ async fn multi_instance_atomicity_identity_and_cancellation() { .await .unwrap(); assert_eq!(reload.list_durable().await.unwrap().len(), 3); + let again = a + .finish_durable(submission("", "cancelled", "topic"), run()) + .await + .unwrap(); + assert_eq!(again.id, id); + assert_eq!(reload.list_durable().await.unwrap().len(), 3); pool.close().await; assert!(a.get_durable(&id).await.is_err()); assert!(a diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index c4ef2ab2b..74627b851 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -84,12 +84,12 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. | | Live harvest | **done** | `crates/proof-harvest` over `harvest-pod`; `PROOF_FORCE_SIM` is local-only. | | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | -| V1 automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | -| Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement; clean static inspection now raises `ContractError` absent agent reproduction and verified FLOP evidence, and forbidden fabric rejects. CLI gates before judge/model calls with no successful metrics. General reproduction and agent-led accounting are not implemented; accounting must fit the experiment with retained reproducible evidence verified by the controller, not a universal formula or model assertion. | -| V1 research persistence | **partial, unproven** | The default v1 store is in memory. The optional SQL journal now passes fresh workspace durability tests (2/2) after fixing embedded-migration tracking. Async writes persist SQL before memory and reject NaN/infinity. Production durability remains unproven: cross-process ID collisions/upserts, separate submission/score transactions, synchronous bypass and cancellation-induced memory lag remain. Public HTTP records are not a durable artifact archive. | +| V1 automatic emission | **lib-only** | `emit_epoch_from_store` now calls `snapshot_durable().await?` before signing and propagates journal errors. `bins/proof-challenge` still does not run an emission loop; HTTP epoch stays `0`. Do not infer payments from `can_score`. Proof/Bounty remain **8000/2000 bps**. | +| Autonomous research judge | **partial** | Python `judge.py` still only acknowledges. Forbidden fabric rejects before measure/judge. Score/baseline refuse without controller training evidence whose retained trace independently recomputes; `adamw.py` is a param lock, not training. Holdout measurement emits a listed-op compute trace (not a universal formula, not hostile-proof of flash/custom kernels). General recipe reproduction is not implemented. | +| V1 research persistence | **partial** | Optional SQL journal: sequence ids (0031), unique frozen digest (0032), atomic finish/update, repeatable-read `snapshot_durable()`, HTTP 503 on backend failure. Sync journal readers refuse instead of empty maps. Still open: network loss during COMMIT; concurrent topic-run last-writer-wins. Public HTTP records are not a durable artifact archive. | | Atlas experiment persistence / API | **partial, opt-in API** | [`proof-autonomy-pg`](../crates/proof-autonomy-pg/README.md) persists commands, consent/nonces, quotas, revisions, fences, resources and observations. Canonical shared migrations are **0020–0028**; `db::test_pool` matches append-only runtime events and column grants. Restricted DB configuration enables v2 routes in `proof-challenge`, not experiment startup. | | Experiment worker | **partial, library** | `proof-worker` implements independent model-free cleanup, strict adoption, quote refresh and durable run identity. Same-fence repeat invocation is refused; the original DB runtime deadline is enforced even if an agent ignores stop. Shutdown aborts lease acquisition and drops suspended operation/heartbeat futures before DB bookkeeping. No strict live Lium, credential or quote adapter is wired. | -| V2 local execution / retained science | **partial, local tests** | `proof-executor` runs paired actual CPU scripts in digest-pinned Docker, retaining stdout/stderr/exit/wall and failures. `proof-measure` adds a trusted observer (optionally wired in `proof-experiment`; `DockerObserver`: pinned image, no network by default, read-only rootfs, read-only holdout bind, bounded logs, per-run anti-replay); `collect` still returns `UnobservedMeasurements` whenever FLOPs are unmeasured, and the default `NoObserver` fails closed with zero runs dispatched. Only a test observer image is exercised: the real `proof-eval` image reports no independent FLOPs. Optional `JudgeEgress` demonstrated a synthetic probe with a real completion and four sampled blocked escapes, not universal isolation or science. The eval helper supports explicit `PROOF_JUDGE_PROXY=1` without Authorization only for the exact alias `http://proof-judge:8080/v1` and `chat/completions`; direct mode still requires a key; arbitrary allowed payloads and artifact code mean proxy confidentiality/integrity are not proven. `proof-research` admission tests use synthetic observations; v1 in-memory records are not migrated. The stock W&B SDK stays unusable (v0.28.0 runtime/environment telemetry exceeds the seven-field allowlist); `proof-wandb` uploads the allowlisted record over direct GraphQL but is wired into no binary and has never contacted W&B. | +| V2 local execution / retained science | **partial, local tests** | `proof-executor` runs paired actual CPU scripts in digest-pinned Docker, retaining stdout/stderr/exit/wall and failures. `proof-measure` adds a trusted observer (optionally wired in `proof-experiment`; `DockerObserver`: pinned image, no network by default, read-only rootfs, read-only holdout bind, bounded logs, per-run anti-replay); `collect` still returns `UnobservedMeasurements` whenever FLOPs are unmeasured, and the default `NoObserver` fails closed with zero runs dispatched. Only a test observer image is exercised: the real `proof-eval` image can now emit a listed-op eval trace, which is not training-FLOP evidence for an arbitrary recipe. Optional `JudgeEgress` demonstrated a synthetic probe with a real completion and four sampled blocked escapes, not universal isolation or science. The eval helper supports explicit `PROOF_JUDGE_PROXY=1` without Authorization only for the exact alias `http://proof-judge:8080/v1` and `chat/completions`; direct mode still requires a key; arbitrary allowed payloads and artifact code mean proxy confidentiality/integrity are not proven. `proof-research` admission tests use synthetic observations; v1 in-memory records are not migrated. The stock W&B SDK stays unusable (v0.28.0 runtime/environment telemetry exceeds the seven-field allowlist); `proof-wandb` uploads the allowlisted record over direct GraphQL but is wired into no binary and has never contacted W&B. | | Private headless runtime | **partial, scoped tests** | Shared `HeadlessProcess` drives the real `CortexRuntime` with host-only config, attempt sockets and original identity/deadline/budget journal. Seventeen supervision tests pass: inherited pipes, leader exit, five-second TERM→KILL grace, future-drop/reaper behavior and `setsid` escape. Optional `headless.pid_namespace` (`unshare --pid --kill-child`) kills escaped descendants; PID containment only. One authorized Astra/kernel/controller synthetic test passed (7.23 s), not science/cost evidence; it is ignored and requires `CORTEX_TEST_HEADLESS_MODEL_CONFIG`. HTTPS by default; literal loopback HTTP requires `allowLoopbackHttp: true`, private `apiKeyFile`, no Factory fallback. | | Finalized Atlas rounds / scheduler | **partial, local tests** | `proof-rounds` freezes 360-block inputs/history and signed-byte outboxes; `proof-atlas-worker` schedules/reconciles rounds. Thirteen isolated scheduler tests passed, plus cancellation coverage preventing a blocking RPC completion after shutdown from freezing a new round. Canonical DB migrations passed. The ignored headless-delivery regression below covers the connected local path; deployed publication/admin-seal coordination and payment remain unverified. | | Atlas service (`bins/proof-atlas`) | **partial, opt-in binary** | `PROOF_ATLAS_CONFIG_FILE` / `--config` selects private operator configuration; raw/hex signer must match the pin. Private-file/signer/restricted-DB checks and three startup/shutdown regressions passed, including SIGTERM while waiting for finality; blocking RPC initialization uses `spawn_blocking`. `--check` does not initialize/validate TS model/provider config, image presence, network/chain/gateway compatibility or deployment. Normal mode starts only the scheduler/publisher, never rental, sealer or chain submit; see the [operator contract](runbooks/proof-autonomy-local.md#separate-atlas-operator-contract). | diff --git a/docs/PROOF.md b/docs/PROOF.md index c7ed3b407..c3abb60f7 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -7,13 +7,27 @@ Read the [overview](OVERVIEW.md) for the purpose and the **Implementation limits:** the Python judge currently performs an authenticated acknowledgement request and static checks, not the paper's autonomous investigation -and arbitrary recipe reproduction. Clean static inspection now raises `ContractError` -without agent reproduction and verified FLOP evidence; forbidden fabric rejects. -The CLI gates before judge/model calls and produces no successful metrics. -Agent-led accounting is not implemented: the agent must determine experiment-appropriate accounting, retain reproducible evidence, and have that evidence verified by the controller. Neither a universal formula nor an arbitrary model assertion is sufficient. -The default v1 store is in memory. The optional SQL journal now passes fresh workspace durability tests (2/2) after fixing embedded-migration tracking. Async writes persist SQL before memory and reject NaN/infinity. Production durability remains unproven: cross-process ID collisions/upserts, separate submission/score transactions, synchronous bypass and cancellation-induced memory lag remain. -The v1 service does not run an automatic reward-leaf emitter. Payout/signing helpers exist, but -readiness checks alone do not establish a complete research-to-payment path. +and arbitrary recipe reproduction. Forbidden fabric still rejects before measure +or judge. `proof-eval score` still refuses success unless a controller-supplied +training-evidence file carries a retained compute trace whose FLOPs independently +recompute; miner-declared numbers and `eval/baselines/adamw.py` (a parameter lock, +not executable training) are not that envelope. Holdout measurement now records a +listed-op trace (`mm` / `addmm` / `bmm` / math SDPA only). Unlisted compute-shaped +ops refuse rather than undercount. `torch.profiler` / `FlopCounterMode` are not +the source of truth. A one-matmul fixture does not attest every recipe. +`artifact_fingerprint` hashes file bytes, not the directory path. +The default v1 store is in memory. The optional SQL journal snapshots with +`snapshot_durable()` on every `emit_epoch_from_store` and on HTTP list/get; +sync readers refuse in journal mode instead of returning empty maps. Sequence +ids and a unique frozen `submission_digest` are in migrations **0031–0032**. +0031 on this checkpoint already creates `proof_submission_id_seq` — do not +replay it on an already-migrated env. Still untested: network loss during +COMMIT. Concurrent `proof_topic_run` updates remain last-writer-wins. +The v1 binary still does not run an automatic reward-leaf emitter. Payout +helpers exist, but readiness checks alone do not establish a complete +research-to-payment path. Journal rows are JSON, not artifact blobs: +`proof-measure` caps observer tars at 64 MiB; `proof-research` retained +artifacts cap at 16 MiB total / 1 MiB each. The rules below describe the current interfaces and scoring functions, not a claim that these gaps are closed. diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index dfed11d29..0a15d43cf 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -5,11 +5,12 @@ Challenge id is `proof`. The two configured challenges are `bounty` (**2000 bps**) and `proof` (**8000 bps**), a 20/80 allocation. -**Implementation warning:** Proof's v1 Python judge is partial. The default v1 store is in memory. The optional SQL journal now passes fresh workspace durability tests (2/2) after fixing embedded-migration tracking. Async writes persist SQL before memory and reject NaN/infinity. Production durability remains unproven: cross-process ID collisions/upserts, separate submission/score transactions, synchronous bypass and cancellation-induced memory lag remain. -The service does not yet drive automatic reward-leaf emission. Clean static -inspection raises `ContractError` absent agent reproduction and verified FLOP -evidence; forbidden fabric rejects. CLI gates before judge/model calls and -produces no successful metrics. Agent-led accounting is not implemented: the agent must determine experiment-appropriate accounting, retain reproducible evidence, and have that evidence verified by the controller. Neither a universal formula nor an arbitrary model assertion is sufficient. +**Implementation warning:** Proof's v1 Python judge is partial. The default v1 store is in memory. The optional SQL journal snapshots before payout emission and refuses sync empty-map reads; sequence ids and a unique frozen digest live in migrations 0031–0032. Production durability remains unproven: network loss during COMMIT and concurrent topic-run last-writer-wins are still open. +The service does not yet drive automatic reward-leaf emission. Forbidden fabric +rejects before measure/judge. Score still refuses without controller-verified +training/FLOP evidence (a retained op trace, independently recomputed — not a +miner-declared number and not `adamw.py`). A tiny listed-op fixture does not +attest every recipe. Artifact fingerprints hash file bytes, not paths. Do not spend compute on the assumption that `can_score` proves the complete research-to-payment path. Read the [paper-to-code comparison](../WHITEPAPER.md#proposal-versus-current-code) and diff --git a/eval/README.md b/eval/README.md index 51830f9eb..1ccaf89f4 100644 --- a/eval/README.md +++ b/eval/README.md @@ -30,14 +30,15 @@ This is not yet the autonomous research judge described in the whitepaper. `judge.py` requests an authenticated acknowledgement and does not parse a research verdict. `agent.py` performs static text checks: clean inspection raises `ContractError` absent agent reproduction and verified FLOP evidence; forbidden -fabric yields a rejection. `cli.py` gates before judge/model calls and produces -no successful metrics. General recipe reproduction is not implemented. - -Agent-led accounting is not implemented: the agent must determine experiment-appropriate accounting, retain reproducible evidence, and have that evidence verified by the controller. Neither a universal formula nor an arbitrary model assertion is sufficient. - -`harness.py` has model-loss and optional-throughput measurement code, but no -independent FLOP accounting; the unsupported `2ND` estimate is removed. Custom -and canary metrics remain unset. Optional observer/judge-egress wiring exists in +fabric yields a rejection. `cli.py` still refuses success unless +`PROOF_TRAINING_EVIDENCE_FILE` carries a retained compute trace whose FLOPs +independently recompute. General recipe reproduction is not implemented. +`adamw.py` is a parameter lock, not executable training. + +Holdout measurement records a listed-op compute trace (`mm` / `addmm` / `bmm` / +math SDPA). Unlisted compute-shaped ops refuse. This is not a universal formula +and not hostile-proof of flash or custom kernels. `artifact_fingerprint` hashes +file bytes. Custom and canary metrics remain unset. Optional observer/judge-egress wiring exists in `proof-experiment`, with explicit `PROOF_JUDGE_PROXY=1` helper transport without Authorization only for the exact alias `http://proof-judge:8080/v1` and `chat/completions`; direct mode still requires a key. A synthetic probe obtained a real completion and blocked four sampled escapes; arbitrary allowed payloads and artifact code leave confidentiality/integrity diff --git a/eval/baselines/adamw.py b/eval/baselines/adamw.py index 0cf6daa07..c32bab7f7 100644 --- a/eval/baselines/adamw.py +++ b/eval/baselines/adamw.py @@ -1,5 +1,6 @@ -"""Locked AdamW recipe the image seals. +"""Locked AdamW *parameter* recipe the image seals. +This file is a dict, not executable training, and is not training-evidence. script_sha256 on an open topic must be SHA-256 of these exact bytes. A topic that claims `optimizer = adamw` with a different script is a strawman and a publish reject on the control plane. diff --git a/eval/src/proof_eval/agent.py b/eval/src/proof_eval/agent.py index 876abaf4c..3848da8fe 100644 --- a/eval/src/proof_eval/agent.py +++ b/eval/src/proof_eval/agent.py @@ -18,7 +18,7 @@ _FAST = re.compile(r"\b(ncclNetIb|NCCL_IB_DISABLE\s*=\s*0|ncclNvls)\b", re.I) -def inspect(request: HarvestRequest, recipe_text: str) -> dict[str, Any]: +def fabric_cheats(request: HarvestRequest, recipe_text: str) -> list[str]: cheats: list[str] = [] hay = f"{request.claim}\n{recipe_text}" c: Constraints = request.constraints @@ -28,7 +28,13 @@ def inspect(request: HarvestRequest, recipe_text: str) -> dict[str, Any]: cheats.append("other") if c.no_nccl_fast_fabric and _FAST.search(hay): cheats.append("other") - # ponytail: static checks only; add agent reproduction with verified compute evidence. + return cheats + + +def inspect(request: HarvestRequest, recipe_text: str) -> dict[str, Any]: + cheats = fabric_cheats(request, recipe_text) + # Static text cannot authorize a pass. Verified traces + training evidence + # are assembled in the CLI after measurement, never invented here. if not cheats: raise ContractError("agent reproduction and verified FLOP evidence are unavailable") return { diff --git a/eval/src/proof_eval/cli.py b/eval/src/proof_eval/cli.py index 41424c19d..9982b32ef 100644 --- a/eval/src/proof_eval/cli.py +++ b/eval/src/proof_eval/cli.py @@ -30,8 +30,10 @@ from .fabric import selftest as fabric_selftest from .judge import require_judge from .request import read_request -from .agent import inspect +from .agent import fabric_cheats, inspect +from .compute_trace import verify_trace from .harness import measure, require_runtime +from .training_evidence import load_training_evidence EXIT_REFUSED = 2 EXIT_ERROR = 1 @@ -108,18 +110,32 @@ def _score(request_path: Path, out: Path, *, baseline: bool) -> int: recipe = request.claim if Path(ADAMW_SCRIPT).is_file(): recipe = f"{recipe}\n{Path(ADAMW_SCRIPT).read_text(encoding='utf-8')}" - agent = inspect(request, recipe) - if not agent["reproduced"]: - raise ContractError(agent["rationale"]) + cheats = fabric_cheats(request, recipe) + if cheats: + raise ContractError(inspect(request, recipe)["rationale"]) + # Training evidence is required before the model or judge is touched. + # adamw.py is a parameter lock, not this envelope. + evidence = load_training_evidence() require_judge(request) artifact_dir = os.environ.get("PROOF_ARTIFACT_DIR") or os.environ.get("PROOF_PROXY_MODEL_DIR") if baseline: artifact_dir = os.environ.get("PROOF_PROXY_MODEL_DIR") or artifact_dir harness = measure(request, artifact_dir) - harness = { - k: v - for k, v in harness.items() - if k != "artifact_fingerprint" + counted = verify_trace(harness.pop("compute_trace")) + if counted != harness.pop("eval_flops"): + raise ContractError("eval FLOPs do not match the retained trace") + agent = { + "verdict": "clean", + "reproduced": True, + "claim_holds_public": True, + "contamination": False, + "canary_hit": False, + "flops_used": evidence["flops_used"], + "flops_budget": request.flops_budget, + "cheat_codes": [], + "rationale": "controller-verified training trace; eval trace independently recomputed", + "topic_id": request.topic_id, + "family": request.family or "nll", } document = { "schema_version": PROOF_METRICS_SCHEMA, diff --git a/eval/src/proof_eval/compute_trace.py b/eval/src/proof_eval/compute_trace.py new file mode 100644 index 000000000..7aba328af --- /dev/null +++ b/eval/src/proof_eval/compute_trace.py @@ -0,0 +1,230 @@ +"""Controller-checkable compute traces. + +This is not a universal FLOP formula and not a hostile-proof of every kernel. +Listed ops have published costs; unlisted compute-shaped ops refuse rather +than undercount. `torch.profiler` and `FlopCounterMode` are not the source +of truth: the retained op list is, and the controller recomputes the same +total. A passing tiny matmul does not attest every training recipe. +""" + +from __future__ import annotations + +from collections import Counter +from contextlib import contextmanager, nullcontext +from typing import Any, Iterator + +from .contract import ContractError + +TRACE_SCHEMA = 1 +_U64 = (1 << 64) - 1 + +# Canonical names after stripping `.default` / `.out`. +_COST = frozenset( + { + "aten::mm", + "aten::addmm", + "aten::bmm", + "aten::baddbmm", + "aten::scaled_dot_product_attention", + "aten::_scaled_dot_product_attention_math", + } +) + +_UNLISTED_MARKERS = ( + "matmul", + "addmm", + "baddbmm", + "addbmm", + "convolution", + "conv2d", + "conv1d", + "conv3d", + "conv_transpose", + "scaled_dot_product", + "flash_attention", + "cudnn_convolution", + "miopen", +) + + +def canonical_op(name: str) -> str: + raw = name.strip() + if raw.startswith("aten."): + raw = "aten::" + raw[5:] + if raw.endswith(".default") or raw.endswith(".out"): + raw = raw.rsplit(".", 1)[0] + return raw + + +def _looks_compute(name: str) -> bool: + n = name.lower() + if n in _COST or n.endswith("::mm") or n.endswith(".mm"): + return True + return any(marker in n for marker in _UNLISTED_MARKERS) + + +def _checked_mul(*values: int) -> int: + total = 1 + for value in values: + if value < 0: + raise ContractError("negative trace shape") + total *= value + if total > _U64: + raise ContractError("trace flop overflow") + return total + + +def _checked_add(left: int, right: int) -> int: + total = left + right + if total > _U64: + raise ContractError("trace flop overflow") + return total + + +def _matmul(a: list[int], b: list[int]) -> int: + if len(a) < 2 or len(b) < 2 or a[-1] != b[-2]: + raise ContractError("matmul shape mismatch") + a_batch, b_batch = a[:-2], b[:-2] + n = max(len(a_batch), len(b_batch)) + a_batch = [1] * (n - len(a_batch)) + a_batch + b_batch = [1] * (n - len(b_batch)) + b_batch + batch = 1 + for left, right in zip(a_batch, b_batch, strict=True): + if left != right and left != 1 and right != 1: + raise ContractError("matmul batch mismatch") + batch = _checked_mul(batch, max(left, right)) + return _checked_mul(2, batch, a[-2], a[-1], b[-1]) + + +def op_flops(op: str, shapes: list[list[int]]) -> int: + name = canonical_op(op) + if name == "aten::mm": + if len(shapes) < 2: + raise ContractError("mm missing operands") + return _matmul(shapes[-2], shapes[-1]) + if name == "aten::addmm": + if len(shapes) < 3: + raise ContractError("addmm missing operands") + return _matmul(shapes[-2], shapes[-1]) + if name == "aten::bmm": + if len(shapes) < 2: + raise ContractError("bmm missing operands") + return _matmul(shapes[-2], shapes[-1]) + if name == "aten::baddbmm": + if len(shapes) < 3: + raise ContractError("baddbmm missing operands") + return _matmul(shapes[-2], shapes[-1]) + if name in { + "aten::scaled_dot_product_attention", + "aten::_scaled_dot_product_attention_math", + }: + if len(shapes) < 3 or any(len(item) < 2 for item in shapes[:3]): + raise ContractError("sdpa missing operands") + q, k, v = shapes[0], shapes[1], shapes[2] + # Math-backend equivalent: QK^T + AV. Flash/efficient kernels refuse. + qk = _matmul(q, k[:-2] + [k[-1], k[-2]]) + av = _matmul(q[:-2] + [q[-2], k[-2]], v) + return _checked_add(qk, av) + if _looks_compute(name): + raise ContractError(f"unlisted compute op: {name}") + return 0 + + +def verify_trace(trace: dict[str, Any]) -> int: + """Recompute FLOPs from a retained op list. Never trust a supplied total.""" + if not isinstance(trace, dict) or trace.get("schema_version") != TRACE_SCHEMA: + raise ContractError("compute trace schema is not 1") + ops = trace.get("ops") + if not isinstance(ops, list) or not ops: + raise ContractError("compute trace is empty") + total = 0 + saw_compute = False + for item in ops: + if not isinstance(item, dict): + raise ContractError("compute trace op is not an object") + op = item.get("op") + shapes = item.get("shapes") + count = item.get("count") + dtype = item.get("dtype") + if not isinstance(op, str) or not isinstance(dtype, str) or not dtype: + raise ContractError("compute trace op is malformed") + if not isinstance(shapes, list) or not all( + isinstance(shape, list) and all(isinstance(dim, int) for dim in shape) + for shape in shapes + ): + raise ContractError("compute trace shapes are malformed") + if not isinstance(count, int) or count < 1: + raise ContractError("compute trace count is malformed") + name = canonical_op(op) + if _looks_compute(name) and name not in _COST: + raise ContractError(f"unlisted compute op: {name}") + flops = _checked_mul(op_flops(name, shapes), count) + if flops: + saw_compute = True + total = _checked_add(total, flops) + if not saw_compute: + raise ContractError("compute trace has no listed matmul/attention work") + return total + + +class ComputeTrace: + def __init__(self) -> None: + self._ops: Counter[tuple[str, tuple[tuple[int, ...], ...], str]] = Counter() + + def add(self, op: str, shapes: list[list[int]], dtype: str) -> None: + key = (canonical_op(op), tuple(tuple(shape) for shape in shapes), dtype) + self._ops[key] += 1 + + def to_dict(self) -> dict[str, Any]: + ops = [ + { + "op": op, + "shapes": [list(shape) for shape in shapes], + "dtype": dtype, + "count": count, + } + for (op, shapes, dtype), count in sorted(self._ops.items()) + ] + return {"schema_version": TRACE_SCHEMA, "ops": ops} + + +@contextmanager +def collect_trace() -> Iterator[ComputeTrace]: + try: + import torch + from torch.utils._python_dispatch import TorchDispatchMode + except ImportError as exc: + raise ContractError(f"no compute-trace runtime: {exc}") from exc + + trace = ComputeTrace() + + class _Mode(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): # type: ignore[no-untyped-def] + kwargs = kwargs or {} + result = func(*args, **kwargs) + name = getattr(func, "_name", None) or str(func) + shapes: list[list[int]] = [] + dtype = "" + for arg in args: + if hasattr(arg, "shape") and hasattr(arg, "dtype"): + shapes.append([int(dim) for dim in arg.shape]) + if not dtype: + dtype = str(arg.dtype).replace("torch.", "") + if dtype: + trace.add(str(name), shapes, dtype) + return result + + attention = nullcontext() + try: + from torch.nn.attention import SDPBackend, sdpa_kernel + + attention = sdpa_kernel(SDPBackend.MATH) + except Exception: # noqa: BLE001 + try: + attention = torch.backends.cuda.sdp_kernel( + enable_flash=False, enable_mem_efficient=False, enable_math=True + ) + except Exception: # noqa: BLE001 + attention = nullcontext() + with attention, _Mode(): + yield trace diff --git a/eval/src/proof_eval/harness.py b/eval/src/proof_eval/harness.py index 673778517..5a980c617 100644 --- a/eval/src/proof_eval/harness.py +++ b/eval/src/proof_eval/harness.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Any +from .compute_trace import collect_trace, verify_trace from .contract import ContractError from .request import HarvestRequest @@ -71,9 +72,42 @@ def check_config(value: Any) -> None: raise ContractError("local safetensors weights are required") except (OSError, ValueError, TypeError, AttributeError) as exc: raise ContractError(f"invalid artifact: {exc}") from exc + # Content hashes, never the directory path. Re-read each shard. + artifact_fingerprint(root) return root +def artifact_fingerprint(root: Path) -> str: + """SHA-256 of sorted (name, file-bytes-digest) pairs. Path is not hashed.""" + digest = hashlib.sha256() + hashed: dict[str, str] = {} + try: + for path in sorted(root.iterdir(), key=lambda item: item.name): + if path.is_symlink() or not path.is_file(): + raise ContractError("artifact entries must be regular files, not links/directories") + body = path.read_bytes() + if not body: + raise ContractError(f"empty artifact file: {path.name}") + shard = hashlib.sha256(body).hexdigest() + hashed[path.name] = shard + digest.update(path.name.encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(shard)) + index = root / "model.safetensors.index.json" + if index.is_file(): + weights = json.loads(index.read_text()).get("weight_map") + if not isinstance(weights, dict) or not weights: + raise ContractError("missing safetensors weight_map") + for shard in weights.values(): + if not isinstance(shard, str) or shard not in hashed: + raise ContractError("safetensors shard was not hashed") + elif "model.safetensors" not in hashed: + raise ContractError("local safetensors weights are required") + except (OSError, ValueError, TypeError, AttributeError) as exc: + raise ContractError(f"invalid artifact: {exc}") from exc + return digest.hexdigest() + + def require_runtime() -> None: try: import torch # noqa: F401 @@ -115,6 +149,7 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any] forbidden here: they would be a sim fallback inside the live image. """ artifact = _artifact_path(artifact_dir) + fingerprint = artifact_fingerprint(artifact) texts = [] for rec in request.holdout: split = rec.get("split") or rec.get("task") @@ -150,7 +185,7 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any] import time t0 = time.perf_counter() - with torch.no_grad(): + with collect_trace() as trace, torch.no_grad(): for split, text in texts: enc = tok(text, return_tensors="pt", truncation=True, max_length=1024) enc = {k: v.to(device) for k, v in enc.items()} @@ -163,6 +198,8 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any] split_nll[split].append(nll) nlls.append(nll) tokens += int(enc["input_ids"].numel()) + eval_trace = trace.to_dict() + eval_flops = verify_trace(eval_trace) wall = time.perf_counter() - t0 if not math.isfinite(wall) or wall <= 0: raise ContractError("invalid measurement duration") @@ -184,5 +221,7 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any] "wall_s": int(wall) if request.family == "throughput" else None, "custom_value": None, "canary_nll": None, - "artifact_fingerprint": hashlib.sha256(str(artifact).encode()).hexdigest()[:16], + "artifact_fingerprint": fingerprint, + "compute_trace": eval_trace, + "eval_flops": eval_flops, } diff --git a/eval/src/proof_eval/training_evidence.py b/eval/src/proof_eval/training_evidence.py new file mode 100644 index 000000000..4c54c52d8 --- /dev/null +++ b/eval/src/proof_eval/training_evidence.py @@ -0,0 +1,65 @@ +"""Controller-supplied training/FLOP evidence. + +Miner-declared FLOP numbers are never accepted. The envelope must carry a +retained compute trace; the controller (and this helper) recompute the total +from that list. Missing or mismatched evidence refuses success. + +`eval/baselines/adamw.py` is a parameter lock, not executable training, and +is not this envelope. A synthetic one-matmul fixture does not support every +recipe. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + +from .compute_trace import verify_trace +from .contract import ContractError + +_DIGEST = re.compile(r"^[0-9a-f]{64}$") + + +def load_training_evidence(path: str | Path | None = None) -> dict[str, Any]: + raw_path = path or os.environ.get("PROOF_TRAINING_EVIDENCE_FILE") + if not raw_path or not str(raw_path).strip(): + raise ContractError("training evidence missing; refuse scoring") + file = Path(raw_path) + if file.is_symlink() or not file.is_file(): + raise ContractError("training evidence must be a local regular file") + try: + data = json.loads(file.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ContractError(f"unreadable training evidence: {exc}") from exc + return verify_training_evidence(data) + + +def verify_training_evidence(data: Any) -> dict[str, Any]: + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise ContractError("training evidence schema is not 1") + script = data.get("script_digest") + log = data.get("log_digest") + seed = data.get("seed") + claimed = data.get("flops_used") + if not isinstance(script, str) or not _DIGEST.fullmatch(script): + raise ContractError("training evidence script_digest is malformed") + if not isinstance(log, str) or not _DIGEST.fullmatch(log): + raise ContractError("training evidence log_digest is malformed") + if not isinstance(seed, int) or seed < 0: + raise ContractError("training evidence seed is malformed") + if not isinstance(claimed, int) or claimed < 1: + raise ContractError("training evidence flops_used is missing") + counted = verify_trace(data.get("trace")) + if counted != claimed: + raise ContractError("training FLOPs do not match the retained trace") + return { + "schema_version": 1, + "script_digest": script, + "log_digest": log, + "seed": seed, + "flops_used": counted, + "trace": data["trace"], + } diff --git a/eval/tests/test_compute_trace.py b/eval/tests/test_compute_trace.py new file mode 100644 index 000000000..0bb97201a --- /dev/null +++ b/eval/tests/test_compute_trace.py @@ -0,0 +1,96 @@ +"""Independent FLOP recomputation. A tiny matmul does not attest every recipe.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from proof_eval.compute_trace import collect_trace, verify_trace +from proof_eval.contract import ContractError +from proof_eval.training_evidence import verify_training_evidence + + +def test_independent_recompute_matches_collector() -> None: + torch = pytest.importorskip("torch") + a = torch.randn(2, 3) + b = torch.randn(3, 4) + with collect_trace() as trace: + _ = a @ b + body = trace.to_dict() + assert verify_trace(body) == 48 + + +def test_unlisted_compute_refuses() -> None: + with pytest.raises(ContractError, match="unlisted"): + verify_trace( + { + "schema_version": 1, + "ops": [ + { + "op": "aten::convolution", + "shapes": [[1, 1, 3, 3], [1, 1, 1, 1]], + "dtype": "float32", + "count": 1, + } + ], + } + ) + + +def test_training_evidence_must_match_the_trace() -> None: + trace = { + "schema_version": 1, + "ops": [ + { + "op": "aten::mm", + "shapes": [[2, 3], [3, 4]], + "dtype": "float32", + "count": 1, + } + ], + } + ok = verify_training_evidence( + { + "schema_version": 1, + "script_digest": "ab" * 32, + "log_digest": "cd" * 32, + "seed": 1, + "flops_used": 48, + "trace": trace, + } + ) + assert ok["flops_used"] == 48 + with pytest.raises(ContractError, match="do not match"): + verify_training_evidence( + { + "schema_version": 1, + "script_digest": "ab" * 32, + "log_digest": "cd" * 32, + "seed": 1, + "flops_used": 99, + "trace": trace, + } + ) + with pytest.raises(ContractError, match="trace"): + verify_training_evidence( + { + "schema_version": 1, + "script_digest": "ab" * 32, + "log_digest": "cd" * 32, + "seed": 1, + "flops_used": 48, + } + ) + + +def test_missing_training_evidence_file_refuses(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from proof_eval.training_evidence import load_training_evidence + + monkeypatch.delenv("PROOF_TRAINING_EVIDENCE_FILE", raising=False) + with pytest.raises(ContractError, match="missing"): + load_training_evidence() + missing = tmp_path / "nope.json" + with pytest.raises(ContractError, match="regular file"): + load_training_evidence(missing) diff --git a/eval/tests/test_harness.py b/eval/tests/test_harness.py index da90102ca..3b4e92ebc 100644 --- a/eval/tests/test_harness.py +++ b/eval/tests/test_harness.py @@ -15,8 +15,9 @@ from tokenizers.pre_tokenizers import Whitespace from transformers import GPT2Config, GPT2LMHeadModel, PreTrainedTokenizerFast +from proof_eval.compute_trace import verify_trace from proof_eval.contract import ContractError -from proof_eval.harness import SCORED_SPLITS, measure +from proof_eval.harness import SCORED_SPLITS, artifact_fingerprint, measure class HarnessTest(unittest.TestCase): @@ -75,6 +76,21 @@ def test_real_nll_matches_independent_forward(self): self.assertGreater(result["tokens_per_sec"], 0) self.assertNotIn("clean", result) self.assertNotIn("reproduced", result) + self.assertEqual(result["artifact_fingerprint"], artifact_fingerprint(self.artifact)) + self.assertEqual(verify_trace(result["compute_trace"]), result["eval_flops"]) + self.assertGreater(result["eval_flops"], 0) + + def test_fingerprint_hashes_bytes_not_path(self): + first = artifact_fingerprint(self.artifact) + other = self.root / "copy" + other.mkdir() + for path in self.artifact.iterdir(): + (other / path.name).write_bytes(path.read_bytes()) + self.assertEqual(first, artifact_fingerprint(other)) + (other / "model.safetensors").write_bytes( + (other / "model.safetensors").read_bytes() + b"\x00" + ) + self.assertNotEqual(first, artifact_fingerprint(other)) def test_no_proxy_fallback(self): with self.assertRaisesRegex(ContractError, "local data-only"): From c5c581a572e09646da9b186f09e8610f4fa61c5d Mon Sep 17 00:00:00 2001 From: Mathis <154886644+echobt@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:40:14 +0400 Subject: [PATCH 2/2] feat(proof): stub-win sim scores against sealed baseline (#232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(proof): stub-win sim scores against sealed baseline Emit harness metrics relative to the sealed vector when PROOF_FORCE_SIM and PROOF_SIM_STUB_WIN are set, so staging submits can reach awaiting_admin. Skill-only sim stays NLL>=1.0. Co-authored-by: Mathis * test(proof): probe both staging topics on droplet :80 Prefer 159.223.159.205/challenge/proof (ready sim) over the stale staging.api Lium instance. Submit both open topic ids. Co-authored-by: Mathis * docs(proof): record staging sim submit→score (rejected vs seal) Live 201 on dt-no-ib-v0 and muon-vs-adamw-10m-v0 against 159.223.159.205; default sim misses the ~0.29 NLL floor. Co-authored-by: Mathis * docs(staging): point proof probe at droplet sim host Co-authored-by: Mathis * feat(proof): sealed-relative sim under force_sim Option A: Sim + sealed baseline always uses sim_win_document. No extra host env. Do not reseal (option B is ops-owned). Co-authored-by: Mathis * test(proof): lock sealed-relative sim inequalities Skill 0.95 (StubScorer::win) still NLL>=1.0 vs a 0.29 seal. Option B reseal is paused; do not reseal from this lane. Co-authored-by: Mathis --------- Co-authored-by: Cursor Agent Co-authored-by: Mathis --- Cargo.lock | 5 + bins/ctx/src/proof.rs | 27 +- bins/proof-challenge/Cargo.toml | 7 + bins/proof-challenge/tests/submit_e2e.rs | 441 +++++++++++++++++++++ crates/proof-challenge/src/lib.rs | 4 +- crates/proof-eval/src/lib.rs | 272 ++++++++++++- crates/proof-http/Cargo.toml | 1 + crates/proof-http/src/lib.rs | 322 ++++++++++++++- crates/proof-http/tests/live_submit_e2e.rs | 170 ++++++++ deploy/compose/env-local.yml | 1 + deploy/env/proof-challenge.env.example | 5 + deploy/scripts/assert-compose-matrix.sh | 2 +- deploy/scripts/local-e2e.sh | 14 + deploy/scripts/proof-submit-e2e.sh | 247 ++++++++++++ docker-compose.e2e.yml | 1 + docker-compose.yml | 2 + docs/AGENTS.md | 1 + docs/PROOF.md | 12 +- docs/runbooks/local-testnet-e2e.md | 9 +- docs/runbooks/proof-submit-e2e.md | 273 +++++++++++++ docs/runbooks/staging-testnet-e2e.md | 13 +- 21 files changed, 1810 insertions(+), 19 deletions(-) create mode 100644 bins/proof-challenge/tests/submit_e2e.rs create mode 100644 crates/proof-http/tests/live_submit_e2e.rs create mode 100755 deploy/scripts/proof-submit-e2e.sh create mode 100644 docs/runbooks/proof-submit-e2e.md diff --git a/Cargo.lock b/Cargo.lock index 30efa1280..cbba96274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3614,8 +3614,10 @@ dependencies = [ "axum", "challenge-keys", "clap", + "crypto", "db", "harvest-pod", + "hex", "prism-lium", "proof-autonomy-http", "proof-autonomy-pg", @@ -3624,7 +3626,9 @@ dependencies = [ "proof-harvest", "proof-store", "proof-task", + "reqwest 0.12.28", "serde_json", + "sha2 0.10.9", "telemetry", "tokio", "tracing", @@ -3741,6 +3745,7 @@ dependencies = [ "proof-score", "proof-store", "proof-task", + "reqwest 0.12.28", "serde", "serde_json", "sha2 0.10.9", diff --git a/bins/ctx/src/proof.rs b/bins/ctx/src/proof.rs index 84876084a..f811807b7 100644 --- a/bins/ctx/src/proof.rs +++ b/bins/ctx/src/proof.rs @@ -113,10 +113,7 @@ pub async fn topics(client: &Client, json_out: bool) -> Result<(), String> { if json_out { return Ok(()); } - let items = reply - .body - .as_array() - .or_else(|| reply.body.get("topics").and_then(Value::as_array)); + let items = topic_list_items(&reply.body); match items { Some(list) if list.is_empty() => { println!("No open topics. Submits answer 503 until an operator publishes one."); @@ -245,6 +242,14 @@ fn print_fields(body: &Value) { } } +/// `GET /v1/proof/topics` returns `{ "items": [...] }`. Older shapes used +/// a bare array or `{ "topics": [...] }`. +fn topic_list_items(body: &Value) -> Option<&Vec> { + body.as_array() + .or_else(|| body.get("items").and_then(Value::as_array)) + .or_else(|| body.get("topics").and_then(Value::as_array)) +} + fn explain_failure(status: u16, message: &str) -> String { match status { 400 => format!("refused ({message}). Nothing was stored and nothing was rented."), @@ -287,4 +292,18 @@ mod tests { let ok = "a".repeat(64); assert_eq!(normalize_hex64(&ok, "hotkey").unwrap(), ok); } + + #[test] + fn topic_list_reads_the_items_wrapper() { + let body = serde_json::json!({ + "items": [ + { "id": "dt-no-ib-v0", "status": "open", "payout_mode": "wta" }, + { "id": "muon-vs-adamw-10m-v0", "status": "open", "payout_mode": "wta" } + ] + }); + let items = topic_list_items(&body).expect("items"); + assert_eq!(items.len(), 2); + assert_eq!(items[0]["id"], "dt-no-ib-v0"); + assert_eq!(items[1]["id"], "muon-vs-adamw-10m-v0"); + } } diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index ca7848c5e..e017ed625 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -31,5 +31,12 @@ telemetry = { path = "../../crates/telemetry" } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } tracing = "0.1" +[dev-dependencies] +crypto = { path = "../../crates/crypto" } +hex = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "time"] } + [lints] workspace = true diff --git a/bins/proof-challenge/tests/submit_e2e.rs b/bins/proof-challenge/tests/submit_e2e.rs new file mode 100644 index 000000000..11760cb1b --- /dev/null +++ b/bins/proof-challenge/tests/submit_e2e.rs @@ -0,0 +1,441 @@ +//! Process-level Proof submit → sim score. +//! +//! Spawns `proof-challenge --force-sim` with disposable synthetic +//! topic / holdout / baseline / offer files. No Lium, no compose, no secrets. +//! Topic ids match the staging open set. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use proof_eval::{sim_document, BaselineMeasurement, BASELINE_SKILL}; +use proof_task::{ + default_adamw, holdout_commitment, inference_config_commitment, synthetic_holdout, Constraints, + InferenceConfig, InferenceMode, InferenceOffer, InferenceProvider, InferenceProviderKind, + MetricDirection, MetricFamily, MetricSpec, OfferStatus, ProofPin, TopicDocument, TopicStatus, + EVAL_IMAGE, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, METRIC_TOKENS_PER_SEC, PRIMARY_HOLDOUT_NLL, + STRATUM_SIZE, +}; +use serde_json::Value; +use tokio::process::Command; + +const DT: &str = "dt-no-ib-v0"; +const MUON: &str = "muon-vs-adamw-10m-v0"; +const OFFER_ID: &str = "openrouter-glm53flash-v0"; + +fn sk() -> [u8; 32] { + let mut s = [3u8; 32]; + s[0] = 17; + s +} + +fn pk_hex() -> String { + hex::encode(crypto::public_key_from_mini_secret(&sk()).expect("pk")) +} + +fn digest(label: &str) -> String { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) +} + +fn pin() -> ProofPin { + let mut p = ProofPin { + topic_pubkey: pk_hex(), + ..ProofPin::default() + }; + p.inference.model = "master-proxy-v0".into(); + p +} + +fn offer() -> InferenceOffer { + let config = InferenceConfig { + mode: InferenceMode::Chat, + model_ref: "master-proxy-v0".into(), + max_input_tokens: 32_768, + max_output_tokens: 8_192, + temperature: Some(0.0), + top_p: None, + timeout_ms: None, + }; + InferenceOffer { + offer_id: OFFER_ID.into(), + provider: InferenceProvider { + kind: InferenceProviderKind::OpenaiCompatible, + base_url: "http://127.0.0.1:8000/v1".into(), + }, + config_commitment: inference_config_commitment(&config, "http://127.0.0.1:8000/v1"), + config, + status: OfferStatus::Open, + } +} + +fn dt_topic() -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.optimizer = "nccl-ib-reference".into(); + baseline.wall_budget_s = 14_400; + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: DT.into(), + statement: "No IB/NVLink; 12.5 Gbit/s cap; beat sealed comms baseline.".into(), + payout_mode: proof_task::PayoutMode::Wta, + constraints: Constraints { + no_infiniband: true, + no_nvlink: true, + no_nccl_fast_fabric: true, + max_inter_node_gbps: Some(12.5), + }, + metric: MetricSpec { + family: MetricFamily::Throughput, + primary: METRIC_TOKENS_PER_SEC.into(), + direction: MetricDirection::Max, + unit: "tokens_per_second".into(), + epsilon_rel: 0.05, + quality_floor_nll: 0.02, + wall_budget_s: 14_400, + custom_id: String::new(), + }, + baseline, + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } +} + +fn muon_topic() -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: MUON.into(), + statement: + "Beat sealed AdamW holdout NLL with Muon at ~10M params under the same FLOP budget." + .into(), + payout_mode: proof_task::PayoutMode::Wta, + metric: MetricSpec { + family: MetricFamily::Nll, + primary: PRIMARY_HOLDOUT_NLL.into(), + direction: MetricDirection::Min, + unit: "nll".into(), + epsilon_rel: 0.0, + quality_floor_nll: 0.0, + wall_budget_s: 0, + custom_id: String::new(), + }, + baseline, + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } +} + +fn seal( + pin: &ProofPin, + mut topic: TopicDocument, +) -> ( + TopicDocument, + BaselineMeasurement, + Vec, +) { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + topic.holdout_commitment = holdout_commitment(&recs); + let doc = sim_document(pin, &topic, "base", "base-art", BASELINE_SKILL, true); + let meas = BaselineMeasurement { + eval_image_digest: pin.eval_image_digest.clone(), + topic_id: topic.id.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + holdout_nll: doc.harness.holdout_nll, + split_nll: doc.harness.split_nll.clone(), + tokens_per_sec: doc.harness.tokens_per_sec, + step_latency_ms: doc.harness.step_latency_ms, + custom_value: doc.harness.custom_value, + }; + topic.baseline.metrics_commitment = meas.commitment(); + topic.signature = topic.sign_with(&sk()).expect("sign"); + topic.validate(pin, &[]).expect("valid"); + topic.verify_signature(pin).expect("sig"); + (topic, meas, recs) +} + +fn write_pin(dir: &Path, pin: &ProofPin) -> PathBuf { + let path = dir.join("pin.toml"); + let body = format!( + r#"challenge_id = "proof" +scoring_version = 1 +base_model_family = "Qwen/Qwen3.8" +proxy_model = "" +proxy_models = [] +inference_config_schema_version = 1 +allowed_modes = ["chat", "completions", "embeddings"] +max_input_tokens_ceiling = 32768 +max_output_tokens_ceiling = 8192 +inference_offer_commitment_alg = "sha256" +eval_image = "{EVAL_IMAGE}" +eval_image_digest = "{digest}" +proof_git = "https://github.com/CortexLM/cortex" +proof_git_sha = "" +topic_pubkey = "{pk}" +flops_budget_max = 2000000000000000000 +epsilon_nll_min = 0.02 +epsilon_topic_max_regress_min = 0.05 +epsilon_throughput_rel_min = 0.05 +quality_floor_nll_max = 0.02 +holdout_size = 120 +stratum_size = 24 + +[inference] +provider = "openai_compatible" +base_url = "" +model = "master-proxy-v0" +mode = "chat" +max_input_tokens = 32768 +max_output_tokens = 8192 +"#, + digest = pin.eval_image_digest, + pk = pin.topic_pubkey, + ); + fs::write(&path, body).expect("pin"); + path +} + +fn workdir() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "proof-submit-e2e-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&dir).expect("dir"); + dir +} + +struct Host { + child: tokio::process::Child, + base: String, + dir: PathBuf, +} + +impl Host { + async fn spawn() -> Self { + let dir = workdir(); + let p = pin(); + let (dt, dt_meas, dt_recs) = seal(&p, dt_topic()); + let (muon, muon_meas, muon_recs) = seal(&p, muon_topic()); + fs::write( + dir.join("topics.json"), + serde_json::to_vec(&[&dt, &muon]).expect("topics"), + ) + .expect("write topics"); + fs::write( + dir.join("holdouts.json"), + serde_json::to_vec(&serde_json::json!({ + DT: dt_recs, + MUON: muon_recs, + })) + .expect("holdouts"), + ) + .expect("write holdouts"); + fs::write( + dir.join("baselines.json"), + serde_json::to_vec(&serde_json::json!({ + DT: dt_meas, + MUON: muon_meas, + })) + .expect("baselines"), + ) + .expect("write baselines"); + fs::write( + dir.join("offer.json"), + serde_json::to_vec(&offer()).expect("offer"), + ) + .expect("write offer"); + let pin_path = write_pin(&dir, &p); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind probe"); + let addr = listener.local_addr().expect("addr"); + drop(listener); + + let mut child = Command::new(env!("CARGO_BIN_EXE_proof-challenge")) + .arg("--bind") + .arg(addr.to_string()) + .arg("--force-sim") + .arg("--pin-file") + .arg(&pin_path) + .arg("--topics-file") + .arg(dir.join("topics.json")) + .arg("--holdout-file") + .arg(dir.join("holdouts.json")) + .arg("--baseline-file") + .arg(dir.join("baselines.json")) + .arg("--inference-offer-file") + .arg(dir.join("offer.json")) + .env("PROOF_FORCE_SIM", "true") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("spawn proof-challenge"); + + let base = format!("http://{addr}"); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("client"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if tokio::time::Instant::now() > deadline { + let _ = child.start_kill(); + let out = child.wait_with_output().await.expect("wait"); + panic!( + "proof-challenge did not become healthy\nstderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + if client + .get(format!("{base}/health")) + .send() + .await + .ok() + .is_some_and(|r| r.status().is_success()) + { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + Self { child, base, dir } + } +} + +impl Drop for Host { + fn drop(&mut self) { + let _ = self.child.start_kill(); + let _ = fs::remove_dir_all(&self.dir); + } +} + +async fn json(method: reqwest::Method, url: &str, body: Option) -> (u16, Value) { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("client"); + let mut req = client.request(method, url); + if let Some(b) = body { + req = req.json(&b); + } + let resp = req.send().await.expect("http"); + let status = resp.status().as_u16(); + let v = resp.json::().await.unwrap_or(Value::Null); + (status, v) +} + +fn submit_body(topic_id: &str, extra: &Value) -> Value { + let mut v = serde_json::json!({ + "miner_hotkey": digest("e2e-miner"), + "artifact_digest": digest(topic_id), + "claim": "e2e sim claim + artifact + declared_flops", + "declared_flops": 1_000_000u64, + "topic_id": topic_id, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] }, + }); + if let Some(obj) = extra.as_object() { + if let Some(dst) = v.as_object_mut() { + for (k, val) in obj { + dst.insert(k.clone(), val.clone()); + } + } + } + v +} + +#[tokio::test] +async fn force_sim_binary_scores_staging_topic_ids() { + let host = Host::spawn().await; + let (st, status) = json( + reqwest::Method::GET, + &format!("{}/v1/status", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{status}"); + assert_eq!(status["challenge_id"], "proof"); + assert_eq!(status["can_score"], true, "{status}"); + assert_eq!(status["eval_backend"], "sim", "{status}"); + assert_eq!(status["force_sim"], true, "{status}"); + assert_eq!(status["sim_stub_win"], true, "{status}"); + assert_eq!(status["baseline_sealed"], true, "{status}"); + assert_eq!(status["inference_offer"]["offer_id"], OFFER_ID); + let open = status["open_topics"].as_array().expect("open_topics"); + let ids: Vec<&str> = open.iter().filter_map(Value::as_str).collect(); + assert!(ids.contains(&DT), "{status}"); + assert!(ids.contains(&MUON), "{status}"); + assert!(!status.to_string().contains("api_key"), "{status}"); + assert!(!status.to_string().contains("8000"), "{status}"); + + let (st, topics) = json( + reqwest::Method::GET, + &format!("{}/v1/proof/topics", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{topics}"); + assert!(!topics.to_string().contains("content_sha256"), "{topics}"); + + for topic_id in [DT, MUON] { + let (st, created) = json( + reqwest::Method::POST, + &format!("{}/v1/submissions", host.base), + Some(submit_body(topic_id, &serde_json::json!({}))), + ) + .await; + assert_eq!(st, 201, "{topic_id}: {created}"); + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "{created}" + ); + assert_eq!(created["topic_id"], topic_id); + assert_eq!(created["eval_backend"], "sim"); + assert_eq!(created["state"], "awaiting_admin", "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + + let id = created["id"].as_str().expect("id"); + let (st, row) = json( + reqwest::Method::GET, + &format!("{}/v1/submissions/{id}", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{row}"); + assert_eq!(row["declared_flops"], 1_000_000); + assert!(row["verdict"]["agent"].is_object(), "judge missing: {row}"); + assert!( + row["verdict"]["harness"]["holdout_nll"].is_number(), + "{row}" + ); + assert_eq!(row["verdict"]["pass"], true, "{row}"); + assert_eq!(row["verdict"]["agent"]["rationale"], "sim stub win"); + assert!( + row["receipt_json"] + .as_str() + .is_some_and(|s| s.contains("sim")), + "{row}" + ); + } + + let (st, bad) = json( + reqwest::Method::POST, + &format!("{}/v1/submissions", host.base), + Some(submit_body("", &serde_json::json!({ "topic_id": "" }))), + ) + .await; + assert_eq!(st, 400, "{bad}"); + assert_eq!(bad["error"], "topic_id is required"); +} diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index a07a40590..7e5e3ab3e 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -16,8 +16,8 @@ use proof_score::{payout_lattices, MinerTopicRun, SealedBaseline}; use proof_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; pub use proof_eval::{ - force_sim, resolve_eval_backend, scoring_readiness, supported_custom, BaselineMeasurement, - EvalBackend, LiveScorer, + force_sim, resolve_eval_backend, scoring_readiness, sim_stub_win, supported_custom, + BaselineMeasurement, EvalBackend, LiveScorer, }; pub use proof_http::{hash_admin_token, proof_router, AppState}; pub use proof_store::{ArtifactManifest, MemoryStore, StoreError}; diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 3c17aa0d3..23f35ea3b 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -29,8 +29,8 @@ use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind, Seale use proof_store::ArtifactManifest; use proof_task::{ canonical_json, contamination, require_open_offer, resolve_inference, HoldoutRecord, - HoldoutSplit, InferenceOffer, MetricFamily, OfferError, ProofPin, TopicDocument, - BASELINE_DOMAIN, + HoldoutSplit, InferenceOffer, MetricDirection, MetricFamily, OfferError, ProofPin, + TopicDocument, BASELINE_DOMAIN, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -58,6 +58,17 @@ pub fn force_sim() -> bool { ) } +/// True when this host opted into sim (`PROOF_FORCE_SIM`). +/// +/// Under [`EvalBackend::Sim`] a sealed baseline scores with +/// [`sim_win_document`] (harness relative to the seal). Skill-only +/// [`sim_document`] cannot beat a real ~0.29 NLL seal. The Lium path never +/// uses either helper. `PROOF_SIM_STUB_WIN` is a leftover no-op. +#[must_use] +pub fn sim_stub_win() -> bool { + force_sim() +} + /// Resolve the scoring backend for this host. Sim is never implicit. #[must_use] pub fn resolve_eval_backend() -> EvalBackend { @@ -462,6 +473,11 @@ fn unit(parts: &[&str], index: u32) -> f64 { } /// Deterministic sim scores. Only used when the host opted into sim. +/// +/// Holdout NLL is `(3.10 - 0.40 * skill).max(1.0)`. Skill=1.0 still yields +/// NLL ≥ 1.0, so this **cannot** clear `quality_floor` against a real sealed +/// baseline near 0.29. Test wins against sim-derived baselines +/// ([`BASELINE_SKILL`]) do not apply on staging. Use [`sim_win_document`]. #[must_use] pub fn sim_document( pin: &ProofPin, @@ -528,6 +544,96 @@ pub fn sim_document( /// Skill of the sealed AdamW / comms reference in sim (so a strong miner wins). pub const BASELINE_SKILL: f64 = 0.40; +fn beat(baseline: f64, direction: MetricDirection, epsilon: f64) -> f64 { + let margin = 0.01; + match direction { + MetricDirection::Max => baseline * (1.0 + epsilon + margin), + MetricDirection::Min => (baseline * (1.0 - epsilon - margin)).max(0.0), + } +} + +/// Sim harness **relative to `sealed`**, not a higher [`sim_document`] skill. +/// +/// Inequalities (option A): holdout NLL ≤ baseline + quality floor, each +/// scored split ≤ baseline + `epsilon_topic_max_regress`, and +/// `tokens_per_sec` ≥ ref × (1 + `epsilon_rel`) when that is the primary. +/// Used for every [`EvalBackend::Sim`] score that has a sealed baseline. +/// Never called from the Lium path. +#[must_use] +pub fn sim_win_document( + pin: &ProofPin, + topic: &TopicDocument, + frozen: &str, + artifact: &str, + sealed: &SealedBaseline, +) -> ProofEvalDocument { + let nll_eps = topic.epsilon_nll.max(0.01); + let holdout = match topic.metric.family { + MetricFamily::Nll => (sealed.holdout_nll - nll_eps).max(0.0), + _ => sealed.holdout_nll, + }; + let mut split = BTreeMap::new(); + for s in HoldoutSplit::SCORED { + let b = sealed.split_nll.get(s.as_str()).copied().unwrap_or(holdout); + let v = match topic.metric.family { + MetricFamily::Nll => (b - nll_eps).max(0.0), + _ => b, + }; + split.insert(s.as_str().to_owned(), v); + } + let tps = match topic.metric.primary.as_str() { + proof_task::METRIC_TOKENS_PER_SEC => Some(beat( + sealed.tokens_per_sec.unwrap_or(100.0), + topic.metric.direction, + topic.metric.epsilon_rel, + )), + _ => sealed.tokens_per_sec, + }; + let latency = match topic.metric.primary.as_str() { + proof_task::METRIC_STEP_LATENCY_MS => Some(beat( + sealed.step_latency_ms.unwrap_or(100.0), + topic.metric.direction, + topic.metric.epsilon_rel, + )), + _ => sealed.step_latency_ms, + }; + let custom = sealed + .custom_value + .map(|b| beat(b, topic.metric.direction, topic.metric.epsilon_rel)); + ProofEvalDocument { + schema_version: PROOF_METRICS_SCHEMA, + submission_digest: frozen.to_owned(), + artifact_digest: artifact.to_owned(), + topic_id: topic.id.clone(), + eval_image_digest: pin.eval_image_digest.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + agent: AgentVerdict { + verdict: ProofKind::Clean, + reproduced: true, + claim_holds_public: true, + contamination: false, + canary_hit: false, + flops_used: topic.flops_budget / 2, + flops_budget: topic.flops_budget, + cheat_codes: Vec::new(), + rationale: "sim stub win".into(), + topic_id: topic.id.clone(), + family: topic.metric.family, + }, + harness: HarnessMetrics { + holdout_nll: holdout, + split_nll: split, + public_nll: Some(holdout), + tokens_per_sec: tps, + step_latency_ms: latency, + wall_s: (topic.metric.family == MetricFamily::Throughput) + .then_some(topic.metric.wall_budget_s / 2), + custom_value: custom, + canary_nll: None, + }, + } +} + /// Score only after the submission digest is frozen and a topic is open. #[allow(clippy::too_many_arguments)] pub async fn eval_after_freeze( @@ -541,6 +647,7 @@ pub async fn eval_after_freeze( backend: EvalBackend, live: Option<&dyn LiveScorer>, judge_api_key: Option<&str>, + sealed: Option<&SealedBaseline>, ) -> Result { if frozen_digest.trim().is_empty() || holdout.is_empty() { return Err(EvalError::HoldoutSealed); @@ -567,8 +674,12 @@ pub async fn eval_after_freeze( } let doc = match backend { EvalBackend::Sim => { - let skill = unit(&[artifact_digest, "skill"], 0); - sim_document(pin, topic, frozen_digest, artifact_digest, skill, true) + if let Some(sealed) = sealed { + sim_win_document(pin, topic, frozen_digest, artifact_digest, sealed) + } else { + let skill = unit(&[artifact_digest, "skill"], 0); + sim_document(pin, topic, frozen_digest, artifact_digest, skill, true) + } } EvalBackend::Lium => { let scorer = live.ok_or(EvalError::LiveHarvestUnavailable)?; @@ -731,6 +842,7 @@ mod tests { EvalBackend::Lium, None, None, + None, ) .await .expect_err("no digest"); @@ -750,6 +862,7 @@ mod tests { EvalBackend::Lium, None, None, + None, ) .await .expect_err("no harvest"); @@ -775,6 +888,7 @@ mod tests { EvalBackend::Lium, Some(&Harvest { reproduced: true }), Some("test-judge-key"), + None, ) .await .expect("live"); @@ -891,4 +1005,154 @@ mod tests { let doc = sim_document(&pin, &t, "f", "art", 1.0, true); assert!(doc.harness.custom_value.is_none()); } + + fn tight_sealed() -> SealedBaseline { + let mut split = BTreeMap::new(); + for s in HoldoutSplit::SCORED { + split.insert(s.as_str().to_owned(), 0.29); + } + SealedBaseline { + holdout_nll: 0.29, + split_nll: split, + tokens_per_sec: Some(80.0), + step_latency_ms: None, + custom_value: None, + } + } + + fn throughput_topic() -> TopicDocument { + let mut t = topic(); + t.id = "dt-no-ib-v0".into(); + t.metric.family = MetricFamily::Throughput; + t.metric.primary = proof_task::METRIC_TOKENS_PER_SEC.into(); + t.metric.direction = MetricDirection::Max; + t.metric.epsilon_rel = 0.05; + t.metric.quality_floor_nll = 0.02; + t.metric.wall_budget_s = 14_400; + t + } + + #[test] + fn stub_win_clears_quality_floor_against_a_tight_sealed_baseline() { + let pin = pin(""); + let t = throughput_topic(); + let sealed = tight_sealed(); + let floor = sealed.holdout_nll + t.metric.quality_floor_nll; + let stub_win_skill = sim_document(&pin, &t, "f", "art", 0.95, true); + let max_skill = sim_document(&pin, &t, "f", "art", 1.0, true); + assert!( + max_skill.harness.holdout_nll >= 1.0, + "skill=1.0 must not dip below the sim NLL floor: {}", + max_skill.harness.holdout_nll + ); + assert!( + stub_win_skill.harness.holdout_nll >= 1.0, + "StubScorer::win skill=0.95 is still NLL≥1.0: {}", + stub_win_skill.harness.holdout_nll + ); + for skill_doc in [&stub_win_skill, &max_skill] { + let reject = proof_score::judge_topic( + &t, + &skill_doc.agent, + &skill_doc.harness, + &sealed, + &[], + &[], + ); + assert!(!reject.pass, "{reject:?}"); + assert!( + reject + .failed + .iter() + .any(|g| matches!(g, proof_score::GateFail::QualityFloor { .. })), + "{reject:?}" + ); + } + + let win = sim_win_document(&pin, &t, "f", "art", &sealed); + assert_eq!(win.agent.rationale, "sim stub win"); + assert!( + win.harness.holdout_nll <= floor, + "holdout {} > baseline+floor {}", + win.harness.holdout_nll, + floor + ); + for s in HoldoutSplit::SCORED { + let h = win.harness.split_nll[s.as_str()]; + let b = sealed.split_nll[s.as_str()]; + assert!( + h <= b + t.epsilon_topic_max_regress, + "split {} {h} > {b}+eps", + s.as_str() + ); + } + let tps = win.harness.tokens_per_sec.expect("tps"); + let ref_tps = sealed.tokens_per_sec.expect("ref tps"); + assert!( + tps >= ref_tps * (1.0 + t.metric.epsilon_rel), + "tps {tps} < ref*(1+eps) {}", + ref_tps * (1.0 + t.metric.epsilon_rel) + ); + let verdict = proof_score::judge_topic(&t, &win.agent, &win.harness, &sealed, &[], &[]); + assert!(verdict.pass, "{verdict:?}"); + assert!(verdict.failed.is_empty(), "{verdict:?}"); + } + + #[tokio::test] + async fn sim_plus_sealed_uses_relative_harness() { + let t = throughput_topic(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let p = pin(""); + let sealed = tight_sealed(); + let out = eval_after_freeze( + &p, + &t, + &offer(), + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Sim, + None, + None, + Some(&sealed), + ) + .await + .expect("sim"); + assert_eq!(out.backend, EvalBackend::Sim); + assert_eq!(out.receipt.provider, "sim"); + assert_eq!(out.agent.rationale, "sim stub win"); + assert!(out.harness.holdout_nll <= sealed.holdout_nll + t.metric.quality_floor_nll); + assert!( + out.harness.tokens_per_sec.expect("tps") + >= sealed.tokens_per_sec.expect("ref") * (1.0 + t.metric.epsilon_rel) + ); + let verdict = proof_score::judge_topic(&t, &out.agent, &out.harness, &sealed, &[], &[]); + assert!(verdict.pass, "{verdict:?}"); + } + + #[tokio::test] + async fn stub_win_is_ignored_on_the_lium_path() { + let t = throughput_topic(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let out = eval_after_freeze( + &p, + &t, + &offer(), + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Lium, + Some(&Harvest { reproduced: true }), + Some("test-judge-key"), + Some(&tight_sealed()), + ) + .await + .expect("live"); + assert_eq!(out.backend, EvalBackend::Lium); + assert_eq!(out.receipt.provider, "lium"); + assert!(out.harness.holdout_nll > 1.0, "must not emit stub-win NLL"); + } } diff --git a/crates/proof-http/Cargo.toml b/crates/proof-http/Cargo.toml index 391595438..ff0b9e1c6 100644 --- a/crates/proof-http/Cargo.toml +++ b/crates/proof-http/Cargo.toml @@ -25,6 +25,7 @@ db = { path = "../db", features = ["testing"] } async-trait = "0.1" crypto = { path = "../crypto" } http-body-util = "0.1" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync"] } tower = { version = "0.5", features = ["util"] } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 613e6f0a5..212d23cdf 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -141,6 +141,7 @@ async fn status(State(st): State) -> impl IntoResponse { }, "eval_backend": st.backend, "force_sim": force_sim(), + "sim_stub_win": st.backend == EvalBackend::Sim, "can_score": st.can_score(), "live_harvest_wired": st.live_scorer.is_some(), "baseline_sealed": baseline_sealed, @@ -316,6 +317,7 @@ async fn submit( st.backend, st.live(), st.judge_api_key.as_deref(), + Some(&sealed), ) .await .map_err(|e| eval_err(&e))?; @@ -618,10 +620,10 @@ mod tests { use proof_eval::{sim_document, BaselineMeasurement, BASELINE_SKILL}; use proof_task::{ default_adamw, holdout_commitment, inference_config_commitment, synthetic_holdout, - Constraints, InferenceConfig, InferenceMode, InferenceOffer, InferenceProvider, - InferenceProviderKind, MetricDirection, MetricFamily, MetricSpec, OfferStatus, - TopicDocument, TopicStatus, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, METRIC_TOKENS_PER_SEC, - STRATUM_SIZE, + Constraints, HoldoutSplit, InferenceConfig, InferenceMode, InferenceOffer, + InferenceProvider, InferenceProviderKind, MetricDirection, MetricFamily, MetricSpec, + OfferStatus, TopicDocument, TopicStatus, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, + METRIC_TOKENS_PER_SEC, STRATUM_SIZE, }; use tower::ServiceExt; @@ -654,6 +656,15 @@ mod tests { } fn offer() -> InferenceOffer { + named_offer("master-v0") + } + + /// Staging sim offer id (operator-published; miners do not bind it). + fn staging_offer() -> InferenceOffer { + named_offer("openrouter-glm53flash-v0") + } + + fn named_offer(offer_id: &str) -> InferenceOffer { let config = InferenceConfig { mode: InferenceMode::Chat, model_ref: "master-proxy-v0".into(), @@ -664,7 +675,7 @@ mod tests { timeout_ms: None, }; InferenceOffer { - offer_id: "master-v0".into(), + offer_id: offer_id.into(), provider: InferenceProvider { kind: InferenceProviderKind::OpenaiCompatible, base_url: "http://127.0.0.1:8000/v1".into(), @@ -708,6 +719,33 @@ mod tests { } } + fn unsigned_muon_topic(recs: &[proof_task::HoldoutRecord]) -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: "muon-vs-adamw-10m-v0".into(), + statement: + "Beat sealed AdamW holdout NLL with Muon at ~10M params under the same FLOP budget." + .into(), + payout_mode: proof_task::PayoutMode::Wta, + metric: MetricSpec { + family: MetricFamily::Nll, + primary: proof_task::PRIMARY_HOLDOUT_NLL.into(), + direction: MetricDirection::Min, + unit: "nll".into(), + epsilon_rel: 0.0, + quality_floor_nll: 0.0, + wall_budget_s: 0, + custom_id: String::new(), + }, + baseline, + holdout_commitment: holdout_commitment(recs), + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } + } + fn seal_topic( pin: &ProofPin, mut topic: TopicDocument, @@ -881,6 +919,7 @@ mod tests { json_req(app("op"), "GET", "/v1/status", serde_json::json!({}), None).await; assert_eq!(st, StatusCode::OK); assert_eq!(body["eval_backend"], "sim"); + assert_eq!(body["sim_stub_win"], true, "{body}"); assert_eq!(body["can_score"], true, "{body}"); assert_eq!(body["baseline_sealed"], true, "{body}"); assert_eq!(body["open_topics"][0], "dt-no-ib-v0"); @@ -1406,6 +1445,279 @@ mod tests { "{body}" ); } + + fn app_staging_sim() -> Router { + let p = pin(""); + let store = MemoryStore::new(); + for draft in [unsigned_topic(&[]), unsigned_muon_topic(&[])] { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, meas) = seal_topic(&p, draft); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, meas.into_sealed()) + .expect("baseline"); + } + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Sim, + live_scorer: None, + offer: Some(staging_offer()), + judge_api_key: None, + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + + fn assert_scored_row(created: &serde_json::Value, topic_id: &str) { + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "silent empty id: {created}" + ); + assert_eq!(created["topic_id"], topic_id, "{created}"); + assert_eq!(created["eval_backend"], "sim", "{created}"); + let state = created["state"].as_str().unwrap_or_default(); + assert!( + state == "awaiting_admin" || state == "rejected", + "non-terminal or empty state: {created}" + ); + assert!(created["eligible"].is_boolean(), "{created}"); + assert!( + created["submission_digest"] + .as_str() + .is_some_and(|d| d.len() == 64), + "{created}" + ); + } + + #[tokio::test] + async fn sim_submit_scores_claim_artifact_and_flops() { + let app = app("op"); + let body = submit_body( + "staging-e2e-artifact", + &serde_json::json!({ + "claim": "beats the sealed reference under the cap", + "declared_flops": 1_000_000_000_000u64, + }), + ); + let (st, created) = json_req(app.clone(), "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_scored_row(&created, "dt-no-ib-v0"); + + let id = created["id"].as_str().expect("id"); + let (st, row) = json_req( + app, + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{row}"); + assert_eq!(row["id"], id); + assert_eq!(row["topic_id"], "dt-no-ib-v0"); + assert_eq!(row["declared_flops"], 1_000_000_000_000u64); + assert_eq!(row["claim"], "beats the sealed reference under the cap"); + assert!(row["verdict"].is_object(), "judge path missing: {row}"); + assert!(row["verdict"]["agent"].is_object(), "{row}"); + assert!( + row["verdict"]["harness"]["holdout_nll"].is_number(), + "{row}" + ); + assert!(row["verdict"]["lattice"].is_number(), "{row}"); + assert!( + row["verdict"]["agent"]["rationale"] + .as_str() + .is_some_and(|s| !s.is_empty()), + "notation missing: {row}" + ); + let receipt = row["receipt_json"].as_str().unwrap_or_default(); + assert!(receipt.contains("sim"), "sim receipt missing: {row}"); + let dump = row.to_string(); + assert!(!dump.contains("content_sha256"), "{dump}"); + assert!(!dump.contains("api_key"), "{dump}"); + } + + #[tokio::test] + async fn sim_submit_accepts_staging_topic_ids() { + let app = app_staging_sim(); + let (st, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["can_score"], true, "{status}"); + assert_eq!(status["eval_backend"], "sim", "{status}"); + assert_eq!(status["baseline_sealed"], true, "{status}"); + assert_eq!( + status["inference_offer"]["offer_id"], + "openrouter-glm53flash-v0" + ); + let open = status["open_topics"].as_array().expect("open_topics"); + let ids: Vec<&str> = open.iter().filter_map(|v| v.as_str()).collect(); + assert!(ids.contains(&"dt-no-ib-v0"), "{status}"); + assert!(ids.contains(&"muon-vs-adamw-10m-v0"), "{status}"); + + let (st, list) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + let dump = list.to_string(); + assert!(!dump.contains("content_sha256"), "{dump}"); + assert!(!dump.contains("synthetic-dev"), "{dump}"); + + for topic_id in ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"] { + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body( + topic_id, + &serde_json::json!({ + "topic_id": topic_id, + "declared_flops": 42u64, + }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{topic_id}: {created}"); + assert_scored_row(&created, topic_id); + } + } + + #[tokio::test] + async fn sim_submit_fail_closed_reasons_are_explicit() { + let app = app_staging_sim(); + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("x", &serde_json::json!({ "topic_id": "" })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "topic_id is required"); + + let (st, body) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("x", &serde_json::json!({ "topic_id": "not-a-live-topic" })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "unknown topic"); + + let (st, body) = json_req( + app, + "POST", + "/v1/submissions", + submit_body("x", &serde_json::json!({ "declared_flops": u64::MAX })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("declared_flops"), + "{body}" + ); + } + + fn tight_sealed() -> proof_score::SealedBaseline { + let mut split = std::collections::BTreeMap::new(); + for s in HoldoutSplit::SCORED { + split.insert(s.as_str().to_owned(), 0.29); + } + proof_score::SealedBaseline { + holdout_nll: 0.29, + split_nll: split, + tokens_per_sec: Some(80.0), + step_latency_ms: None, + custom_value: None, + } + } + + fn app_tight_sim() -> Router { + let p = pin(""); + let store = MemoryStore::new(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, _) = seal_topic(&p, unsigned_topic(&recs)); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, tight_sealed()) + .expect("baseline"); + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Sim, + live_scorer: None, + offer: Some(staging_offer()), + judge_api_key: None, + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + + #[tokio::test] + async fn sim_stub_win_submit_reaches_awaiting_admin() { + let app = app_tight_sim(); + let (st, status) = json_req( + app.clone(), + "GET", + "/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["eval_backend"], "sim"); + assert_eq!(status["sim_stub_win"], true, "{status}"); + + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("tight-win", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "awaiting_admin", "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + assert_eq!(created["eval_backend"], "sim"); + let id = created["id"].as_str().expect("id"); + let (st, row) = json_req( + app, + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{row}"); + assert_eq!(row["verdict"]["pass"], true, "{row}"); + assert_eq!(row["verdict"]["agent"]["rationale"], "sim stub win"); + assert_eq!(row["verdict"]["failed"].as_array().map(Vec::len), Some(0)); + } } #[cfg(test)] diff --git a/crates/proof-http/tests/live_submit_e2e.rs b/crates/proof-http/tests/live_submit_e2e.rs new file mode 100644 index 000000000..218938cb7 --- /dev/null +++ b/crates/proof-http/tests/live_submit_e2e.rs @@ -0,0 +1,170 @@ +//! Optional live probe of a running Proof host (`PROOF_E2E_BASE`). +//! +//! Set `PROOF_E2E_BASE` to the challenge origin (no trailing slash), e.g. +//! `http://127.0.0.1:28100` or `http://staging.api.joinbase.ai/challenge/proof`. +//! +//! Never POSTs when the host is live Lium and `can_score` (would rent). +//! Never talks to production. Staging sim (`eval_backend=sim`) is the intended +//! target. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use serde_json::Value; + +const STAGING_TOPICS: [&str; 2] = ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"]; + +fn base_url() -> Option { + std::env::var("PROOF_E2E_BASE") + .ok() + .map(|s| s.trim().trim_end_matches('/').to_owned()) + .filter(|s| !s.is_empty()) +} + +fn is_prod_host(base: &str) -> bool { + base.contains("network.cortex.foundation") || base.contains("chain.joinbase.ai") +} + +fn hex64(label: &str) -> String { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) +} + +async fn get(client: &reqwest::Client, url: &str) -> (u16, Value) { + let resp = client.get(url).send().await.expect("GET"); + let status = resp.status().as_u16(); + let body = resp.json::().await.unwrap_or(Value::Null); + (status, body) +} + +async fn post(client: &reqwest::Client, url: &str, body: &Value) -> (u16, Value) { + let resp = client.post(url).json(body).send().await.expect("POST"); + let status = resp.status().as_u16(); + let body = resp.json::().await.unwrap_or(Value::Null); + (status, body) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn live_host_submit_scores_or_fails_closed() { + let Some(base) = base_url() else { + eprintln!("skip live_submit_e2e: set PROOF_E2E_BASE to probe a running host"); + return; + }; + assert!( + !is_prod_host(&base), + "refusing production host {base} (staging/local only)" + ); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("client"); + + let (st, health) = get(&client, &format!("{base}/health")).await; + assert_eq!(st, 200, "{health}"); + assert_eq!(health["challenge_id"], "proof"); + + let (st, status) = get(&client, &format!("{base}/v1/status")).await; + assert_eq!(st, 200, "{status}"); + assert_eq!(status["challenge_id"], "proof"); + assert!(status["can_score"].is_boolean(), "{status}"); + assert!(status["eval_backend"].is_string(), "{status}"); + assert!( + status.get("error").is_none(), + "status must not be an error object: {status}" + ); + let dump = status.to_string(); + assert!(!dump.contains("api_key"), "{dump}"); + assert!(!dump.contains("content_sha256"), "{dump}"); + + let (st, topics) = get(&client, &format!("{base}/v1/proof/topics")).await; + assert_eq!(st, 200, "{topics}"); + let items = topics["items"].as_array().cloned().unwrap_or_default(); + let listed: Vec = items + .iter() + .filter_map(|t| t.get("id").and_then(Value::as_str).map(ToOwned::to_owned)) + .collect(); + assert!( + !topics.to_string().contains("content_sha256"), + "holdout leak: {topics}" + ); + + let (st, missing) = post( + &client, + &format!("{base}/v1/submissions"), + &serde_json::json!({ + "miner_hotkey": hex64("e2e-hotkey"), + "artifact_digest": hex64("e2e-artifact"), + "claim": "e2e probe", + "declared_flops": 1, + "topic_id": "", + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }), + ) + .await; + assert_eq!(st, 400, "empty topic_id must 400, got {st} {missing}"); + assert!( + missing["error"].as_str().is_some_and(|e| !e.is_empty()), + "silent empty 400: {missing}" + ); + + let can_score = status["can_score"].as_bool().unwrap_or(false); + let backend = status["eval_backend"].as_str().unwrap_or_default(); + if can_score && backend == "lium" { + eprintln!("skip live POST: host is Lium + can_score (would rent)"); + return; + } + + let mut topic_ids: Vec<&str> = STAGING_TOPICS + .iter() + .copied() + .filter(|id| listed.iter().any(|got| got == *id)) + .collect(); + if topic_ids.is_empty() { + topic_ids.push(listed.first().map_or("dt-no-ib-v0", String::as_str)); + } + + for topic_id in topic_ids { + let (st, created) = post( + &client, + &format!("{base}/v1/submissions"), + &serde_json::json!({ + "miner_hotkey": hex64("e2e-hotkey"), + "artifact_digest": hex64(&format!("e2e-artifact-{topic_id}")), + "claim": "e2e sim submit against an open topic", + "declared_flops": 1, + "topic_id": topic_id, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }), + ) + .await; + assert!( + st == 201 || st == 400 || st == 503, + "unexpected {st} {created}" + ); + if st == 201 { + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "silent empty create: {created}" + ); + assert_eq!(created["topic_id"], topic_id); + assert!(created["eval_backend"].is_string(), "{created}"); + let id = created["id"].as_str().expect("id"); + let (gst, row) = get(&client, &format!("{base}/v1/submissions/{id}")).await; + assert_eq!(gst, 200, "{row}"); + assert!( + row["verdict"].is_object() || row["state"] == "rejected", + "{row}" + ); + } else { + assert!( + created["error"].as_str().is_some_and(|e| !e.is_empty()), + "silent empty fail-closed: HTTP {st} {created}" + ); + } + } +} diff --git a/deploy/compose/env-local.yml b/deploy/compose/env-local.yml index 7ced428d1..1f00e7655 100644 --- a/deploy/compose/env-local.yml +++ b/deploy/compose/env-local.yml @@ -68,6 +68,7 @@ services: BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} # No Lium spend on a laptop unless operator opts in. PROOF_FORCE_SIM: "${LOCAL_PROOF_FORCE_SIM:-true}" + PROOF_SIM_STUB_WIN: "${LOCAL_PROOF_SIM_STUB_WIN:-true}" bounty-challenge: ports: diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 62beab457..f6bad040a 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -14,6 +14,11 @@ BASE_NETUID=541 # live_harvest_wired, baseline_sealed. PROOF_FORCE_SIM=false +# Leftover no-op. Under PROOF_FORCE_SIM a sealed topic already emits +# harness numbers relative to the seal. Do not set this in +# deploy/compose/env-staging.yml or env-prod.yml. +PROOF_SIM_STUB_WIN=false + # The miner pays for the eval pod; this key is the master's Lium account used # to provision and terminate it. Never logged, never echoed on /v1/status. # LIUM_API_KEY= diff --git a/deploy/scripts/assert-compose-matrix.sh b/deploy/scripts/assert-compose-matrix.sh index 414e6e6da..4cdd13f79 100755 --- a/deploy/scripts/assert-compose-matrix.sh +++ b/deploy/scripts/assert-compose-matrix.sh @@ -132,7 +132,7 @@ for env_file in deploy/compose/env-staging.yml deploy/compose/env-prod.yml; do if echo "$rendered" | grep -qE 'DESIGN_FORCE_SIM:[[:space:]]*["'\'']?(1|true|TRUE|yes)["'\'']?'; then fail "$env_file enables DESIGN_FORCE_SIM (retired; must not ship)" fi - for sim_var in RELEARN_FORCE_SIM RELEARN_T2I_FORCE_SIM RELEARN_AGENT_FORCE_SIM RELEARN_MM_FORCE_SIM PROOF_FORCE_SIM; do + for sim_var in RELEARN_FORCE_SIM RELEARN_T2I_FORCE_SIM RELEARN_AGENT_FORCE_SIM RELEARN_MM_FORCE_SIM PROOF_FORCE_SIM PROOF_SIM_STUB_WIN; do if echo "$rendered" | grep -qE "${sim_var}:[[:space:]]*[\"']?(1|true|TRUE|yes)[\"']?"; then fail "$env_file enables $sim_var (sim is local-only; must not ship on droplets)" fi diff --git a/deploy/scripts/local-e2e.sh b/deploy/scripts/local-e2e.sh index 5f5bee66f..2786992bb 100755 --- a/deploy/scripts/local-e2e.sh +++ b/deploy/scripts/local-e2e.sh @@ -664,6 +664,18 @@ probe_bounty_fail_closed() { fi } +# Proof submit → score (or explicit 400/503). Never rents Lium. +# Full matrix + curl/ctx contract: docs/runbooks/proof-submit-e2e.md +probe_proof_submit() { + local base="http://127.0.0.1:${PROOF_HOST_PORT}" + if ! curl -fsS -m 5 "${base}/health" >/dev/null 2>&1; then + log "warning: proof /health unavailable (skipping submit probe)" + return 0 + fi + log "proof submit e2e probe against ${base}" + PROOF_E2E_BASE="$base" "$ROOT/deploy/scripts/proof-submit-e2e.sh" --probe "$base" +} + print_summary() { local pub="" if [[ -f "$TUNNEL_ENV" ]]; then @@ -678,6 +690,7 @@ Internal (compose network): bounty: http://127.0.0.1:${BOUNTY_HOST_PORT}/health (scorer: GET /v1/status → scoring_backend, can_score) proof: http://127.0.0.1:${PROOF_HOST_PORT}/health + (submit: docs/runbooks/proof-submit-e2e.md) EOF if [[ -n "$pub" ]]; then @@ -773,6 +786,7 @@ wait_all_health || die "health checks failed — see logs above" # tunnel flake cannot mask a weights regression. probe_weights_latest || die "weights seal smoke failed" probe_bounty_fail_closed +probe_proof_submit || die "proof submit e2e probe failed" if [[ "$DO_TUNNEL" -eq 1 ]]; then start_tunnel diff --git a/deploy/scripts/proof-submit-e2e.sh b/deploy/scripts/proof-submit-e2e.sh new file mode 100755 index 000000000..8ab1ead42 --- /dev/null +++ b/deploy/scripts/proof-submit-e2e.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Proof submit → score E2E (staging / local sim only). +# +# Never: production hosts, set_weights, master Lium rent. +# POST /v1/submissions is skipped when eval_backend=lium and can_score=true. +# +# Usage: +# ./deploy/scripts/proof-submit-e2e.sh --http-tests +# ./deploy/scripts/proof-submit-e2e.sh --local-sim +# ./deploy/scripts/proof-submit-e2e.sh --probe [BASE] +# ./deploy/scripts/proof-submit-e2e.sh --bounty [BASE] +# ./deploy/scripts/proof-submit-e2e.sh --all +# +# Optional env: +# PROOF_E2E_BASE challenge origin (no trailing slash) +# BOUNTY_E2E_BASE bounty origin +# PROOF_E2E_TOPIC override topic id (default: first of dt-no-ib-v0 / muon-vs-adamw-10m-v0) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +RED() { printf '\033[31m%s\033[0m\n' "$*"; } +GRN() { printf '\033[32m%s\033[0m\n' "$*"; } +LOG() { printf '[proof-e2e] %s\n' "$*"; } + +PROD_HOSTS='network.cortex.foundation|chain.joinbase.ai' +STAGING_TOPICS=(dt-no-ib-v0 muon-vs-adamw-10m-v0) + +refuse_prod() { + local url="${1:-}" + if echo "$url" | grep -Eq "$PROD_HOSTS"; then + RED "refusing production host: $url" + exit 2 + fi +} + +http_tests() { + LOG "cargo test -p proof-http (in-process submit→score + live skip unless PROOF_E2E_BASE)" + cargo test -p proof-http -- --nocapture + LOG "cargo test -p ctx topic list items wrapper" + cargo test -p ctx -- topic_list_reads_the_items_wrapper + GRN "PASS --http-tests" +} + +local_sim() { + LOG "cargo test -p proof-challenge-bin --test submit_e2e (force_sim binary + both staging topic ids)" + cargo test -p proof-challenge-bin --test submit_e2e -- --nocapture + GRN "PASS --local-sim" +} + +# Probe a running Proof origin. Prints status / topics / submit result shapes. +# Exit 0 on scored 201 or explicit 400/503. Exit 1 on silent empty / unexpected. +probe_proof() { + local base="${1:-${PROOF_E2E_BASE:-}}" + if [[ -z "$base" ]]; then + for cand in \ + http://127.0.0.1:28100 \ + http://127.0.0.1:8100 \ + http://159.223.159.205/challenge/proof \ + http://159.223.159.205:8080/challenge/proof \ + http://159.223.159.205:8100 \ + http://staging.api.joinbase.ai/challenge/proof + do + if curl -fsS -m 3 "$cand/health" >/dev/null 2>&1; then + base="$cand" + break + fi + done + fi + if [[ -z "$base" ]]; then + LOG "no reachable Proof origin (set PROOF_E2E_BASE); skip --probe" + return 0 + fi + base="${base%/}" + refuse_prod "$base" + LOG "probing $base" + + local health status topics code body + health="$(curl -fsS -m 8 "$base/health")" + echo "$health" | grep -q '"challenge_id":"proof"' || { RED "health is not proof: $health"; return 1; } + LOG "GET /health → $health" + + status="$(curl -fsS -m 8 "$base/v1/status")" + LOG "GET /v1/status → $status" + echo "$status" | grep -q '"challenge_id":"proof"' || { RED "status missing challenge_id"; return 1; } + if echo "$status" | grep -q 'api_key\|content_sha256'; then + RED "status leaked a secret or holdout fingerprint" + return 1 + fi + + topics="$(curl -fsS -m 8 "$base/v1/proof/topics")" + LOG "GET /v1/proof/topics → $(echo "$topics" | head -c 400)…" + echo "$topics" | grep -q 'content_sha256' && { RED "topics leaked holdout records"; return 1; } + + code="$(curl -sS -m 8 -o /tmp/proof-e2e-empty.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d '{"miner_hotkey":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","artifact_digest":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","claim":"probe","declared_flops":1,"topic_id":"","manifest":{"train_dataset_ids":["e2e-mix-v0"]}}' \ + "$base/v1/submissions")" + body="$(cat /tmp/proof-e2e-empty.json)" + LOG "POST /v1/submissions empty topic_id → HTTP $code $body" + [[ "$code" == "400" ]] || { RED "empty topic_id expected 400, got $code"; return 1; } + echo "$body" | grep -q '"error"' || { RED "400 had no error field"; return 1; } + + local backend can_score + backend="$(echo "$status" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("eval_backend",""))')" + can_score="$(echo "$status" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("can_score", False))')" + + if [[ "$can_score" == "True" && "$backend" == "lium" ]]; then + LOG "skip POST: host is Lium + can_score (would rent). Fail-closed probe already passed." + GRN "PASS --probe $base (status+400 only; no Lium rent)" + return 0 + fi + + local topics_to_hit=() + if [[ -n "${PROOF_E2E_TOPIC:-}" ]]; then + topics_to_hit=("$PROOF_E2E_TOPIC") + else + for id in "${STAGING_TOPICS[@]}"; do + echo "$topics" | grep -q "\"$id\"" && topics_to_hit+=("$id") + done + if [[ ${#topics_to_hit[@]} -eq 0 ]]; then + topics_to_hit=(dt-no-ib-v0) + fi + fi + + local topic hex sid row any_scored=0 + for topic in "${topics_to_hit[@]}"; do + hex="$(printf '%s' "e2e-$topic-$RANDOM-$$" | sha256sum | awk '{print $1}')" + code="$(curl -sS -m 20 -o /tmp/proof-e2e-submit.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d "{\"miner_hotkey\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"artifact_digest\":\"$hex\",\"claim\":\"e2e sim submit against $topic\",\"declared_flops\":1,\"topic_id\":\"$topic\",\"manifest\":{\"train_dataset_ids\":[\"e2e-mix-v0\"]}}" \ + "$base/v1/submissions")" + body="$(cat /tmp/proof-e2e-submit.json)" + LOG "POST /v1/submissions topic_id=$topic → HTTP $code $body" + case "$code" in + 201) + echo "$body" | grep -q '"id":"pf_' || { RED "201 missing pf_ id"; return 1; } + echo "$body" | grep -q "\"topic_id\":\"$topic\"" || { RED "201 topic_id mismatch"; return 1; } + sid="$(echo "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))')" + row="$(curl -fsS -m 8 "$base/v1/submissions/$sid")" + LOG "GET /v1/submissions/$sid → $row" + echo "$row" | grep -q '"verdict"' || { RED "scored row missing verdict"; return 1; } + any_scored=1 + GRN "PASS --probe $base submit→score HTTP 201 topic=$topic" + ;; + 400|503) + echo "$body" | grep -q '"error"' || { RED "HTTP $code silent empty"; return 1; } + GRN "PASS --probe $base fail-closed HTTP $code topic=$topic (explicit error)" + ;; + *) + RED "unexpected HTTP $code topic=$topic (want 201/400/503, never silent empty)" + return 1 + ;; + esac + done + if [[ "$any_scored" == "1" ]]; then + GRN "PASS --probe $base scored ${#topics_to_hit[@]} topic(s)" + fi +} + +probe_bounty() { + local base="${1:-${BOUNTY_E2E_BASE:-}}" + if [[ -z "$base" ]]; then + for cand in \ + http://127.0.0.1:28096 \ + http://127.0.0.1:8096 \ + http://159.223.159.205:8096 \ + http://159.223.159.205:8080/challenge/bounty \ + http://staging.api.joinbase.ai/challenge/bounty + do + if curl -fsS -m 3 "$cand/health" >/dev/null 2>&1; then + base="$cand" + break + fi + done + fi + if [[ -z "$base" ]]; then + LOG "no reachable Bounty origin (set BOUNTY_E2E_BASE); skip --bounty" + return 0 + fi + base="${base%/}" + refuse_prod "$base" + LOG "probing bounty $base" + local status + status="$(curl -fsS -m 8 "$base/v1/status")" + LOG "GET /v1/status → $status" + echo "$status" | grep -q '"challenge_id":"bounty"' || { RED "not bounty"; return 1; } + + local code + code="$(curl -sS -m 8 -o /tmp/bounty-e2e-report.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d '{"session":"not-a-session","title":"e2e","body":"e2e","repro_steps":"e2e"}' \ + "$base/v1/reports")" + LOG "POST /v1/reports (thin) → HTTP $code $(cat /tmp/bounty-e2e-report.json)" + if echo "$status" | grep -q '"scoring_backend":"unconfigured"'; then + [[ "$code" == "503" ]] || { RED "unconfigured bounty must 503, got $code"; return 1; } + GRN "PASS --bounty $base fail-closed 503" + else + # Feed configured: do not file a real report. Session gate is enough. + [[ "$code" == "401" || "$code" == "400" || "$code" == "503" ]] \ + || { RED "configured bounty unexpected $code"; return 1; } + GRN "PASS --bounty $base ingest reached an explicit gate HTTP $code (no prod write)" + fi +} + +usage() { + sed -n '2,20p' "$0" +} + +DO_HTTP=0 +DO_LOCAL=0 +DO_PROBE=0 +DO_BOUNTY=0 +PROBE_BASE="" +BOUNTY_BASE="" + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --http-tests) DO_HTTP=1; shift ;; + --local-sim) DO_LOCAL=1; shift ;; + --probe) + DO_PROBE=1 + if [[ "${2:-}" != --* && -n "${2:-}" ]]; then PROBE_BASE="$2"; shift; fi + shift + ;; + --bounty) + DO_BOUNTY=1 + if [[ "${2:-}" != --* && -n "${2:-}" ]]; then BOUNTY_BASE="$2"; shift; fi + shift + ;; + --all) DO_HTTP=1; DO_LOCAL=1; DO_PROBE=1; DO_BOUNTY=1; shift ;; + -h|--help) usage; exit 0 ;; + *) RED "unknown arg: $1"; usage; exit 1 ;; + esac +done + +[[ "$DO_HTTP" -eq 1 ]] && http_tests +[[ "$DO_LOCAL" -eq 1 ]] && local_sim +[[ "$DO_PROBE" -eq 1 ]] && probe_proof "$PROBE_BASE" +[[ "$DO_BOUNTY" -eq 1 ]] && probe_bounty "$BOUNTY_BASE" +GRN "proof-submit-e2e done" diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 7df8ac766..4cd381c61 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -18,6 +18,7 @@ services: environment: BASE_CHALLENGE_BIND: 0.0.0.0:8100 PROOF_FORCE_SIM: "true" + PROOF_SIM_STUB_WIN: "true" bounty-challenge: ports: - "8096:8096" diff --git a/docker-compose.yml b/docker-compose.yml index 017a18deb..ed18f2f30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -253,6 +253,8 @@ services: BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk # Sim eval unless the operator opts in. Never log LIUM_API_KEY. PROOF_FORCE_SIM: "${PROOF_FORCE_SIM:-false}" + # Staging/dev only. Ignored unless PROOF_FORCE_SIM is on. Never a Lium path. + PROOF_SIM_STUB_WIN: "${PROOF_SIM_STUB_WIN:-false}" PROOF_PIN_FILE: /etc/base/config/proof-pin.toml PROOF_TOPICS_FILE: /run/base/proof/topics.json PROOF_HOLDOUT_FILE: /run/base/proof/holdouts.json diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 7bd57759a..d1a435a59 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -19,6 +19,7 @@ When a spike or evidence report conflicts with a frozen spec or runbook, the nor | [`runbooks/promote-rollback-restore.md`](runbooks/promote-rollback-restore.md) | Digest promote, rollback, Postgres backup/restore | | [`runbooks/local-testnet-e2e.md`](runbooks/local-testnet-e2e.md) | Local laptop/VM full subnet stack on testnet 541 + ephemeral gateway tunnel | | [`runbooks/staging-testnet-e2e.md`](runbooks/staging-testnet-e2e.md) | Staging droplet testnet end-to-end validation | +| [`runbooks/proof-submit-e2e.md`](runbooks/proof-submit-e2e.md) | Proof (and Bounty) submit → score: cargo tests, local `--force-sim`, staging curl/ctx | | [`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md) | Trust-root key rotation | | [`runbooks/gateway-failover.md`](runbooks/gateway-failover.md) | Gateway kill/restart / failover checks | | [`runbooks/measurement-repin-socket-proxy.md`](runbooks/measurement-repin-socket-proxy.md) | Socket-proxy measurement re-pin | diff --git a/docs/PROOF.md b/docs/PROOF.md index c3abb60f7..26960cbe3 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -150,7 +150,11 @@ baseline + an open topic are on the host. publish that topic; scoring fail-closes until the real harness fills `custom_value`. - `PROOF_FORCE_SIM` is CI/local opt-in only. Never a fallback. Forbidden on - droplet overlays. + droplet overlays. Under sim, a sealed topic scores with harness numbers + relative to the seal (`sim_win_document`); skill-only `sim_document` + cannot beat a real ~0.29 NLL baseline. `PROOF_SIM_STUB_WIN` is a leftover + no-op. Resealing staging to `BASELINE_SKILL=0.40` (NLL ≈ 2.94) is an + operator lane, not this binary. - No Modal. No secrets, hosts, holdout records, or teacher endpoints in git. ## Publish a research topic @@ -268,6 +272,12 @@ takes the topic. } ``` +### `muon-vs-adamw-10m-v0` — NLL **wta** + +Operator **example**, not in the pin. Beat sealed AdamW holdout NLL with Muon +at ~10M params under the same FLOP budget. Staging may publish this id next +to `dt-no-ib-v0`; miners still discover it from `GET /v1/proof/topics`. + ### `agent-harness-improve-v0` — custom **discovery** Operator POST, not in git. `custom_id = harness_success_rate` is listed so diff --git a/docs/runbooks/local-testnet-e2e.md b/docs/runbooks/local-testnet-e2e.md index f8076ad73..2c5768293 100644 --- a/docs/runbooks/local-testnet-e2e.md +++ b/docs/runbooks/local-testnet-e2e.md @@ -79,7 +79,14 @@ cargo run -q --release -p weights-smoke -- \ ./deploy/scripts/local-e2e.sh --down ``` -Challenge verification must **simulate a submission** (harness/intake) and probe failures (bad harness, sanitize, quota, routes) in addition to the weights seal smoke above — see root [`AGENTS.md`](../../AGENTS.md). +Challenge verification must **simulate a submission** (harness/intake) and probe failures (bad harness, sanitize, quota, routes) in addition to the weights seal smoke above — see root [`AGENTS.md`](../../AGENTS.md). Proof submit → score (Sim or explicit 400/503) is [`proof-submit-e2e.md`](proof-submit-e2e.md); `--smoke` now probes it when `proof-challenge` is healthy. + +```bash +# In-process + disposable --force-sim binary (no Docker, no Lium): +./deploy/scripts/proof-submit-e2e.sh --http-tests --local-sim +# Against the compose Proof port: +PROOF_E2E_BASE=http://127.0.0.1:28100 ./deploy/scripts/proof-submit-e2e.sh --probe +``` Compose matrix equivalent (what the script runs): diff --git a/docs/runbooks/proof-submit-e2e.md b/docs/runbooks/proof-submit-e2e.md new file mode 100644 index 000000000..e4fac7b66 --- /dev/null +++ b/docs/runbooks/proof-submit-e2e.md @@ -0,0 +1,273 @@ +# Proof submit → score E2E (staging / local sim) + +Operator check that `POST /v1/submissions` returns a **score** or an **explicit +fail-closed reason**. Healthz alone is not enough. + +**Scope:** staging + disposable local sim. **Not** production. +**Never:** `set_weights`, master Lium rent, commit secrets. + +Staging (operator-ready at time of writing): `can_score=true`, +`PROOF_FORCE_SIM=true`, `baseline_sealed=true`, offer +`openrouter-glm53flash-v0`, open topics `dt-no-ib-v0` and +`muon-vs-adamw-10m-v0`. + +### Ownership: StubWin (A) vs reseal (B) + +Skill-only `sim_document` uses `nll = (3.10 - 0.40 * skill).max(1.0)`. +Even skill=1.0 (and `StubScorer::win` skill=0.95) stays at NLL ≥ 1.0, so +a CPU-sealed ~0.29 baseline always trips `quality_floor`. Prefer **A**. + +| Option | Owner | What | +|--------|--------|------| +| **A (lasting)** | this PR / code | Under `PROOF_FORCE_SIM`, a sealed topic emits harness numbers relative to the seal: holdout ≤ baseline+floor, splits ≤ baseline+`epsilon_topic_max_regress`, `tokens_per_sec` ≥ ref×(1+`epsilon_rel`). No extra host env. Lium never takes this path. | +| **B (ops, paused)** | Développeur | Reseal staging to `BASELINE_SKILL=0.40` (NLL ≈ 2.94), resign topics, retest. **Paused** — Mathis redirected that lane to prod RLM E2E (1× GPU). | + +Do **not** reseal or edit staging host files from the code lane. Deploy A. + +Local compose (`env-local.yml`) defaults `LOCAL_PROOF_FORCE_SIM=true`. +`PROOF_SIM_STUB_WIN` is a leftover no-op. Droplet overlays stay sim-off +(`assert-compose-matrix.sh`). + +## Commands (in-repo, CI-safe) + +```bash +# In-process HTTP contract (Sim + fail-closed 400/503). Always run. +./deploy/scripts/proof-submit-e2e.sh --http-tests + +# Spawn proof-challenge --force-sim with synthetic topic/holdout/baseline/offer. +# Scores both staging topic ids. No Docker, no Lium. +./deploy/scripts/proof-submit-e2e.sh --local-sim + +# Equivalent cargo invocations: +cargo test -p proof-http +cargo test -p proof-challenge-bin --test submit_e2e +cargo test -p ctx -- topic_list_reads_the_items_wrapper +``` + +Pass: every test above is green. Fail: any assertion on silent empty body, +missing `error`, missing `verdict` after 201, or holdout leak. + +## Probe a running host (local compose or staging) + +Do **not** point this at `https://network.cortex.foundation` or +`https://chain.joinbase.ai`. + +```bash +# Auto-detect first healthy origin among loopback + documented staging URLs: +./deploy/scripts/proof-submit-e2e.sh --probe + +# Or pin the origin (no trailing slash). Prefer the droplet IP on :80 — +# staging.api.joinbase.ai has historically answered a stale Lium/fail-closed +# instance while 159.223.159.205/challenge/proof is the ready sim host. +PROOF_E2E_BASE=http://127.0.0.1:28100 ./deploy/scripts/proof-submit-e2e.sh --probe +PROOF_E2E_BASE=http://159.223.159.205/challenge/proof \ + ./deploy/scripts/proof-submit-e2e.sh --probe + +# Same contract as a Rust test (skip if unset): +PROOF_E2E_BASE=http://127.0.0.1:28100 cargo test -p proof-http --test live_submit_e2e +``` + +The probe **skips POST** when `eval_backend=lium` and `can_score=true` +(would rent a miner-paid pod). Staging sim is the intended POST target. + +### Exact curl (Proof) + +Gateway prefix on staging is `/challenge/proof` (reachable on the droplet +at `http://159.223.159.205/challenge/proof`; host-local +`http://127.0.0.1:8080/challenge/proof/...`). Direct service is `:8100` +(local overlay `:28100`). Do not POST to `staging.api.joinbase.ai` while +it still reports `eval_backend=lium`. + +```bash +BASE="${PROOF_E2E_BASE:-http://127.0.0.1:28100}" # or …/challenge/proof + +curl -sS "$BASE/health" +# {"ok":true,"challenge_id":"proof","scoring_version":1} + +curl -sS "$BASE/v1/status" +# { +# "challenge_id": "proof", +# "eval_backend": "sim", +# "force_sim": true, +# "sim_stub_win": true, +# "can_score": true, +# "baseline_sealed": true, +# "open_topics": ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"], +# "inference_offer": { "offer_id": "openrouter-glm53flash-v0", "status": "open", ... } +# } +# Never contains api_key, base_url, or holdout records. + +curl -sS "$BASE/v1/proof/topics" +# { "items": [ { "id": "dt-no-ib-v0", ... }, { "id": "muon-vs-adamw-10m-v0", ... } ] } +# Never contains content_sha256. + +curl -sS -X POST "$BASE/v1/submissions" \ + -H 'content-type: application/json' \ + -d '{ + "miner_hotkey": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "topic_id": "dt-no-ib-v0", + "artifact_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "claim": "beats the sealed reference under the cap", + "declared_flops": 1000000000000, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }' +``` + +### Exact ctx (Proof) + +`ctx` talks to a **gateway** (`/challenge/proof/...`). For local compose: + +```bash +ctx --gateway http://127.0.0.1:8080 proof status +ctx --gateway http://127.0.0.1:8080 proof topics +ctx --gateway http://127.0.0.1:8080 proof submit \ + --hotkey aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --topic-id dt-no-ib-v0 \ + --artifact-digest bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ + --claim "beats the sealed reference under the cap" \ + --declared-flops 1000000000000 \ + --train-dataset e2e-mix-v0 +ctx --gateway http://127.0.0.1:8080 proof show +``` + +On staging, talk to the droplet gateway +(`http://159.223.159.205`, host-local `http://127.0.0.1:8080`). Do not +use `http://staging.api.joinbase.ai` while it still reports +`eval_backend=lium`. Do not send `X-Lium-Api-Key` over `http://` — +`ctx` refuses keyed cleartext. + +Second topic (same body, different id): + +```bash +# topic_id: muon-vs-adamw-10m-v0 +``` + +### Expected response shapes (no secrets) + +**201 scored** (`SubmitResp` then full row on GET): + +```json +{ + "id": "pf_<16 hex>", + "submission_digest": "<64 hex>", + "topic_id": "dt-no-ib-v0", + "state": "awaiting_admin", + "eval_backend": "sim", + "eligible": true +} +``` + +`state` may be `rejected` (gates failed) — that is still a **score**, not +silence. GET `/v1/submissions/{id}` then includes: + +```json +{ + "id": "pf_…", + "topic_id": "dt-no-ib-v0", + "claim": "…", + "declared_flops": 1000000000000, + "state": "awaiting_admin", + "verdict": { + "pass": true, + "agent": { "verdict": "clean", "reproduced": true, "rationale": "sim reproduced", "topic_id": "dt-no-ib-v0" }, + "harness": { "holdout_nll": 0.29, "tokens_per_sec": 84.8 }, + "failed": [], + "lattice": 65535 + }, + "receipt_json": "{\"provider\":\"sim\",…}" +} +``` + +**400 / 503 fail-closed** (no row, no rent): + +```json +{ "error": "topic_id is required" } +{ "error": "unknown topic" } +{ "error": "declared_flops exceeds the topic budget" } +{ "error": "inference offer missing; refuse scoring" } +``` + +A 2xx/4xx/5xx with an empty `{}` and no `id` / no `error` is a **fail**. + +| HTTP | When | Stored? | +|------|------|---------| +| 201 | Sim (or stub) finished judge+harness | yes | +| 400 | bad/missing/unknown/not-open `topic_id`, bad hex, FLOP over budget | no | +| 503 | host cannot score (no open sealed topic, no offer, unpinned Lium, …) | no | + +## Local compose (disposable) + +`env-local.yml` defaults `LOCAL_PROOF_FORCE_SIM=true`. Droplet overlays +(`env-staging.yml` / `env-prod.yml`) must keep `PROOF_FORCE_SIM` (and the +leftover `PROOF_SIM_STUB_WIN`) false; `assert-compose-matrix.sh` fails if +they do not. A staging **host** may already have `PROOF_FORCE_SIM=true` +in `deploy/env/proof-challenge.env` — that is operator state, not a git +overlay. Do not reseal from this lane. + +```bash +./deploy/scripts/materialize-env.sh +./deploy/scripts/local-e2e.sh --smoke --no-tunnel +# soft-probes Proof health + this submit contract when the service is up + +# Minimal cleartext (no testnet): +docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d proof-challenge +# still needs operator files under deploy/secrets/proof/ or submits 503 +# Prefer --local-sim (self-contained fixtures) when those files are absent. +``` + +`local-e2e.sh` will not invent a sha256 digest and will not POST if the +running host is Lium + `can_score`. + +## Bounty smoke (optional) + +```bash +./deploy/scripts/proof-submit-e2e.sh --bounty +# GET /v1/status → scoring_backend, can_score, backend_public_configured +# unconfigured feed: POST /v1/reports → 503 + error (no offline scorer) +# configured feed: thin POST must 401/400/503 — do not file a real report +``` + +## Pass / fail log (fill when you run) + +| Step | Command | Result | +|------|---------|--------| +| In-process Sim submit | `cargo test -p proof-http sim_submit` | PASS (in-repo) | +| StubWin → awaiting_admin | `cargo test -p proof-http sim_stub_win_submit_reaches_awaiting_admin` | PASS (in-repo) | +| Sealed-relative win (0.29 NLL) | `cargo test -p proof-eval stub_win_clears_quality_floor` | PASS (in-repo) | +| Both staging topic ids | `cargo test -p proof-http sim_submit_accepts_staging_topic_ids` | PASS (in-repo) | +| Process-level `--force-sim` | `cargo test -p proof-challenge-bin --test submit_e2e` | PASS (in-repo) | +| Live probe | `PROOF_E2E_BASE=http://159.223.159.205/challenge/proof ./deploy/scripts/proof-submit-e2e.sh --probe` | PASS 201×2 `rejected` (see below) | +| Live Rust | `PROOF_E2E_BASE=http://159.223.159.205/challenge/proof cargo test -p proof-http --test live_submit_e2e` | PASS | + +### Live staging 2026-09-07 (sim, no Lium, no merge) + +Origin: `http://159.223.159.205/challenge/proof` (`eval_backend=sim`, +`force_sim=true`, `can_score=true`, `baseline_sealed=true`, offer +`openrouter-glm53flash-v0` open). Status has **no** `sim_stub_win` field +— host binary predates this PR / env is unset. + +| topic | HTTP | id | state | eligible | gates | +|-------|------|----|-------|----------|-------| +| (empty) | 400 | — | — | — | `topic_id is required` | +| `not-a-real-topic` | 400 | — | — | — | `unknown topic` | +| `dt-no-ib-v0` | 201 | `pf_0000000000000002` | `rejected` | false | QualityFloor holdout 2.827 vs baseline 0.291 floor 0.02; split_regress; ThroughputMiss 113.9 vs 213.4 | +| `muon-vs-adamw-10m-v0` | 201 | `pf_0000000000000003` | `rejected` | false | NllMiss holdout 3.042 vs baseline 0.344 ε 0.02; split_regress | + +Receipts: `"provider":"sim"`. Agent: `clean` / `reproduced`. No rent. +`staging.api.joinbase.ai` still answers `eval_backend=lium` / +`can_score=false` / empty topics — do not POST there. + +To reach `awaiting_admin` on this host (**option A**): deploy this branch +(no host-file edit, no reseal). Status then reports `sim_stub_win=true` +whenever `eval_backend=sim`. Re-run `--probe`. **Option B** (reseal to +`BASELINE_SKILL=0.40` / NLL ≈ 2.94 and resign topics) is Développeur-only +— do not reseal from this lane. Admin adjudicate needs the host bearer at +`/opt/base/deploy/secrets/proof/admin_tokens` (do not log). No +`set_weights`. + +## Related + +- Miner HTTP: [`../external-miner/proof.md`](../external-miner/proof.md) +- Operator Proof: [`../PROOF.md`](../PROOF.md) +- Local stack: [`local-testnet-e2e.md`](local-testnet-e2e.md) +- Staging droplets: [`staging-testnet-e2e.md`](staging-testnet-e2e.md) diff --git a/docs/runbooks/staging-testnet-e2e.md b/docs/runbooks/staging-testnet-e2e.md index 2fc3a2721..7a0335f40 100644 --- a/docs/runbooks/staging-testnet-e2e.md +++ b/docs/runbooks/staging-testnet-e2e.md @@ -117,4 +117,15 @@ git checkout - `FakeChain` is the default backend; `BASE_CHAIN_BACKEND=live` switches to `chain-live`. - CRV4 tlock encryption is implemented (`tle` / Drand Quicknet); when commit-reveal is off, `set_weights` is used instead. -- Proof live submits stay **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. +- Proof live (Lium) submits stay **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. +- When staging is opted into **sim** (`PROOF_FORCE_SIM=true` on the host, not in `env-staging.yml`), submit → score is the contract in [`proof-submit-e2e.md`](proof-submit-e2e.md). Topic ids in that window: `dt-no-ib-v0`, `muon-vs-adamw-10m-v0`. **Option A (code):** a sealed topic under `force_sim` emits a sealed-relative harness (deploy the binary; do not edit host files). **Option B (Développeur, paused):** reseal to `BASELINE_SKILL=0.40` (NLL ≈ 2.94); Mathis redirected that lane to prod RLM E2E. Do not reseal from the code lane. Do not `set_weights`. Do not POST if `eval_backend` is `lium` and `can_score` is true. + +```bash +# From a machine that can reach the staging gateway (no SSH required). +# Prefer the droplet IP: staging.api.joinbase.ai has answered a stale +# Lium/fail-closed instance while 159.223.159.205 is the ready sim host. +PROOF_E2E_BASE=http://159.223.159.205/challenge/proof \ + ./deploy/scripts/proof-submit-e2e.sh --probe +# Host-local: http://127.0.0.1:8080/challenge/proof +# Direct service if published: http://:8100 +```