diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 983c52f..8430078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,9 +174,19 @@ jobs: - name: Set up Rust toolchain uses: ./.github/actions/setup-rust - name: Run the e2e suite + # Its own timeout, below the job's, so a hang still leaves time to upload the job dirs. + timeout-minutes: 30 run: scripts/e2e.sh env: E2E_REQUIRE_NFT: "1" + E2E_JOBS_DIR: ${{ runner.temp }}/e2e-jobs + - name: Upload the e2e job dirs + if: failure() || cancelled() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-jobs + path: ${{ runner.temp }}/e2e-jobs + retention-days: 7 hbench-action-smoke-test: name: hbench-action smoke test diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 6cfd79c..72d02f4 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -5,7 +5,10 @@ set -euo pipefail cd "$(dirname "$0")/.." cargo build -q BIN=target/debug/hbench -JOBS=$(mktemp -d) +# E2E_JOBS_DIR keeps the job dirs somewhere known (CI uploads it when a run fails). +JOBS=${E2E_JOBS_DIR:-$(mktemp -d)} +mkdir -p "$JOBS" +trap 'rc=$?; [ $rc = 0 ] || echo "e2e failed (exit $rc); job dirs: $JOBS" >&2' EXIT $BIN run -p tests/fixtures -a oracle -n 4 -k 2 --jobs-dir "$JOBS" --job-name fixtures @@ -23,20 +26,31 @@ expect = { "no-network": ({"reward": 1.0}, None), "compose-sidecar": ({"reward": 1.0}, None), "prebuilt": ({"reward": 1.0}, None), + "planted-reward": ({"reward": 1.0}, None), } -seen = {} -for r in job.glob("*/result.json"): +def explain(trial): + """What a failed trial left behind: its exception, the end of its log, and the verifier's output.""" + out = [] + for name in ("exception.txt", "trial.log", "verifier/test-stdout.txt"): + p = trial / name + if p.exists(): + out.append(f"--- {name} (tail)\n" + "\n".join(p.read_text(errors="replace").splitlines()[-30:])) + return "\n".join(out) +seen, bad = {}, [] +for r in sorted(job.glob("*/result.json")): d = json.loads(r.read_text()) task = d["task_name"] rewards = (d["verifier_result"] or {}).get("rewards") exc = (d["exception_info"] or {}).get("exception_type") want_r, want_e = expect[task] - assert (rewards, exc) == (want_r, want_e), f"{task}: got {rewards} {exc}, want {want_r} {want_e}" + if (rewards, exc) != (want_r, want_e): + bad.append(f"{r.parent.name}: got {rewards} {exc}, want {want_r} {want_e}\n{explain(r.parent)}") seen[task] = seen.get(task, 0) + 1 +assert not bad, "\n\n".join(bad) assert seen == {t: 2 for t in expect}, seen res = json.loads((job / "result.json").read_text()) s = res["stats"] -assert res["finished_at"] and s["n_completed_trials"] == 20 and s["n_errored_trials"] == 8, s +assert res["finished_at"] and s["n_completed_trials"] == 22 and s["n_errored_trials"] == 8, s ev = s["evals"]["oracle__fixtures"] assert ev["exception_stats"].keys() == {"AgentTimeoutError", "RewardFileNotFoundError"}, ev print("fixture assertions OK") @@ -60,9 +74,14 @@ EOF # Docker Desktop included; E2E_REQUIRE_NFT insists on it) and under the proxy-only fallback. if docker info --format '{{.OperatingSystem}}' | grep -q 'Docker Desktop'; then model_host=host.docker.internal; else model_host=172.17.0.1; fi model_port=18080 -python3 -m http.server "$model_port" --bind 0.0.0.0 --directory "$JOBS" >/dev/null 2>&1 & +# An empty dir: the stand-in must not serve the job dirs to whoever can reach the port. +mkdir -p "$JOBS/model-standin" +python3 -m http.server "$model_port" --bind 0.0.0.0 --directory "$JOBS/model-standin" >"$JOBS/model-standin.log" 2>&1 & model_pid=$! -trap 'kill $model_pid 2>/dev/null' EXIT +trap 'rc=$?; kill $model_pid 2>/dev/null; [ $rc = 0 ] || echo "e2e failed (exit $rc); job dirs: $JOBS" >&2' EXIT +# A stand-in that never came up would read as blocked, not as a pass, but fail here with the reason. +for _ in $(seq 50); do curl -fs -o /dev/null "http://127.0.0.1:$model_port/" && break; sleep 0.1; done +curl -fs -o /dev/null "http://127.0.0.1:$model_port/" || { cat "$JOBS/model-standin.log"; exit 1; } default_method=${E2E_REQUIRE_NFT:+nft+proxy} runs="false true true:proxy" for run in $runs; do diff --git a/src/diff.rs b/src/diff.rs index 8d595c2..0fcbebd 100644 --- a/src/diff.rs +++ b/src/diff.rs @@ -123,25 +123,31 @@ fn without_task(v: &Value) -> Value { v } +/// The name a target's results will carry (`[task].name`, else the dir), which is what sources match on. +/// The lock's name is always the dir, so it can't be compared with `result.task_name`. +fn target_task_name(c: &TrialConfig, lock: &Value) -> String { + crate::task::Task::load(&crate::registry::local_path(&c.task)) + .map(|t| t.name()) + .unwrap_or_else(|_| lock["task"]["name"].as_str().unwrap_or_default().to_string()) +} + pub fn compare(targets: &[TrialConfig], sources: &[Source]) -> Result> { let locks: Vec = targets.iter().map(trial_lock).collect(); - for (c, l) in targets.iter().zip(&locks) { - let name = l["task"]["name"].as_str().unwrap_or_default(); + let names: Vec = targets.iter().zip(&locks).map(|(c, l)| target_task_name(c, l)).collect(); + for (l, name) in locks.iter().zip(&names) { if let Some(v) = l["task"]["version"].as_str() { semver(v)?; - for s in sources.iter().filter(|s| s.result.task_name == name && s.version() == Some(v)) { + for s in sources.iter().filter(|s| s.result.task_name == *name && s.version() == Some(v)) { if s.lock["task"]["digest"] != l["task"]["digest"] { bail!("Task '{name}' has version {v} in both source and target locks, but the digests differ."); } } } - let _ = c; } let mut consumed: HashSet = HashSet::new(); let mut regradable: BTreeMap> = BTreeMap::new(); let mut out = vec![]; - for (c, l) in targets.iter().zip(&locks) { - let task_name = l["task"]["name"].as_str().unwrap_or_default().to_string(); + for ((c, l), task_name) in targets.iter().zip(&locks).zip(names) { let rerun = |reason: String, downgraded: bool| TrialDiff { action: Action::Rerun, target_trial_name: c.trial_name.clone(), @@ -271,4 +277,25 @@ mod tests { assert_ne!(without_task(&lock("0.2.5", None)), without_task(&lock("0.2.7", Some("0.2")))); assert_ne!(without_task(&lock("0.2.5", None)), without_task(&lock("0.2.6", None))); } + + #[test] + fn a_named_task_is_reused_from_its_earlier_trial() { + // hello-world's [task].name is "hbench/hello-world", not its dir name. + let task = crate::config::TaskRef { path: Some("examples/tasks/hello-world".into()), ..Default::default() }; + let target = TrialConfig::new(task, "hello-world__new".into(), "jobs/new".into()); + let lock = trial_lock(&target); + let result: TrialResult = serde_json::from_value(serde_json::json!({ + "id": uuid::Uuid::new_v4(), "task_name": "hbench/hello-world", "trial_name": "hello-world__old", + "trial_uri": "", "task_id": null, "source": null, "task_checksum": "", "config": target, + "agent_info": {"name": "oracle", "version": "1", "model_info": null}, + "agent_result": null, "verifier_result": {"rewards": {"reward": 1}}, "exception_info": null, + "started_at": "2026-01-01T00:00:00Z", "finished_at": "2026-01-01T00:01:00Z", + "environment_setup": null, "agent_setup": null, "agent_execution": null, "verifier": null, + })) + .unwrap(); + let sources = [Source { dir: "jobs/old/hello-world__old".into(), result, lock }]; + let d = compare(&[target], &sources).unwrap(); + assert_eq!(d[0].action, Action::Reuse, "{}", d[0].reason); + assert_eq!(d[0].task_name, "hbench/hello-world"); + } } diff --git a/src/env.rs b/src/env.rs index aed7774..248b18b 100644 --- a/src/env.rs +++ b/src/env.rs @@ -328,6 +328,8 @@ fn egress_compose( ) -> Result { let mut sidecar = json!({ "image": image, + // Built locally; a missing image must not turn into an anonymous Docker Hub pull. + "pull_policy": "never", "entrypoint": ["/opt/egress-sidecar/entrypoint.sh"], "init": true, "environment": { @@ -473,9 +475,13 @@ fn ssh_not_connected(r: &ExecResult) -> bool { r.code != 0 && e.contains("dial-stdio") && pre_session } -/// Pull/registry/daemon failures worth another `up`, as opposed to errors in the task's compose setup. +/// Pull/registry/builder/daemon failures worth another `up` or `build`, as opposed to errors in the +/// task's compose setup or Dockerfile. fn transient_docker_error(stderr: &str) -> bool { let s = stderr.to_ascii_lowercase(); + if s.contains("no space left on device") { + return false; + } [ "toomanyrequests", "too many requests", @@ -488,6 +494,11 @@ fn transient_docker_error(stderr: &str) -> bool { "temporary failure in name resolution", "503 service unavailable", "502 bad gateway", + "504 gateway timeout", + "500 internal server error", + // BuildKit losing its builder or connection mid-build (e.g. another job switched or removed it). + "graceful_stop", + "code = unavailable", ] .iter() .any(|p| s.contains(p)) @@ -602,45 +613,42 @@ async fn docker_output(args: &[&str], timeout: Duration) -> Result Result { - static IMAGE: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); - IMAGE - .get_or_try_init(|| async { - let files: [(&str, &str, u32); 5] = [ - ("Dockerfile", include_str!("../assets/egress-sidecar/Dockerfile"), 0o644), - ("gost.yaml", include_str!("../assets/egress-sidecar/gost.yaml"), 0o644), - ("allowlist.txt", "", 0o644), - ("entrypoint.sh", include_str!("../assets/egress-sidecar/entrypoint.sh"), 0o755), - ("bin/network-policy", include_str!("../assets/egress-sidecar/bin/network-policy"), 0o755), - ]; - let mut h = sha2::Sha256::default(); - for (n, c, _) in files { - sha2::Digest::update(&mut h, n.as_bytes()); - sha2::Digest::update(&mut h, c.as_bytes()); - } - let hash = crate::task::hex(&sha2::Digest::finalize(h))[..12].to_string(); - let image = format!("hbench-egress-sidecar:{hash}"); - let exists = docker_output(&["image", "inspect", &image], QUICK_COMMAND_TIMEOUT).await?.status.success(); - if !exists { - let dir = cache_root().join("egress-sidecar").join(&hash); - std::fs::create_dir_all(dir.join("bin"))?; - for (n, c, mode) in files { - let p = dir.join(n); - std::fs::write(&p, c)?; - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&p, std::fs::Permissions::from_mode(mode))?; - } - log.line(&format!("building egress sidecar image {image}")); - let out = docker_output(&["build", "-t", &image, &dir.to_string_lossy()], SIDECAR_BUILD_TIMEOUT).await?; - if !out.status.success() { - bail!("building egress sidecar failed: {}", String::from_utf8_lossy(&out.stderr)); - } - } - Ok(image) - }) - .await - .cloned() + static BUILD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + let _guard = BUILD.lock().await; + let files: [(&str, &str, u32); 5] = [ + ("Dockerfile", include_str!("../assets/egress-sidecar/Dockerfile"), 0o644), + ("gost.yaml", include_str!("../assets/egress-sidecar/gost.yaml"), 0o644), + ("allowlist.txt", "", 0o644), + ("entrypoint.sh", include_str!("../assets/egress-sidecar/entrypoint.sh"), 0o755), + ("bin/network-policy", include_str!("../assets/egress-sidecar/bin/network-policy"), 0o755), + ]; + let mut h = sha2::Sha256::default(); + for (n, c, _) in files { + sha2::Digest::update(&mut h, n.as_bytes()); + sha2::Digest::update(&mut h, c.as_bytes()); + } + let hash = crate::task::hex(&sha2::Digest::finalize(h))[..12].to_string(); + let image = format!("hbench-egress-sidecar:{hash}"); + let exists = docker_output(&["image", "inspect", &image], QUICK_COMMAND_TIMEOUT).await?.status.success(); + if !exists { + let dir = cache_root().join("egress-sidecar").join(&hash); + std::fs::create_dir_all(dir.join("bin"))?; + for (n, c, mode) in files { + let p = dir.join(n); + std::fs::write(&p, c)?; + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(mode))?; + } + log.line(&format!("building egress sidecar image {image}")); + let out = docker_output(&["build", "-t", &image, &dir.to_string_lossy()], SIDECAR_BUILD_TIMEOUT).await?; + if !out.status.success() { + bail!("building egress sidecar failed: {}", String::from_utf8_lossy(&out.stderr)); + } + } + Ok(image) } pub fn cache_root() -> PathBuf { @@ -884,13 +892,16 @@ impl DockerEnv { let started = async move { if let Some(mut built) = built { if !*built { - self.run_ok(to_owned_args(&["build"]), None).await?; + self.retrying(&["build"]).await?; *built = true; } } else if self.spec.env_dir.join("docker-compose.yaml").exists() { - self.run_ok(to_owned_args(&["build"]), None).await?; + self.retrying(&["build"]).await?; } let _ = self.run(to_owned_args(&["down", "--remove-orphans"]), Some(STOP_TIMEOUT), "down (stale)", None).await; + if self.egress.is_some() { + egress_image(&self.log).await?; + } self.up().await?; if self.egress == Some(EgressMethod::Proxy) && self.spec.baseline.mode != NetworkMode::Public { self.check_isolated().await?; @@ -927,17 +938,29 @@ impl DockerEnv { Ok(()) } - /// `up --detach --wait`, retried when a pull or the daemon fails transiently. async fn up(&self) -> Result<()> { + self.retrying(&["up", "--detach", "--wait"]).await + } + + /// `docker compose args`, retried when a pull, the builder or the daemon fails transiently. + async fn retrying(&self, args: &[&str]) -> Result<()> { + let label = args.join(" "); for attempt in 1.. { - let r = self.run(to_owned_args(&["up", "--detach", "--wait"]), None, "up --detach --wait", None).await?; + let r = self.run(to_owned_args(args), None, &label, None).await?; if r.code == 0 { break; } - if attempt == 3 || !transient_docker_error(&r.stderr) { - bail!("docker compose up --detach --wait failed (exit {}): {}{}", r.code, tail(&r.stdout, 2000), tail(&r.stderr, 4000)); + // Pruned between the check in start() and here: rebuild it and go again. + let sidecar_gone = self.egress.is_some() + && r.stderr.contains("hbench-egress-sidecar") + && r.stderr.to_ascii_lowercase().contains("no such image"); + if attempt == 3 || !(sidecar_gone || transient_docker_error(&r.stderr)) { + bail!("docker compose {label} failed (exit {}): {}{}", r.code, tail(&r.stdout, 2000), tail(&r.stderr, 4000)); + } + if sidecar_gone { + egress_image(&self.log).await?; } - self.log.line(&format!("transient docker error; retrying `up` (attempt {})", attempt + 1)); + self.log.line(&format!("transient docker error; retrying `{label}` (attempt {})", attempt + 1)); tokio::time::sleep(Duration::from_secs(10 * attempt)).await; } Ok(()) @@ -1019,15 +1042,27 @@ impl DockerEnv { pub async fn download_dir(&self, service: &str, src: &str, dst: &Path) -> Result<()> { std::fs::create_dir_all(dst)?; self.run_ok(vec!["cp".into(), format!("{service}:{}/.", src.trim_end_matches('/')), dst.display().to_string()], Some(COPY_TIMEOUT)) - .await - .map(|_| ()) + .await?; + self.strip_links(dst); + Ok(()) + } + + /// `docker cp` keeps container symlinks (absolute targets intact) and special files; hbench's own + /// host readers and writers would follow them out of the trial dir, so nothing but files and dirs stays. + fn strip_links(&self, root: &Path) { + let n = strip_links(root); + if n > 0 { + self.log.line(&format!("removed {n} symlink(s) or special file(s) from {}", root.display())); + } } pub async fn download_file(&self, service: &str, src: &str, dst: &Path) -> Result<()> { if let Some(p) = dst.parent() { std::fs::create_dir_all(p)?; } - self.run_ok(vec!["cp".into(), format!("{service}:{src}"), dst.display().to_string()], Some(COPY_TIMEOUT)).await.map(|_| ()) + self.run_ok(vec!["cp".into(), format!("{service}:{src}"), dst.display().to_string()], Some(COPY_TIMEOUT)).await?; + self.strip_links(dst); + Ok(()) } /// Tar inside the service with `--exclude` patterns, copy the archive out, extract locally. @@ -1048,6 +1083,7 @@ impl DockerEnv { if !out.status.success() { bail!("extracting artifact archive failed: {}", String::from_utf8_lossy(&out.stderr)); } + self.strip_links(dst); Ok(()) } .await; @@ -1267,6 +1303,20 @@ const PROC_LIST: &str = r#"for_each_proc() { /// $3 pgrp, $4 session, $5 tty, $6 terminal foreground pgrp, $20 starttime. A `docker exec` process has /// ppid 0 inside the container; an orphan is reparented to PID 1. Everything picked is stopped first, and /// the scan repeats until it finds nothing new, so no child is orphaned (and so spared) mid-way. +/// Remove every symlink and non-regular file at or under `root` (links are not followed). Returns the count. +fn strip_links(root: &Path) -> usize { + let Ok(meta) = root.symlink_metadata() else { return 0 }; + let t = meta.file_type(); + if t.is_dir() { + let Ok(entries) = std::fs::read_dir(root) else { return 0 }; + entries.flatten().map(|e| strip_links(&e.path())).sum() + } else if t.is_file() { + 0 + } else { + usize::from(std::fs::remove_file(root).is_ok()) + } +} + fn kill_agent_script(snapshot: &str) -> String { format!( r#"keep=" {snapshot} " @@ -1350,6 +1400,23 @@ mod tests { use super::*; + #[test] + fn strip_links_keeps_only_files_and_dirs() { + let d = std::env::temp_dir().join(format!("hbench-strip-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(d.join("sub")).unwrap(); + std::fs::write(d.join("sub/keep.txt"), "x").unwrap(); + std::os::unix::fs::symlink("/etc/hosts", d.join("trajectory.json")).unwrap(); + std::os::unix::fs::symlink("/etc", d.join("sub/etc")).unwrap(); + std::os::unix::fs::symlink("keep.txt", d.join("sub/rel")).unwrap(); + assert!(std::process::Command::new("mkfifo").arg(d.join("reward.json")).status().unwrap().success()); + assert_eq!(strip_links(&d), 4); + assert!(d.join("sub/keep.txt").is_file()); + for gone in ["trajectory.json", "sub/etc", "sub/rel", "reward.json"] { + assert!(d.join(gone).symlink_metadata().is_err(), "{gone} survived"); + } + std::fs::remove_dir_all(&d).unwrap(); + } + /// What `docker compose exec ... main bash -c EXEC_FROM_STDIN` runs, with the same stdin. fn run_from_stdin(env: &[(&str, &str)], command: &str) -> (i32, String) { use std::io::Write as _; @@ -1430,6 +1497,10 @@ mod tests { assert!(transient_docker_error("Error response from daemon: toomanyrequests: You have reached your pull rate limit")); assert!(transient_docker_error("net/http: TLS handshake timeout")); assert!(!transient_docker_error("service \"main\" refers to undefined network foo: invalid compose project")); + assert!(transient_docker_error( + "ERROR: failed to solve: rpc error: code = Unavailable desc = closing transport due to: graceful_stop" + )); + assert!(!transient_docker_error("failed to copy: write /var/lib/docker/tmp/x: no space left on device: unexpected EOF")); } #[test] diff --git a/src/trial.rs b/src/trial.rs index 47b39b3..43c0f96 100644 --- a/src/trial.rs +++ b/src/trial.rs @@ -996,6 +996,11 @@ impl Trial { } if !self.cfg.verifier.disable { let t = TimingInfo::start(); + if !separate { + // The agent could write here; a planted reward.json would outrank test.sh's reward.txt. + self.env().empty_dirs(&["/logs/verifier"], true).await?; + self.env().empty_dirs(&["/tests"], false).await?; + } let r = self.verify(None, &self.dir.join("artifacts")).await; self.result.verifier = Some(t.finish()); if !separate { diff --git a/tests/egress/public-task/solution/solve.sh b/tests/egress/public-task/solution/solve.sh index 1a749e5..dad7688 100755 --- a/tests/egress/public-task/solution/solve.sh +++ b/tests/egress/public-task/solution/solve.sh @@ -1,7 +1,8 @@ #!/bin/bash # Run under an allowlist of pypi.org, *.python.org and the model host ($MODEL_PROBE, host:port). # Bash only (the image has no curl). A bare TCP connect can succeed under the egress proxy; -# only a real HTTP exchange proves reachability. +# only a real HTTP answer proves reachability. The proxy answers 403 for a denied host and 5xx when +# it can't reach an allowed one, so only what a live server sends (2xx, 3xx, 404) counts. probe() { local host=$1 port=$2 via_host=$1 via_port=$2 target=/ status if [ -n "${http_proxy:-}" ]; then @@ -10,7 +11,7 @@ probe() { via_host=${hp%:*} via_port=${hp##*:} target="http://$host:$port/" fi status=$(timeout 8 bash -c "exec 3<>/dev/tcp/$via_host/$via_port; printf 'GET $target HTTP/1.0\r\nHost: $host\r\n\r\n' >&3; head -n1 <&3" 2>/dev/null | awk '{print $2}') - case "$status" in '' | 403) echo "$host blocked" ;; *) echo "$host reachable" ;; esac + case "$status" in 2?? | 3?? | 404) echo "$host reachable" ;; *) echo "$host blocked" ;; esac } # A DNS query over TCP straight to 1.1.1.1, past any proxy: no answer means it was blocked. dns() { diff --git a/tests/fixtures/planted-reward/environment/Dockerfile b/tests/fixtures/planted-reward/environment/Dockerfile new file mode 100644 index 0000000..51412a6 --- /dev/null +++ b/tests/fixtures/planted-reward/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu:24.04 +WORKDIR /app diff --git a/tests/fixtures/planted-reward/instruction.md b/tests/fixtures/planted-reward/instruction.md new file mode 100644 index 0000000..d40529a --- /dev/null +++ b/tests/fixtures/planted-reward/instruction.md @@ -0,0 +1 @@ +Do nothing diff --git a/tests/fixtures/planted-reward/solution/solve.sh b/tests/fixtures/planted-reward/solution/solve.sh new file mode 100755 index 0000000..73a5486 --- /dev/null +++ b/tests/fixtures/planted-reward/solution/solve.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Leave grading files for the verifier: a reward.json would outrank test.sh's reward.txt. +echo '{"reward": 1, "planted": 1}' > /logs/verifier/reward.json +mkdir -p /tests && echo planted > /tests/planted diff --git a/tests/fixtures/planted-reward/task.toml b/tests/fixtures/planted-reward/task.toml new file mode 100644 index 0000000..7800f84 --- /dev/null +++ b/tests/fixtures/planted-reward/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 60.0 diff --git a/tests/fixtures/planted-reward/tests/test.sh b/tests/fixtures/planted-reward/tests/test.sh new file mode 100755 index 0000000..16e569c --- /dev/null +++ b/tests/fixtures/planted-reward/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Both planted files must be gone before grading; the real verdict is reward.txt. +if [ -e /logs/verifier/reward.json ] || [ -e /tests/planted ]; then echo 0; else echo 1; fi > /logs/verifier/reward.txt