diff --git a/.github/workflows/publish-proof-eval-image.yml b/.github/workflows/publish-proof-eval-image.yml index 2b34cd2af..8c2784857 100644 --- a/.github/workflows/publish-proof-eval-image.yml +++ b/.github/workflows/publish-proof-eval-image.yml @@ -125,6 +125,8 @@ jobs: 'test ! -L /usr/bin/proof-eval' docker run --rm --entrypoint /bin/sh "${ref}" -c \ 'env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help' + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'test -x /usr/bin/gcc && test -x /usr/bin/g++ && test "$CC" = gcc && test "$CXX" = g++' docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \ -c 'import torch, transformers' docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \ diff --git a/eval/Dockerfile.scoring b/eval/Dockerfile.scoring index 890e3f78e..281b200ef 100644 --- a/eval/Dockerfile.scoring +++ b/eval/Dockerfile.scoring @@ -2,11 +2,23 @@ # # Scoring image. This is the digest the control plane pins. # -# CUDA Ubuntu base — the live pod is a GPU machine. `/usr/bin/proof-eval` is -# a regular file (COPY), not a symlink. The image ships no HF bake -# (`baked_proxies.json` is empty) and enforces the 12.5 Gbit/s / no-IB / -# no-NVLink / no-NCCL-fast-fabric cap. Harvest stages local weights at -# `PROOF_PROXY_MODEL_DIR` and holdout shards at `PROOF_HOLDOUT_STORE`. +# CUDA Ubuntu *runtime* base — the live pod is a GPU machine. +# `/usr/bin/proof-eval` is a regular file (COPY), not a symlink. The image +# ships no HF bake (`baked_proxies.json` is empty) and enforces the +# 12.5 Gbit/s / no-IB / no-NVLink / no-NCCL-fast-fabric cap. Harvest stages +# local weights at `PROOF_PROXY_MODEL_DIR` and holdout shards at +# `PROOF_HOLDOUT_STORE`. +# +# Triton (pulled in via torch) JIT-compiles CUDA kernels on the first GPU +# forward after proxy weights load (Qwen SDPA/flash). That needs a host C +# compiler. We stay on the runtime tag and install `build-essential` +# (~200MB: gcc/g++) rather than switching to +# `nvidia/cuda:12.8.1-devel-ubuntu24.04` (several GB: nvcc + CUDA headers). +# Triton's LLVM backend does not need nvcc for this stack. The live +# failure on pin `ff21fd98` was "Failed to find C compiler" / exit 1 +# after weight load — not a missing `cuda.h`. If a later kernel compile +# fails on missing CUDA headers, switch BASE_IMAGE to the matching devel +# digest (do not invent a sha256). # # docker build -f eval/Dockerfile.scoring -t proof-eval:scoring . # @@ -28,6 +40,9 @@ ENV PYTHONUNBUFFERED=1 \ HF_HUB_DISABLE_TELEMETRY=1 \ TRANSFORMERS_VERBOSITY=error \ TOKENIZERS_PARALLELISM=false \ + CC=gcc \ + CXX=g++ \ + CUDAHOSTCXX=g++ \ PROOF_GIT_SHA=${PROOF_GIT_SHA} \ PROOF_SELFTEST_REQUIRE_RUNTIME=1 @@ -35,6 +50,7 @@ RUN set -eux; \ apt-get update; \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates curl openssh-server iproute2 \ + build-essential \ python3 python3-pip python3-venv; \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tini \ || curl -fsSL -o /usr/bin/tini \ @@ -47,7 +63,9 @@ RUN set -eux; \ mkdir -p /run/sshd /root/.ssh /opt/proof-eval/baselines; \ chmod 700 /root/.ssh; \ test -x /usr/bin/python3; \ - test -x /usr/bin/tini + test -x /usr/bin/tini; \ + test -x /usr/bin/gcc; \ + test -x /usr/bin/g++ WORKDIR /opt/proof-eval COPY eval/pyproject.toml ./ @@ -80,6 +98,7 @@ RUN set -eux; \ test -f /usr/bin/proof-eval; \ test ! -L /usr/bin/proof-eval; \ test -x /usr/bin/proof-eval; \ + env -i PATH=/usr/bin:/bin /usr/bin/gcc --version >/dev/null; \ env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help >/dev/null; \ env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help >/dev/null; \ env -i PATH=/usr/bin:/bin HOME=/root PROOF_SELFTEST_REQUIRE_RUNTIME=1 \ diff --git a/eval/README.md b/eval/README.md index 4f42c4f30..36cb67d94 100644 --- a/eval/README.md +++ b/eval/README.md @@ -13,11 +13,15 @@ Harvest wrappers print `PROOF_METRICS=` and `PROOF_EVAL_OK`. non-zero with no marker. Pin the **scoring** image (`eval/Dockerfile.scoring`, CUDA + torch), never -the contract-only digest. The RLM **judge** is the live `InferenceOffer` -(OpenAI-compatible HTTP). Auth is `teacher.env` (`OPENAI_API_KEY` / -`PROOF_INFERENCE_API_KEY`) staged by harvest — never request.json, never -git. No HF bake into the judge path. Fabric: no InfiniBand, no NVLink, no -NCCL fast path, 12.5 Gbit/s cap. +the contract-only digest. The scoring image installs a host C compiler +(`build-essential`, `CC=gcc`). Torch/Triton JIT-compiles CUDA kernels on +the first GPU forward after measurement weights load; without gcc the +process exits 1 and harvest never prints `PROOF_EVAL_OK` (control plane +503). The RLM **judge** is the live `InferenceOffer` (OpenAI-compatible +HTTP). Auth is `teacher.env` (`OPENAI_API_KEY` / `PROOF_INFERENCE_API_KEY`) +staged by harvest — never request.json, never git. No HF bake into the +judge path. Fabric: no InfiniBand, no NVLink, no NCCL fast path, +12.5 Gbit/s cap. No secrets, holdout text, teacher hosts, or Modal references are baked in. The pin ships **no HF bake** and this image does not download a default diff --git a/eval/src/proof_eval/harness.py b/eval/src/proof_eval/harness.py index d89356dde..423110287 100644 --- a/eval/src/proof_eval/harness.py +++ b/eval/src/proof_eval/harness.py @@ -19,6 +19,22 @@ SCORED_SPLITS = ("web_ood", "code_ood", "math_ood", "longctx", "multilingual_ood") +def _ensure_host_cc() -> None: + """Point Triton at gcc when harvest SSH dropped Docker ENV. + + First CUDA kernel compile (SDPA / flash after Qwen weight load) needs a + host C compiler. The scoring image installs `build-essential`. This only + sets `CC`/`CXX`/`CUDAHOSTCXX` when unset — scores are unchanged. Do not + disable flash/SDPA here: that would change tokens/sec and possibly NLL. + """ + if not os.environ.get("CC"): + os.environ["CC"] = "gcc" + if not os.environ.get("CXX"): + os.environ["CXX"] = "g++" + if not os.environ.get("CUDAHOSTCXX"): + os.environ["CUDAHOSTCXX"] = "g++" + + def require_runtime() -> None: try: import torch # noqa: F401 @@ -70,6 +86,7 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any] The pin ships no HF bake: weights must already be a local directory. """ require_runtime() + _ensure_host_cc() import torch from transformers import AutoModelForCausalLM, AutoTokenizer diff --git a/eval/tests/test_contract.py b/eval/tests/test_contract.py index 3d9c40385..43d461981 100644 --- a/eval/tests/test_contract.py +++ b/eval/tests/test_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from threading import Thread @@ -12,7 +13,7 @@ from proof_eval.baked import baked_proxies, require_baked from proof_eval.contract import DEFAULT_PROXY, METRICS_MARKER, OK_MARKER from proof_eval.fabric import DT_NO_IB_GBPS, enforce -from proof_eval.harness import _shard_text, require_local_model_dir +from proof_eval.harness import _ensure_host_cc, _shard_text, require_local_model_dir from proof_eval.judge import call_judge, load_judge_api_key, require_judge from proof_eval.request import Constraints, HarvestRequest, canonical_json, holdout_commitment @@ -34,6 +35,26 @@ def test_unknown_proxy_is_refused() -> None: require_baked("Qwen/Qwen3-0.6B") +def test_ensure_host_cc_sets_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CC", raising=False) + monkeypatch.delenv("CXX", raising=False) + monkeypatch.delenv("CUDAHOSTCXX", raising=False) + _ensure_host_cc() + assert os.environ["CC"] == "gcc" + assert os.environ["CXX"] == "g++" + assert os.environ["CUDAHOSTCXX"] == "g++" + + +def test_ensure_host_cc_does_not_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CC", "clang") + monkeypatch.setenv("CXX", "clang++") + monkeypatch.setenv("CUDAHOSTCXX", "clang++") + _ensure_host_cc() + assert os.environ["CC"] == "clang" + assert os.environ["CXX"] == "clang++" + assert os.environ["CUDAHOSTCXX"] == "clang++" + + def test_local_model_dir_is_required() -> None: with pytest.raises(Exception, match="PROOF_PROXY_MODEL_DIR is required"): require_local_model_dir(None)