fix(eval): install host C compiler for Triton on scoring image

.github/workflows/publish-proof-eval-image.yml

@@ -125,6 +125,8 @@ jobs:
             'test ! -L /usr/bin/proof-eval'
           docker run --rm --entrypoint /bin/sh "${ref}" -c \
             'env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help'
+          docker run --rm --entrypoint /bin/sh "${ref}" -c \
+            'test -x /usr/bin/gcc && test -x /usr/bin/g++ && test "$CC" = gcc && test "$CXX" = g++'
           docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \
             -c 'import torch, transformers'
           docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \

eval/Dockerfile.scoring

@@ -2,11 +2,23 @@
 #
 # Scoring image. This is the digest the control plane pins.
 #
-# CUDA Ubuntu base — the live pod is a GPU machine. `/usr/bin/proof-eval` is
-# a regular file (COPY), not a symlink. The image ships no HF bake
-# (`baked_proxies.json` is empty) and enforces the 12.5 Gbit/s / no-IB /
-# no-NVLink / no-NCCL-fast-fabric cap. Harvest stages local weights at
-# `PROOF_PROXY_MODEL_DIR` and holdout shards at `PROOF_HOLDOUT_STORE`.
+# CUDA Ubuntu *runtime* base — the live pod is a GPU machine.
+# `/usr/bin/proof-eval` is a regular file (COPY), not a symlink. The image
+# ships no HF bake (`baked_proxies.json` is empty) and enforces the
+# 12.5 Gbit/s / no-IB / no-NVLink / no-NCCL-fast-fabric cap. Harvest stages
+# local weights at `PROOF_PROXY_MODEL_DIR` and holdout shards at
+# `PROOF_HOLDOUT_STORE`.
+#
+# Triton (pulled in via torch) JIT-compiles CUDA kernels on the first GPU
+# forward after proxy weights load (Qwen SDPA/flash). That needs a host C
+# compiler. We stay on the runtime tag and install `build-essential`
+# (~200MB: gcc/g++) rather than switching to
+# `nvidia/cuda:12.8.1-devel-ubuntu24.04` (several GB: nvcc + CUDA headers).
+# Triton's LLVM backend does not need nvcc for this stack. The live
+# failure on pin `ff21fd98` was "Failed to find C compiler" / exit 1
+# after weight load — not a missing `cuda.h`. If a later kernel compile
+# fails on missing CUDA headers, switch BASE_IMAGE to the matching devel
+# digest (do not invent a sha256).
 #
 #   docker build -f eval/Dockerfile.scoring -t proof-eval:scoring .
 #
@@ -28,13 +40,17 @@ ENV PYTHONUNBUFFERED=1 \
     HF_HUB_DISABLE_TELEMETRY=1 \
     TRANSFORMERS_VERBOSITY=error \
     TOKENIZERS_PARALLELISM=false \
+    CC=gcc \
+    CXX=g++ \
+    CUDAHOSTCXX=g++ \
     PROOF_GIT_SHA=${PROOF_GIT_SHA} \
     PROOF_SELFTEST_REQUIRE_RUNTIME=1
 
 RUN set -eux; \
     apt-get update; \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates curl openssh-server iproute2 \
+        build-essential \
         python3 python3-pip python3-venv; \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tini \
         || curl -fsSL -o /usr/bin/tini \
@@ -47,7 +63,9 @@ RUN set -eux; \
     mkdir -p /run/sshd /root/.ssh /opt/proof-eval/baselines; \
     chmod 700 /root/.ssh; \
     test -x /usr/bin/python3; \
-    test -x /usr/bin/tini
+    test -x /usr/bin/tini; \
+    test -x /usr/bin/gcc; \
+    test -x /usr/bin/g++
 
 WORKDIR /opt/proof-eval
 COPY eval/pyproject.toml ./
@@ -80,6 +98,7 @@ RUN set -eux; \
     test -f /usr/bin/proof-eval; \
     test ! -L /usr/bin/proof-eval; \
     test -x /usr/bin/proof-eval; \
+    env -i PATH=/usr/bin:/bin /usr/bin/gcc --version >/dev/null; \
     env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help >/dev/null; \
     env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help >/dev/null; \
     env -i PATH=/usr/bin:/bin HOME=/root PROOF_SELFTEST_REQUIRE_RUNTIME=1 \

eval/README.md

@@ -13,11 +13,15 @@ Harvest wrappers print `PROOF_METRICS=<document>` and `PROOF_EVAL_OK`.
 non-zero with no marker.
 
 Pin the **scoring** image (`eval/Dockerfile.scoring`, CUDA + torch), never
-the contract-only digest. The RLM **judge** is the live `InferenceOffer`
-(OpenAI-compatible HTTP). Auth is `teacher.env` (`OPENAI_API_KEY` /
-`PROOF_INFERENCE_API_KEY`) staged by harvest — never request.json, never
-git. No HF bake into the judge path. Fabric: no InfiniBand, no NVLink, no
-NCCL fast path, 12.5 Gbit/s cap.
+the contract-only digest. The scoring image installs a host C compiler
+(`build-essential`, `CC=gcc`). Torch/Triton JIT-compiles CUDA kernels on
+the first GPU forward after measurement weights load; without gcc the
+process exits 1 and harvest never prints `PROOF_EVAL_OK` (control plane
+503). The RLM **judge** is the live `InferenceOffer` (OpenAI-compatible
+HTTP). Auth is `teacher.env` (`OPENAI_API_KEY` / `PROOF_INFERENCE_API_KEY`)
+staged by harvest — never request.json, never git. No HF bake into the
+judge path. Fabric: no InfiniBand, no NVLink, no NCCL fast path,
+12.5 Gbit/s cap.
 
 No secrets, holdout text, teacher hosts, or Modal references are baked in.
 The pin ships **no HF bake** and this image does not download a default

eval/src/proof_eval/harness.py

@@ -19,6 +19,22 @@
 SCORED_SPLITS = ("web_ood", "code_ood", "math_ood", "longctx", "multilingual_ood")
 
 
+def _ensure_host_cc() -> None:
+    """Point Triton at gcc when harvest SSH dropped Docker ENV.
+
+    First CUDA kernel compile (SDPA / flash after Qwen weight load) needs a
+    host C compiler. The scoring image installs `build-essential`. This only
+    sets `CC`/`CXX`/`CUDAHOSTCXX` when unset — scores are unchanged. Do not
+    disable flash/SDPA here: that would change tokens/sec and possibly NLL.
+    """
+    if not os.environ.get("CC"):
+        os.environ["CC"] = "gcc"
+    if not os.environ.get("CXX"):
+        os.environ["CXX"] = "g++"
+    if not os.environ.get("CUDAHOSTCXX"):
+        os.environ["CUDAHOSTCXX"] = "g++"
+
+
 def require_runtime() -> None:
     try:
         import torch  # noqa: F401
@@ -70,6 +86,7 @@ def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any]
     The pin ships no HF bake: weights must already be a local directory.
     """
     require_runtime()
+    _ensure_host_cc()
     import torch
     from transformers import AutoModelForCausalLM, AutoTokenizer
 

eval/tests/test_contract.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import os
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
 from threading import Thread
@@ -12,7 +13,7 @@
 from proof_eval.baked import baked_proxies, require_baked
 from proof_eval.contract import DEFAULT_PROXY, METRICS_MARKER, OK_MARKER
 from proof_eval.fabric import DT_NO_IB_GBPS, enforce
-from proof_eval.harness import _shard_text, require_local_model_dir
+from proof_eval.harness import _ensure_host_cc, _shard_text, require_local_model_dir
 from proof_eval.judge import call_judge, load_judge_api_key, require_judge
 from proof_eval.request import Constraints, HarvestRequest, canonical_json, holdout_commitment
 
@@ -34,6 +35,26 @@ def test_unknown_proxy_is_refused() -> None:
         require_baked("Qwen/Qwen3-0.6B")
 
 
+def test_ensure_host_cc_sets_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("CC", raising=False)
+    monkeypatch.delenv("CXX", raising=False)
+    monkeypatch.delenv("CUDAHOSTCXX", raising=False)
+    _ensure_host_cc()
+    assert os.environ["CC"] == "gcc"
+    assert os.environ["CXX"] == "g++"
+    assert os.environ["CUDAHOSTCXX"] == "g++"
+
+
+def test_ensure_host_cc_does_not_override(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("CC", "clang")
+    monkeypatch.setenv("CXX", "clang++")
+    monkeypatch.setenv("CUDAHOSTCXX", "clang++")
+    _ensure_host_cc()
+    assert os.environ["CC"] == "clang"
+    assert os.environ["CXX"] == "clang++"
+    assert os.environ["CUDAHOSTCXX"] == "clang++"
+
+
 def test_local_model_dir_is_required() -> None:
     with pytest.raises(Exception, match="PROOF_PROXY_MODEL_DIR is required"):
         require_local_model_dir(None)