fix(harbor): build the runtime image from a clean clone of HEAD

.dockerignore

@@ -1,12 +1,17 @@
 # Docker reads THIS file, not .gitignore. Anything not listed here is pulled
 # into the build context by `COPY . .` in benchmarks/harbor/base-image/Dockerfile
 # and baked into the runtime image, including files git ignores.
+#
+# benchmarks/harbor/scripts/build-runtime-image.sh builds from `git archive HEAD`,
+# so untracked files never reach it, and base-image/Dockerfile refuses a build
+# that does not come through it. This file still applies to that context.
 
 # Credentials. AssetOpsBench's run_sdk_cli calls load_dotenv(), so a local .env
 # would otherwise end up inside every task image and every trial container.
-.env
-.env.*
-!.env.public
+# Unlike .gitignore, a bare name matches only at the context root, hence **/.
+**/.env
+**/.env.*
+!**/.env.public
 
 # Host virtualenv: the image builds its own with `uv sync --frozen`.
 .venv/
@@ -28,6 +33,25 @@ benchmarks/harbor/datasets/
 traces/
 .DS_Store
 
-# NOTE: .git is deliberately NOT ignored. StirrupAgent.get_version_command runs
-# `git rev-parse --short HEAD` inside the container so every trial records the
-# AssetOpsBench commit it ran against in result.json.
+# Answers. `main` is the agent's container, and the verifier reads the copies
+# Harbor uploads to /tests after the agent phase, never these. Every other file
+# of a scenario folder stays: manifest.json is what init_data.py loads.
+src/couchdb/scenarios_data/**/groundtruth*
+src/couchdb/scenarios_data/**/rubric.json
+src/couchdb/scenarios_data/**/reference_answer.json
+src/couchdb/scenarios_data/**/scenario_meta.json
+
+# Local work that is not part of the benchmark. reports/ once carried a results
+# table with a ground_truth column for private scenarios into the image.
+reports/
+logs/
+src/tmp/
+.claude/
+notebook/kdd_tutorial/
+artifacts/kdd_tutorial/
+
+# Git data. Even a one-commit .git holds every committed blob, answers included
+# (`git show HEAD:src/couchdb/scenarios_data/scenario_1/groundtruth.txt`), and
+# its reflog names whoever built the image. The commit reaches the image as
+# /opt/aob/.aob-commit instead, from the AOB_COMMIT build arg.
+.git

benchmarks/harbor/CODE-SANDBOX.md

@@ -57,7 +57,7 @@ Treat anything other than local Docker as untested for this overlay.
 The runtime image:
 
 ```bash
-docker build -t assetopsbench/runtime:dev -f benchmarks/harbor/base-image/Dockerfile .
+bash benchmarks/harbor/scripts/build-runtime-image.sh
 ```
 
 A `.env` with your model credentials. `harbor run` is invoked through

benchmarks/harbor/QUICKSTART.md

@@ -33,10 +33,13 @@ docker tag assetopsbench/runtime:latest assetopsbench/runtime:dev
 Or build it yourself, which takes a few minutes and needs no registry:
 
 ```bash
-docker build -t assetopsbench/runtime:dev \
-  -f benchmarks/harbor/base-image/Dockerfile .
+bash benchmarks/harbor/scripts/build-runtime-image.sh
 ```
 
+The script builds `assetopsbench/runtime:dev` from `git archive HEAD`, not
+from your working tree, so untracked files such as local results never reach
+the container the agent runs in. Commit a change first to include it.
+
 Either way the local tag `assetopsbench/runtime:dev` is what the task
 Dockerfiles reference, through the `AOB_RUNTIME_IMAGE` build arg in
 `benchmarks/harbor/template/environment/Dockerfile`. Change that default if you
@@ -170,7 +173,8 @@ and check `docker images assetopsbench/runtime`.
 
 **`No module named 'google.protobuf'` during a run** — the image was built
 without the `otel` dependency group, which the file trace exporter needs.
-Rebuild from `benchmarks/harbor/base-image/Dockerfile`, which passes it.
+Rebuild with `benchmarks/harbor/scripts/build-runtime-image.sh`, whose
+Dockerfile passes it.
 
 **The verifier scores 0 but the agent clearly answered** — check
 `verifier/test-stderr.txt`. The evaluator joins records to scenarios on

benchmarks/harbor/README.md

@@ -39,11 +39,11 @@ Prerequisites: Docker running, and `uv sync --dev --extra harbor`. Run every
 command from the repo root.
 
 ```bash
-# 1. Build the runtime base (once per AssetOpsBench commit). The tag is local
-#    and is the default AOB_RUNTIME_IMAGE in template/environment/Dockerfile;
-#    nothing is pulled or pushed.
-docker build -t assetopsbench/runtime:dev \
-  -f benchmarks/harbor/base-image/Dockerfile .
+# 1. Build the runtime base (once per AssetOpsBench commit). The script builds
+#    from `git archive HEAD`, so untracked files and uncommitted changes stay
+#    out of the image. The tag is local and is the default AOB_RUNTIME_IMAGE in
+#    template/environment/Dockerfile; nothing is pulled or pushed.
+bash benchmarks/harbor/scripts/build-runtime-image.sh
 
 # 2. Generate one task per scenario in the open profile. The defaults point at
 #    src/couchdb/scenarios_data, benchmarks/scenario_suite/open.yaml and

benchmarks/harbor/base-image/Dockerfile

@@ -6,8 +6,25 @@
 # the whole environment/ directory, so a fat per-task context defeats the cache
 # and rebuilds the world for every scenario.
 #
-#   docker build -t assetopsbench/runtime:$(git rev-parse --short HEAD) \
-#     -f base-image/Dockerfile <path-to-AssetOpsBench>
+#   bash benchmarks/harbor/scripts/build-runtime-image.sh \
+#     -t assetopsbench/runtime:$(git rev-parse --short HEAD)
+#
+# The script builds from `git archive HEAD`, so `COPY . .` below sees committed
+# files only, and passes that commit as AOB_COMMIT. A plain `docker build` of the
+# working tree would also copy untracked files, which .dockerignore alone cannot
+# anticipate, so without AOB_COMMIT the `commit` stage fails the build.
+
+# Its own stage so the check fails fast without the ARG reaching the stage
+# below: every RUN after an ARG misses the cache when its value changes, and
+# AOB_COMMIT changes with every commit.
+FROM python:3.12-slim AS commit
+ARG AOB_COMMIT
+RUN test -n "$AOB_COMMIT" || { \
+        echo "build with benchmarks/harbor/scripts/build-runtime-image.sh," \
+             "not docker build: it builds HEAD, never the working tree" >&2; \
+        exit 1; } \
+    && echo "$AOB_COMMIT" > /aob-commit
+
 FROM python:3.12-slim
 
 # HF_HOME fixes the Hugging Face cache at one path. The default follows $HOME,
@@ -52,4 +69,8 @@ RUN uv sync --frozen --group otel --extra tsfm
 RUN uv run python benchmarks/harbor/scripts/preload_models.py \
         --from-list benchmarks/harbor/base-image/models.txt --download
 
+# The image has no .git (see .dockerignore): the commit it was built from is
+# this file, which StirrupAgent.get_version_command reads into result.json.
+COPY --from=commit /aob-commit /opt/aob/.aob-commit
+
 CMD ["bash", "-lc", "sleep infinity"]

benchmarks/harbor/run.sh

@@ -11,8 +11,7 @@
 #
 # Prerequisites: Docker running, `uv sync --extra harbor`, and the runtime image
 #
-#   docker build -t assetopsbench/runtime:dev \
-#     -f benchmarks/harbor/base-image/Dockerfile .
+#   bash benchmarks/harbor/scripts/build-runtime-image.sh
 #
 # Credentials are read from ENV_FILE (default .env) by `uv run --env-file`, into
 # the Harbor process only; StirrupAgent forwards them to the agent phase. They
@@ -86,7 +85,7 @@ dataset_dir=benchmarks/harbor/datasets/assetopsbench-suite
 
 if ! docker image inspect "$runtime_image" >/dev/null 2>&1; then
   printf 'Runtime image %s not found. Build it first:\n' "$runtime_image" >&2
-  printf '  docker build -t %s -f benchmarks/harbor/base-image/Dockerfile .\n' "$runtime_image" >&2
+  printf '  bash benchmarks/harbor/scripts/build-runtime-image.sh\n' >&2
   exit 1
 fi
 

benchmarks/harbor/scripts/build-runtime-image.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Build the runtime image from `git archive HEAD`, never from the working tree.
+#
+#   bash benchmarks/harbor/scripts/build-runtime-image.sh
+#   bash benchmarks/harbor/scripts/build-runtime-image.sh --no-cache
+#   bash benchmarks/harbor/scripts/build-runtime-image.sh -t assetopsbench/runtime:abc1234
+#
+# Arguments go to `docker buildx build` unchanged. Unless they name a tag (-t)
+# the image is tagged assetopsbench/runtime:dev, the tag the task template
+# builds FROM, and unless they name an output (--push, --output, --load) it is
+# loaded into the local image store. publish-images.sh passes --platform, its
+# own tags and --push.
+#
+# Why an archive: base-image/Dockerfile does `COPY . .`, so a build from the
+# working tree takes whatever sits there, untracked and git-ignored files
+# included. An untracked reports/results_table.csv, with a ground_truth column
+# for 56 private scenarios, once reached the image that way, readable by any
+# agent running code in `main`. The archive holds only what HEAD commits and no
+# .git, whose blobs would hold the answers .dockerignore drops. The commit
+# travels as the AOB_COMMIT build arg instead, which the Dockerfile requires.
+#
+# Uncommitted changes and new files are not built. The script warns about both;
+# commit them first to include them.
+set -euo pipefail
+
+repo_root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)"
+commit="$(git -C "$repo_root" rev-parse HEAD)"
+
+if ! docker buildx version >/dev/null 2>&1; then
+  printf 'docker buildx is required (Docker Desktop and docker-ce ship it)\n' >&2
+  exit 1
+fi
+
+if ! git -C "$repo_root" diff --quiet HEAD --; then
+  printf 'warning: tracked files differ from HEAD; building %s without those changes\n' \
+    "${commit:0:7}" >&2
+fi
+untracked="$(git -C "$repo_root" ls-files --others --exclude-standard | wc -l | tr -d ' ')"
+if (( untracked > 0 )); then
+  printf 'warning: %s untracked file(s) are not in the image; `git add` and commit any it needs\n' \
+    "$untracked" >&2
+fi
+
+context="$(mktemp -d "${TMPDIR:-/tmp}/aob-runtime-context.XXXXXX")"
+trap 'rm -rf "$context"' EXIT
+git -C "$repo_root" archive --format=tar HEAD | tar -x -C "$context"
+
+has_tag=false
+has_output=false
+for arg in "$@"; do
+  case "$arg" in
+    -t | --tag | --tag=*) has_tag=true ;;
+    --push | --load | --output | --output=* | -o) has_output=true ;;
+  esac
+done
+defaults=()
+$has_tag || defaults+=(-t assetopsbench/runtime:dev)
+$has_output || defaults+=(--load)
+
+printf 'Building the runtime image from %s (git archive)\n' "${commit:0:7}" >&2
+docker buildx build \
+  -f "$context/benchmarks/harbor/base-image/Dockerfile" \
+  --build-arg "AOB_COMMIT=$commit" \
+  ${defaults[@]+"${defaults[@]}"} "$@" \
+  "$context"

benchmarks/harbor/scripts/publish-images.sh

@@ -15,11 +15,12 @@
 #   <namespace>/code     the sandbox for the code track (numpy, pandas, scipy).
 #                        Only needed by the code-sandbox overlay.
 #
-# The runtime build context is the whole repository, so .dockerignore decides
-# what lands in the published image. It keeps .env out and keeps .git in, the
-# latter because StirrupAgent.get_version_command runs `git rev-parse` inside
-# the container to record the commit each trial ran against. Check it before
-# publishing if you have added anything sensitive to the tree.
+# The runtime image is built by build-runtime-image.sh from `git archive HEAD`,
+# so only committed files can land in the published image, and .dockerignore
+# then drops env files and scenario answers from those. The image carries no
+# .git; the commit it was built from is /opt/aob/.aob-commit, which
+# StirrupAgent.get_version_command records for each trial. Uncommitted changes
+# are not published; commit them first.
 set -euo pipefail
 
 NAMESPACE="${1:?usage: publish-images.sh <dockerhub-namespace> [tag]}"
@@ -41,17 +42,12 @@ if ! docker buildx inspect >/dev/null 2>&1; then
     exit 1
 fi
 
-if [ -f .env ]; then
-    echo "note: .env exists and is excluded by .dockerignore, so it stays out" >&2
-fi
-
 echo "==> ${NAMESPACE}/runtime:${TAG} for ${PLATFORMS}"
-docker buildx build \
+bash benchmarks/harbor/scripts/build-runtime-image.sh \
     --platform "${PLATFORMS}" \
     -t "${NAMESPACE}/runtime:${TAG}" \
     -t "${NAMESPACE}/runtime:latest" \
-    -f benchmarks/harbor/base-image/Dockerfile \
-    --push .
+    --push
 
 echo "==> ${NAMESPACE}/code:${TAG} for ${PLATFORMS}"
 docker buildx build \

src/assetops_harbor/stirrup.py

@@ -235,10 +235,12 @@ def get_version_command(self) -> str | None:
         """Record the AssetOpsBench commit as the agent version.
 
         Harbor writes this into result.json, so every trial carries the exact
-        repo state it ran against. Detection is best-effort in Harbor, so a
-        task image built without git history simply reports no version.
+        repo state it ran against. The runtime image has no .git, since its
+        blobs would hold scenario answers; build-runtime-image.sh records the
+        commit in .aob-commit instead. Detection is best-effort in Harbor, so
+        an image built without that file simply reports no version.
         """
-        return f"git -C {AOB_HOME} rev-parse --short HEAD"
+        return f"cut -c1-7 {AOB_HOME}/.aob-commit"
 
     async def run(
         self,