diff --git a/.dockerignore b/.dockerignore index f780feb78..e834e663b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,17 @@ # Docker reads THIS file, not .gitignore. Anything not listed here is pulled # into the build context by `COPY . .` in benchmarks/harbor/base-image/Dockerfile # and baked into the runtime image, including files git ignores. +# +# benchmarks/harbor/scripts/build-runtime-image.sh builds from `git archive HEAD`, +# so untracked files never reach it, and base-image/Dockerfile refuses a build +# that does not come through it. This file still applies to that context. # Credentials. AssetOpsBench's run_sdk_cli calls load_dotenv(), so a local .env # would otherwise end up inside every task image and every trial container. -.env -.env.* -!.env.public +# Unlike .gitignore, a bare name matches only at the context root, hence **/. +**/.env +**/.env.* +!**/.env.public # Host virtualenv: the image builds its own with `uv sync --frozen`. .venv/ @@ -28,6 +33,25 @@ benchmarks/harbor/datasets/ traces/ .DS_Store -# NOTE: .git is deliberately NOT ignored. StirrupAgent.get_version_command runs -# `git rev-parse --short HEAD` inside the container so every trial records the -# AssetOpsBench commit it ran against in result.json. +# Answers. `main` is the agent's container, and the verifier reads the copies +# Harbor uploads to /tests after the agent phase, never these. Every other file +# of a scenario folder stays: manifest.json is what init_data.py loads. +src/couchdb/scenarios_data/**/groundtruth* +src/couchdb/scenarios_data/**/rubric.json +src/couchdb/scenarios_data/**/reference_answer.json +src/couchdb/scenarios_data/**/scenario_meta.json + +# Local work that is not part of the benchmark. reports/ once carried a results +# table with a ground_truth column for private scenarios into the image. +reports/ +logs/ +src/tmp/ +.claude/ +notebook/kdd_tutorial/ +artifacts/kdd_tutorial/ + +# Git data. Even a one-commit .git holds every committed blob, answers included +# (`git show HEAD:src/couchdb/scenarios_data/scenario_1/groundtruth.txt`), and +# its reflog names whoever built the image. The commit reaches the image as +# /opt/aob/.aob-commit instead, from the AOB_COMMIT build arg. +.git diff --git a/benchmarks/harbor/CODE-SANDBOX.md b/benchmarks/harbor/CODE-SANDBOX.md index 8aa3021d1..a48c31bea 100644 --- a/benchmarks/harbor/CODE-SANDBOX.md +++ b/benchmarks/harbor/CODE-SANDBOX.md @@ -57,7 +57,7 @@ Treat anything other than local Docker as untested for this overlay. The runtime image: ```bash -docker build -t assetopsbench/runtime:dev -f benchmarks/harbor/base-image/Dockerfile . +bash benchmarks/harbor/scripts/build-runtime-image.sh ``` A `.env` with your model credentials. `harbor run` is invoked through diff --git a/benchmarks/harbor/QUICKSTART.md b/benchmarks/harbor/QUICKSTART.md index 3a73e5ec8..f6780d5ca 100644 --- a/benchmarks/harbor/QUICKSTART.md +++ b/benchmarks/harbor/QUICKSTART.md @@ -33,10 +33,13 @@ docker tag assetopsbench/runtime:latest assetopsbench/runtime:dev Or build it yourself, which takes a few minutes and needs no registry: ```bash -docker build -t assetopsbench/runtime:dev \ - -f benchmarks/harbor/base-image/Dockerfile . +bash benchmarks/harbor/scripts/build-runtime-image.sh ``` +The script builds `assetopsbench/runtime:dev` from `git archive HEAD`, not +from your working tree, so untracked files such as local results never reach +the container the agent runs in. Commit a change first to include it. + Either way the local tag `assetopsbench/runtime:dev` is what the task Dockerfiles reference, through the `AOB_RUNTIME_IMAGE` build arg in `benchmarks/harbor/template/environment/Dockerfile`. Change that default if you @@ -170,7 +173,8 @@ and check `docker images assetopsbench/runtime`. **`No module named 'google.protobuf'` during a run** — the image was built without the `otel` dependency group, which the file trace exporter needs. -Rebuild from `benchmarks/harbor/base-image/Dockerfile`, which passes it. +Rebuild with `benchmarks/harbor/scripts/build-runtime-image.sh`, whose +Dockerfile passes it. **The verifier scores 0 but the agent clearly answered** — check `verifier/test-stderr.txt`. The evaluator joins records to scenarios on diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index ceffdef3d..b49cc7672 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -39,11 +39,11 @@ Prerequisites: Docker running, and `uv sync --dev --extra harbor`. Run every command from the repo root. ```bash -# 1. Build the runtime base (once per AssetOpsBench commit). The tag is local -# and is the default AOB_RUNTIME_IMAGE in template/environment/Dockerfile; -# nothing is pulled or pushed. -docker build -t assetopsbench/runtime:dev \ - -f benchmarks/harbor/base-image/Dockerfile . +# 1. Build the runtime base (once per AssetOpsBench commit). The script builds +# from `git archive HEAD`, so untracked files and uncommitted changes stay +# out of the image. The tag is local and is the default AOB_RUNTIME_IMAGE in +# template/environment/Dockerfile; nothing is pulled or pushed. +bash benchmarks/harbor/scripts/build-runtime-image.sh # 2. Generate one task per scenario in the open profile. The defaults point at # src/couchdb/scenarios_data, benchmarks/scenario_suite/open.yaml and diff --git a/benchmarks/harbor/base-image/Dockerfile b/benchmarks/harbor/base-image/Dockerfile index ad9f16bf1..25b450f25 100644 --- a/benchmarks/harbor/base-image/Dockerfile +++ b/benchmarks/harbor/base-image/Dockerfile @@ -6,8 +6,25 @@ # the whole environment/ directory, so a fat per-task context defeats the cache # and rebuilds the world for every scenario. # -# docker build -t assetopsbench/runtime:$(git rev-parse --short HEAD) \ -# -f base-image/Dockerfile +# bash benchmarks/harbor/scripts/build-runtime-image.sh \ +# -t assetopsbench/runtime:$(git rev-parse --short HEAD) +# +# The script builds from `git archive HEAD`, so `COPY . .` below sees committed +# files only, and passes that commit as AOB_COMMIT. A plain `docker build` of the +# working tree would also copy untracked files, which .dockerignore alone cannot +# anticipate, so without AOB_COMMIT the `commit` stage fails the build. + +# Its own stage so the check fails fast without the ARG reaching the stage +# below: every RUN after an ARG misses the cache when its value changes, and +# AOB_COMMIT changes with every commit. +FROM python:3.12-slim AS commit +ARG AOB_COMMIT +RUN test -n "$AOB_COMMIT" || { \ + echo "build with benchmarks/harbor/scripts/build-runtime-image.sh," \ + "not docker build: it builds HEAD, never the working tree" >&2; \ + exit 1; } \ + && echo "$AOB_COMMIT" > /aob-commit + FROM python:3.12-slim # HF_HOME fixes the Hugging Face cache at one path. The default follows $HOME, @@ -52,4 +69,8 @@ RUN uv sync --frozen --group otel --extra tsfm RUN uv run python benchmarks/harbor/scripts/preload_models.py \ --from-list benchmarks/harbor/base-image/models.txt --download +# The image has no .git (see .dockerignore): the commit it was built from is +# this file, which StirrupAgent.get_version_command reads into result.json. +COPY --from=commit /aob-commit /opt/aob/.aob-commit + CMD ["bash", "-lc", "sleep infinity"] diff --git a/benchmarks/harbor/run.sh b/benchmarks/harbor/run.sh index 118643fa8..a51ed6aed 100755 --- a/benchmarks/harbor/run.sh +++ b/benchmarks/harbor/run.sh @@ -11,8 +11,7 @@ # # Prerequisites: Docker running, `uv sync --extra harbor`, and the runtime image # -# docker build -t assetopsbench/runtime:dev \ -# -f benchmarks/harbor/base-image/Dockerfile . +# bash benchmarks/harbor/scripts/build-runtime-image.sh # # Credentials are read from ENV_FILE (default .env) by `uv run --env-file`, into # the Harbor process only; StirrupAgent forwards them to the agent phase. They @@ -86,7 +85,7 @@ dataset_dir=benchmarks/harbor/datasets/assetopsbench-suite if ! docker image inspect "$runtime_image" >/dev/null 2>&1; then printf 'Runtime image %s not found. Build it first:\n' "$runtime_image" >&2 - printf ' docker build -t %s -f benchmarks/harbor/base-image/Dockerfile .\n' "$runtime_image" >&2 + printf ' bash benchmarks/harbor/scripts/build-runtime-image.sh\n' >&2 exit 1 fi diff --git a/benchmarks/harbor/scripts/build-runtime-image.sh b/benchmarks/harbor/scripts/build-runtime-image.sh new file mode 100755 index 000000000..88d7b6576 --- /dev/null +++ b/benchmarks/harbor/scripts/build-runtime-image.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Build the runtime image from `git archive HEAD`, never from the working tree. +# +# bash benchmarks/harbor/scripts/build-runtime-image.sh +# bash benchmarks/harbor/scripts/build-runtime-image.sh --no-cache +# bash benchmarks/harbor/scripts/build-runtime-image.sh -t assetopsbench/runtime:abc1234 +# +# Arguments go to `docker buildx build` unchanged. Unless they name a tag (-t) +# the image is tagged assetopsbench/runtime:dev, the tag the task template +# builds FROM, and unless they name an output (--push, --output, --load) it is +# loaded into the local image store. publish-images.sh passes --platform, its +# own tags and --push. +# +# Why an archive: base-image/Dockerfile does `COPY . .`, so a build from the +# working tree takes whatever sits there, untracked and git-ignored files +# included. An untracked reports/results_table.csv, with a ground_truth column +# for 56 private scenarios, once reached the image that way, readable by any +# agent running code in `main`. The archive holds only what HEAD commits and no +# .git, whose blobs would hold the answers .dockerignore drops. The commit +# travels as the AOB_COMMIT build arg instead, which the Dockerfile requires. +# +# Uncommitted changes and new files are not built. The script warns about both; +# commit them first to include them. +set -euo pipefail + +repo_root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +commit="$(git -C "$repo_root" rev-parse HEAD)" + +if ! docker buildx version >/dev/null 2>&1; then + printf 'docker buildx is required (Docker Desktop and docker-ce ship it)\n' >&2 + exit 1 +fi + +if ! git -C "$repo_root" diff --quiet HEAD --; then + printf 'warning: tracked files differ from HEAD; building %s without those changes\n' \ + "${commit:0:7}" >&2 +fi +untracked="$(git -C "$repo_root" ls-files --others --exclude-standard | wc -l | tr -d ' ')" +if (( untracked > 0 )); then + printf 'warning: %s untracked file(s) are not in the image; `git add` and commit any it needs\n' \ + "$untracked" >&2 +fi + +context="$(mktemp -d "${TMPDIR:-/tmp}/aob-runtime-context.XXXXXX")" +trap 'rm -rf "$context"' EXIT +git -C "$repo_root" archive --format=tar HEAD | tar -x -C "$context" + +has_tag=false +has_output=false +for arg in "$@"; do + case "$arg" in + -t | --tag | --tag=*) has_tag=true ;; + --push | --load | --output | --output=* | -o) has_output=true ;; + esac +done +defaults=() +$has_tag || defaults+=(-t assetopsbench/runtime:dev) +$has_output || defaults+=(--load) + +printf 'Building the runtime image from %s (git archive)\n' "${commit:0:7}" >&2 +docker buildx build \ + -f "$context/benchmarks/harbor/base-image/Dockerfile" \ + --build-arg "AOB_COMMIT=$commit" \ + ${defaults[@]+"${defaults[@]}"} "$@" \ + "$context" diff --git a/benchmarks/harbor/scripts/publish-images.sh b/benchmarks/harbor/scripts/publish-images.sh index 6f93730a1..25a793189 100755 --- a/benchmarks/harbor/scripts/publish-images.sh +++ b/benchmarks/harbor/scripts/publish-images.sh @@ -15,11 +15,12 @@ # /code the sandbox for the code track (numpy, pandas, scipy). # Only needed by the code-sandbox overlay. # -# The runtime build context is the whole repository, so .dockerignore decides -# what lands in the published image. It keeps .env out and keeps .git in, the -# latter because StirrupAgent.get_version_command runs `git rev-parse` inside -# the container to record the commit each trial ran against. Check it before -# publishing if you have added anything sensitive to the tree. +# The runtime image is built by build-runtime-image.sh from `git archive HEAD`, +# so only committed files can land in the published image, and .dockerignore +# then drops env files and scenario answers from those. The image carries no +# .git; the commit it was built from is /opt/aob/.aob-commit, which +# StirrupAgent.get_version_command records for each trial. Uncommitted changes +# are not published; commit them first. set -euo pipefail NAMESPACE="${1:?usage: publish-images.sh [tag]}" @@ -41,17 +42,12 @@ if ! docker buildx inspect >/dev/null 2>&1; then exit 1 fi -if [ -f .env ]; then - echo "note: .env exists and is excluded by .dockerignore, so it stays out" >&2 -fi - echo "==> ${NAMESPACE}/runtime:${TAG} for ${PLATFORMS}" -docker buildx build \ +bash benchmarks/harbor/scripts/build-runtime-image.sh \ --platform "${PLATFORMS}" \ -t "${NAMESPACE}/runtime:${TAG}" \ -t "${NAMESPACE}/runtime:latest" \ - -f benchmarks/harbor/base-image/Dockerfile \ - --push . + --push echo "==> ${NAMESPACE}/code:${TAG} for ${PLATFORMS}" docker buildx build \ diff --git a/src/assetops_harbor/stirrup.py b/src/assetops_harbor/stirrup.py index 8a73f5218..f5bb3f114 100644 --- a/src/assetops_harbor/stirrup.py +++ b/src/assetops_harbor/stirrup.py @@ -235,10 +235,12 @@ def get_version_command(self) -> str | None: """Record the AssetOpsBench commit as the agent version. Harbor writes this into result.json, so every trial carries the exact - repo state it ran against. Detection is best-effort in Harbor, so a - task image built without git history simply reports no version. + repo state it ran against. The runtime image has no .git, since its + blobs would hold scenario answers; build-runtime-image.sh records the + commit in .aob-commit instead. Detection is best-effort in Harbor, so + an image built without that file simply reports no version. """ - return f"git -C {AOB_HOME} rev-parse --short HEAD" + return f"cut -c1-7 {AOB_HOME}/.aob-commit" async def run( self,