Keep the job name a config sets, and add a job-name input

.github/workflows/test.yml

@@ -191,3 +191,44 @@ jobs:
           N_TRIALS: ${{ steps.hbench.outputs.n-trials }}
         run: |
           [[ "$MEAN_REWARD" == 1.0 && "$N_TRIALS" == 1 ]]
+
+  run-config:
+    name: Keep the job name a config sets, unless job-name replaces it
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Run a config that sets job_name
+        id: named
+        uses: ./
+        with:
+          config: tests/configs/named-job.json
+          min-mean-reward: '1'
+          upload-artifact: 'false'
+      - name: Run it again under the name job-name sets
+        id: renamed
+        uses: ./
+        with:
+          config: tests/configs/named-job.json
+          job-name: renamed-job
+          min-mean-reward: '1'
+          upload-artifact: 'false'
+      - name: Run a config that sets no job_name
+        id: unnamed
+        uses: ./
+        with:
+          config: tests/configs/unnamed-job.yaml
+          min-mean-reward: '1'
+          upload-artifact: 'false'
+      - name: Check the job directories and the names hbench recorded
+        env:
+          NAMED: ${{ steps.named.outputs.job-dir }}
+          RENAMED: ${{ steps.renamed.outputs.job-dir }}
+          UNNAMED: ${{ steps.unnamed.outputs.job-dir }}
+        run: |
+          [[ "$(basename "$NAMED")" == named-job && "$(jq -r .job_name "$NAMED/config.json")" == named-job ]]
+          [[ "$(basename "$RENAMED")" == renamed-job && "$(jq -r .job_name "$RENAMED/config.json")" == renamed-job ]]
+          [[ "$(basename "$UNNAMED")" == "$GITHUB_JOB"-* && -f "$UNNAMED/result.json" ]]

README.md

@@ -73,7 +73,11 @@ To run Claude Code instead, add your Anthropic key as a repository secret and pa
           n-tasks: 5
 ```
 
-The scores go to the job summary, and the job's files are uploaded as an artifact.
+The scores go to the job summary, and the job's files are uploaded as an artifact. `n-tasks: 5` runs 5 of the
+dataset's 89 tasks; leave it out to run them all. A score compares only with runs of the same tasks: the same
+dataset and version from the same registry, with the same `n-tasks`, `include` and `exclude`. To run a job
+spec you were given, under the job name it sets, pass it as `config` alone (see
+[Run a job spec](docs/REFERENCE.md#more-examples)).
 
 For a whole repository to fork, with three tasks, the workflow that checks them on every pull request and
 workflows that run agents on them, see
@@ -100,7 +104,7 @@ For hbench itself, the docs are at [nodeq.tech/docs](https://nodeq.tech/docs):
     # registry: oci://ghcr.io/node-q/hbench-datasets
     # dataset: my-tasks@1.0                       # or a dataset listed in a registry.json
     # registry: registry.json                     #   a path, or an https:// URL
-    # config: job.yaml                            # or a job config (YAML/JSON)
+    # config: job.yaml                            # or a job config (YAML/JSON), run under its job_name
 
     # Who runs it
     agent: claude-code                            # default: oracle

action.yml

@@ -8,7 +8,7 @@ branding:
 
 inputs:
   version:
-    description: hbench release to install, such as v0.3.0, or latest.
+    description: hbench release to install, such as v0.3.5, or latest.
     default: latest
   verify-provenance:
     description: Check the release archive's build provenance with gh attestation verify. true, false, or auto (when the GitHub CLI is installed).
@@ -20,13 +20,13 @@ inputs:
     description: A task directory, or a directory of task directories (hbench run --path). Set path, dataset or config to run; with none of them the action only installs hbench.
     default: ''
   dataset:
-    description: Dataset to run, name@version (hbench run --dataset). Needs registry.
+    description: Dataset to run, name@version (hbench run --dataset), from registry. With no registry, hbench 0.3.3 and later read it from $HBENCH_REGISTRY, else oci://ghcr.io/node-q/hbench-datasets.
     default: ''
   registry:
-    description: Where the dataset is listed. An OCI registry, oci://<registry>/<namespace>, or OCI image layouts on disk, oci-layout:<dir> (hbench run --registry); or a registry.json, by https:// URL (--registry-url) or path (--registry-path).
+    description: Where the dataset is listed. An OCI registry, oci://<registry>/<namespace>, or OCI image layouts on disk, oci-layout:<dir> (hbench run --registry); or a registry.json, by https:// URL (--registry-url) or path (--registry-path). Empty uses hbench's default registry.
     default: ''
   config:
-    description: Job config, YAML or JSON (hbench run --config). The other inputs override it.
+    description: Job config, YAML or JSON (hbench run --config). The other inputs override it. Unless job-name is set, the job keeps the config's job_name.
     default: ''
   agent:
     description: Agent to run (hbench run --agent); hbench agents list shows them. Empty runs hbench's default, the oracle.
@@ -61,6 +61,9 @@ inputs:
   jobs-dir:
     description: Where hbench writes the job directory. Empty uses hbench-jobs under the runner's temporary directory.
     default: ''
+  job-name:
+    description: Name of the job directory (hbench run --job-name). Empty keeps the job_name the config sets, and otherwise uses <job id>-<time>-<random>.
+    default: ''
   min-mean-reward:
     description: Fail the step when the mean reward over every trial is below this number, from 0 to 1. Empty never fails on the score.
     default: ''
@@ -76,7 +79,7 @@ inputs:
 
 outputs:
   version:
-    description: The hbench release installed, such as v0.3.0.
+    description: The hbench release installed, such as v0.3.5.
     value: ${{ steps.install.outputs.version }}
   hbench-path:
     description: Path of the installed hbench binary.
@@ -141,6 +144,7 @@ runs:
         INPUT_LOCKED: ${{ inputs.locked }}
         INPUT_ARGS: ${{ inputs.args }}
         INPUT_JOBS_DIR: ${{ inputs.jobs-dir }}
+        INPUT_JOB_NAME: ${{ inputs.job-name }}
         INPUT_MIN_MEAN_REWARD: ${{ inputs.min-mean-reward }}
         INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }}
         INPUT_UPLOAD_ARTIFACT: ${{ inputs.upload-artifact }}

docs/REFERENCE.md

@@ -13,13 +13,13 @@ The [README](../README.md) has the short version.
 
 | Input | Default | Description |
 |---|---|---|
-| `version` | `latest` | hbench release to install, such as v0.3.0, or latest. |
+| `version` | `latest` | hbench release to install, such as v0.3.5, or latest. |
 | `verify-provenance` | `auto` | Check the release archive's build provenance with gh attestation verify. true, false, or auto (when the GitHub CLI is installed). |
 | `token` | `${{ github.token }}` | GitHub token for gh attestation verify. |
 | `path` |  | A task directory, or a directory of task directories (hbench run --path). Set path, dataset or config to run; with none of them the action only installs hbench. |
-| `dataset` |  | Dataset to run, name@version (hbench run --dataset). Needs registry. |
-| `registry` |  | Where the dataset is listed. An OCI registry, oci://<registry>/<namespace>, or OCI image layouts on disk, oci-layout:<dir> (hbench run --registry); or a registry.json, by https:// URL (--registry-url) or path (--registry-path). |
-| `config` |  | Job config, YAML or JSON (hbench run --config). The other inputs override it. |
+| `dataset` |  | Dataset to run, name@version (hbench run --dataset), from registry. With no registry, hbench 0.3.3 and later read it from $HBENCH_REGISTRY, else oci://ghcr.io/node-q/hbench-datasets. |
+| `registry` |  | Where the dataset is listed. An OCI registry, oci://<registry>/<namespace>, or OCI image layouts on disk, oci-layout:<dir> (hbench run --registry); or a registry.json, by https:// URL (--registry-url) or path (--registry-path). Empty uses hbench's default registry. |
+| `config` |  | Job config, YAML or JSON (hbench run --config). The other inputs override it. Unless job-name is set, the job keeps the config's job_name. |
 | `agent` |  | Agent to run (hbench run --agent); hbench agents list shows them. Empty runs hbench's default, the oracle. |
 | `model` |  | Model, as provider/name (hbench run --model). |
 | `n-tasks` |  | Run at most this many tasks (hbench run --n-tasks). |
@@ -31,6 +31,7 @@ The [README](../README.md) has the short version.
 | `args` |  | More hbench run arguments, one per line, each line a whole argument. |
 | `working-directory` | `.` | Directory to run hbench in. Relative paths and hbench.lock are read from it. |
 | `jobs-dir` |  | Where hbench writes the job directory. Empty uses hbench-jobs under the runner's temporary directory. |
+| `job-name` |  | Name of the job directory (hbench run --job-name). Empty keeps the job_name the config sets, and otherwise uses <job id>-<time>-<random>. |
 | `min-mean-reward` |  | Fail the step when the mean reward over every trial is below this number, from 0 to 1. Empty never fails on the score. |
 | `upload-artifact` | `true` | Upload the job directory as a workflow artifact. |
 | `artifact-name` |  | Name of the artifact. Empty uses hbench-<job id>-<time>-<random>; set it to something stable in a matrix. |
@@ -40,7 +41,7 @@ The [README](../README.md) has the short version.
 
 | Output | Description |
 |---|---|
-| `version` | The hbench release installed, such as v0.3.0. |
+| `version` | The hbench release installed, such as v0.3.5. |
 | `hbench-path` | Path of the installed hbench binary. |
 | `job-dir` | The job directory hbench wrote. |
 | `result-path` | The job's result.json. |
@@ -137,16 +138,40 @@ Run a dataset listed in a `registry.json`, by path or by https:// URL. A task wi
     registry: registry.json   # or https://raw.githubusercontent.com/you/tasks/<commit>/registry.json
 ```
 
+Run a job spec as it was given to you, such as one a leaderboard issues for a run, under the job name it
+sets:
+
+```yaml
+- uses: actions/checkout@v7
+- uses: node-q/hbench-action@v1
+  env:
+    DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+  with:
+    config: spec.json
+```
+
+A config that sets `job_name` keeps it: the action passes no `--job-name`, and `job-dir` is
+`<jobs-dir>/<job_name>`. A config that sets none runs as `<job id>-<time>-<random>`, as `path` and `dataset`
+do, and `job-name` sets a name of your own. hbench-action v1.1.0 and earlier passed their own `--job-name`
+every time. hbench starts a job name only once in a jobs directory, so running the same spec twice in one job
+needs another `jobs-dir`. With a spec, set only `config` and inputs that leave the job as it is, such as
+`version`, `jobs-dir`, `min-mean-reward` and the artifact inputs: a server that checks the job against its
+spec refuses one that differs. `--job-name` in `args` is refused; set `job-name` instead.
+
 Install only, then call hbench yourself:
 
 ```yaml
 - uses: node-q/hbench-action@v1
   id: hbench
   with:
-    version: v0.3.0
+    version: v0.3.5
 - run: hbench tasks check --path tasks
 ```
 
+`hbench tasks check` validates each task's files and test scripts and needs no key. `hbench check` is another
+command: it reviews tasks against a rubric with an agent, `claude-code` by default, so it needs that agent's
+key.
+
 Pass any other `hbench run` flag through `args`, one argument per line:
 
 ```yaml
@@ -182,16 +207,19 @@ Pass any other `hbench run` flag through `args`, one argument per line:
 action.yml          the inputs, outputs and three steps: install, run, upload
 scripts/install.sh  download, checksum, provenance, tool cache
 scripts/run.sh      build the hbench run command line, run it, report
+scripts/config_job_name.py  the job_name a config sets, which run.sh keeps
 scripts/report.py   the job's results to outputs, job summary and the threshold
 tests/test_report.py  the report, on the two job directories in tests/report/
+tests/test_config_job_name.py  reading job_name from JSON and YAML configs
 tests/tasks/        a one-task folder the tests run
+tests/configs/      job configs with and without job_name
 ```
 
-`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint, shellcheck and the report's unit
+`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint, shellcheck and the scripts' unit
 tests; installing `latest` and a pinned release on Linux x86_64 and arm64 and macOS arm64 and x86_64, with
 provenance checked; the oracle on the local task with `min-mean-reward: '1'`; the `nop` agent, which must fail
-the step on its score; two tasks of an OCI dataset with `--lock-write`; and a dataset listed in a
-`registry.json`, by path and by URL.
+the step on its score; two tasks of an OCI dataset with `--lock-write`; a dataset listed in a
+`registry.json`, by path and by URL; and job configs with and without `job_name`, with and without `job-name`.
 
 To try the scripts outside GitHub, set the variables the runner sets (`RUNNER_TEMP`, `GITHUB_ACTION_PATH`,
 `GITHUB_OUTPUT`, `GITHUB_PATH`, `GITHUB_STEP_SUMMARY`) and the `INPUT_*` ones `action.yml` passes, then run

scripts/config_job_name.py

@@ -0,0 +1,71 @@
+"""Print the job_name a job config sets, or an empty line when it sets none.
+
+Usage: config_job_name.py CONFIG
+
+Reads the config as hbench run --config does: a .json file as JSON, any other as YAML. Without PyYAML, a YAML
+config's job_name is read from its top-level job_name line.
+"""
+
+import json
+import pathlib
+import re
+import sys
+
+TOP_LEVEL_JOB_NAME = re.compile(r"""^(?:job_name|"job_name"|'job_name')[ \t]*:(?:[ \t]+(.*))?$""", re.MULTILINE)
+DOUBLE_QUOTED = re.compile(r'"(?:[^"\\]|\\.)*"')
+SINGLE_QUOTED = re.compile(r"'(?:[^']|'')*'")
+
+
+def yaml_scalar(text):
+    text = text.strip()
+    if match := DOUBLE_QUOTED.match(text):
+        return json.loads(match.group())
+    if match := SINGLE_QUOTED.match(text):
+        return match.group()[1:-1].replace("''", "'")
+    plain = re.split(r"[ \t]#", text, maxsplit=1)[0].strip()
+    return None if plain in ("", "~", "null", "Null", "NULL") else plain
+
+
+def top_level_job_name(text):
+    try:
+        return json.loads(text)
+    except ValueError:
+        pass
+    match = TOP_LEVEL_JOB_NAME.search(text)
+    return {"job_name": yaml_scalar(match.group(1) or "")} if match else {}
+
+
+def read_config(path):
+    text = path.read_text(encoding="utf-8")
+    if path.suffix == ".json":
+        return json.loads(text)
+    try:
+        import yaml
+    except ImportError:
+        return top_level_job_name(text)
+    return yaml.safe_load(text)
+
+
+def job_name(config):
+    name = config.get("job_name") if isinstance(config, dict) else None
+    return name if isinstance(name, str) and name.strip() else ""
+
+
+def main(arguments):
+    path = pathlib.Path(arguments[0])
+    try:
+        config = read_config(path)
+    except OSError as error:
+        print(f"could not read {path}: {error.strerror}", file=sys.stderr)
+        return 1
+    except Exception as error:
+        kind = "JSON" if path.suffix == ".json" else "YAML"
+        detail = str(error).strip().splitlines()[0] if str(error).strip() else type(error).__name__
+        print(f"{path} is not valid {kind}: {detail}", file=sys.stderr)
+        return 1
+    print(job_name(config))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/run.sh

@@ -33,11 +33,22 @@ esac
 if [[ -n "${INPUT_MIN_MEAN_REWARD}" ]]; then
   [[ "$INPUT_MIN_MEAN_REWARD" =~ ^(0(\.[0-9]+)?|1(\.0+)?)$ ]] || fail "min-mean-reward is '$INPUT_MIN_MEAN_REWARD'; use a number from 0 to 1"
 fi
+while IFS= read -r line; do
+  case "${line#"${line%%[![:space:]]*}"}" in
+    --job-name | --job-name=*) fail "args sets --job-name; set the job-name input instead" ;;
+  esac
+done <<<"$INPUT_ARGS"
 
 jobs_directory="${INPUT_JOBS_DIR:-$RUNNER_TEMP/hbench-jobs}"
 mkdir -p "$jobs_directory"
 jobs_directory="$(cd "$jobs_directory" && pwd)"
-job_name="${GITHUB_JOB:-job}-$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM"
+run_id="${GITHUB_JOB:-job}-$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM"
+config_job_name=""
+if [[ -n "$INPUT_CONFIG" && -z "$INPUT_JOB_NAME" ]]; then
+  config_job_name="$(python3 "$GITHUB_ACTION_PATH/scripts/config_job_name.py" "$INPUT_CONFIG")" ||
+    fail "could not read config '$INPUT_CONFIG'"
+fi
+job_name="${INPUT_JOB_NAME:-${config_job_name:-$run_id}}"
 job_directory="$jobs_directory/$job_name"
 
 arguments=(run)
@@ -61,11 +72,12 @@ add_option --n-concurrent "$INPUT_N_CONCURRENT"
 add_lines --include-task-name "$INPUT_INCLUDE"
 add_lines --exclude-task-name "$INPUT_EXCLUDE"
 [[ "${INPUT_LOCKED:-false}" == true ]] && arguments+=(--locked)
-arguments+=(--jobs-dir "$jobs_directory" --job-name "$job_name")
+arguments+=(--jobs-dir "$jobs_directory")
+[[ -z "$config_job_name" ]] && arguments+=(--job-name "$job_name")
 add_lines "" "$INPUT_ARGS"
 
 echo "job-dir=$job_directory" >> "$GITHUB_OUTPUT"
-artifact_name="${INPUT_ARTIFACT_NAME:-hbench-$job_name}"
+artifact_name="${INPUT_ARTIFACT_NAME:-hbench-$run_id}"
 echo "artifact-name=$artifact_name" >> "$GITHUB_OUTPUT"
 
 printf 'hbench'

tests/configs/named-job.json

@@ -0,0 +1,5 @@
+{
+  "job_name": "named-job",
+  "agents": [{"name": "oracle"}],
+  "tasks": [{"path": "tests/tasks/hello-world"}]
+}

tests/configs/unnamed-job.yaml

@@ -0,0 +1,4 @@
+agents:
+  - name: oracle
+tasks:
+  - path: tests/tasks/hello-world