From ad098b27902546dd28a1fc6227f5fec94581be4a Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 14 Aug 2026 10:28:25 -0400 Subject: [PATCH] fix(harbor): default to shared Harbor project Use a stable Harbor project when no Braintrust project name or ID is configured, avoiding one project per timestamp-named job. Keep explicit project configuration authoritative and align run and backfill documentation with the shared fallback. --- examples/harbor/README.md | 10 +++------ examples/harbor/backfill.py | 5 ++++- .../braintrust/integrations/harbor/plugin.py | 21 +++++++++++++------ .../integrations/harbor/test_harbor.py | 10 +++++++++ 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/examples/harbor/README.md b/examples/harbor/README.md index a7813a98..834af879 100644 --- a/examples/harbor/README.md +++ b/examples/harbor/README.md @@ -34,27 +34,23 @@ uv run harbor run \ --jobs-dir jobs \ --env-file ../../.env \ --plugin braintrust \ - --plugin-kwarg project_name=example-harbor \ --yes ``` The agent solves the task in `task/`, and Harbor's verifier emits a normalized `reward` plus an `answer_length` metric. The plugin creates `jobs/braintrust-harbor-example/braintrust-sync.json` after synchronization. -Harbor also accepts plugin options through `HARBOR_BRAINTRUST_*` variables. For example, setting this in `.env` removes the need for the `project_name` plugin argument: +By default, the plugin uses `Harbor` as the Braintrust project name. Override it with `--plugin-kwarg project_name=example-harbor` or through `.env`: ```dotenv HARBOR_BRAINTRUST_PROJECT=example-harbor ``` -Then omit `--plugin-kwarg project_name=example-harbor` from the command. - ## Backfill an existing job To synchronize the persisted job again without rerunning the agent or verifier: ```bash -uv run --env-file ../../.env python backfill.py jobs/braintrust-harbor-example \ - --project example-harbor +uv run --env-file ../../.env python backfill.py jobs/braintrust-harbor-example ``` -Backfill uses the same deterministic dataset, experiment, and span identities, so it reconciles the existing Braintrust data instead of creating duplicate rows. +Backfill uses the same `Harbor` project default as the initial sync. It also uses the same deterministic dataset, experiment, and span identities, so it reconciles the existing Braintrust data instead of creating duplicate rows. diff --git a/examples/harbor/backfill.py b/examples/harbor/backfill.py index 69baa0cc..dc23c8f9 100755 --- a/examples/harbor/backfill.py +++ b/examples/harbor/backfill.py @@ -11,7 +11,10 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("job_dir", type=Path, help="Persisted Harbor job directory") - parser.add_argument("--project", help="Braintrust project name (otherwise read from the environment)") + parser.add_argument( + "--project", + help="Braintrust project override (otherwise use the environment or the Harbor project)", + ) args = parser.parse_args() options = {"project_name": args.project} if args.project else {} diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py index 83e794af..29758db8 100644 --- a/py/src/braintrust/integrations/harbor/plugin.py +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -56,6 +56,7 @@ logger = logging.getLogger(__name__) _PLUGIN_VERSION = "1" _MANIFEST_VERSION = 1 +_DEFAULT_PROJECT = "Harbor" @dataclass @@ -84,6 +85,11 @@ class RuntimeState: partitions: dict[str, Partition] +def _resolve_project(config: PluginConfig) -> tuple[str | None, str | None]: + project_name = config.project_name or (_DEFAULT_PROJECT if config.project_id is None else None) + return project_name, config.project_id + + def _seconds(value: Any, fallback: float) -> float: if isinstance(value, datetime): # Harbor trial timestamps are timezone-aware, but job timestamps are @@ -479,6 +485,7 @@ async def _dispatch(self, identity: str, event: TrialEvent) -> None: def _initialize(self, snapshot: JobSnapshot) -> RuntimeState: previous = self._load_manifest(snapshot.job_dir) self._manifest = previous + project_name, project_id = _resolve_project(self.config) plan_by_trial = {plan.trial_name: plan for plan in snapshot.plans} datasets: dict[str, DatasetBinding] = {} source_tasks: dict[str, dict[str, Any]] = {} @@ -497,8 +504,8 @@ def _initialize(self, snapshot: JobSnapshot) -> RuntimeState: else: name = dataset_display_name(source, prefix=self.config.dataset_name or "harbor") dataset = init_dataset( - project=self.config.project_name, - project_id=self.config.project_id, + project=project_name, + project_id=project_id, name=name, use_output=False, metadata={"harbor": {"source": source, "scope": scope, "schema_version": _PLUGIN_VERSION}}, @@ -560,8 +567,8 @@ def _initialize(self, snapshot: JobSnapshot) -> RuntimeState: } dataset = datasets[scope].dataset experiment = init( - project=self.config.project_name, - project_id=self.config.project_id, + project=project_name, + project_id=project_id, experiment=name, update=True, dataset=dataset, @@ -875,11 +882,12 @@ def _sync_final_result(self, result: Any) -> None: def _persist_disabled_manifest(self) -> None: if self._snapshot is None: return + project_name, project_id = _resolve_project(self.config) manifest = { "manifest_version": _MANIFEST_VERSION, "plugin_version": _PLUGIN_VERSION, "job_id": self._snapshot.job_id, - "project": {"name": self.config.project_name, "id": self.config.project_id}, + "project": {"name": project_name, "id": project_id}, "datasets": {}, "experiments": {}, "trials": {}, @@ -908,11 +916,12 @@ def _persist_manifest(self, completed: bool) -> None: if self._runtime is None: return snapshot = self._runtime.snapshot + project_name, project_id = _resolve_project(self.config) manifest = { "manifest_version": _MANIFEST_VERSION, "plugin_version": _PLUGIN_VERSION, "job_id": snapshot.job_id, - "project": {"name": self.config.project_name, "id": self.config.project_id}, + "project": {"name": project_name, "id": project_id}, "datasets": { scope: { "id": binding.dataset.id if binding.dataset is not None else None, diff --git a/py/src/braintrust/integrations/harbor/test_harbor.py b/py/src/braintrust/integrations/harbor/test_harbor.py index cc22d035..c95cad8b 100644 --- a/py/src/braintrust/integrations/harbor/test_harbor.py +++ b/py/src/braintrust/integrations/harbor/test_harbor.py @@ -31,6 +31,7 @@ RuntimeState, _artifact_attachments, _attachment, + _resolve_project, _seconds, _timing, ) @@ -115,6 +116,15 @@ def test_config_environment_fallback_and_explicit_precedence(): os.environ[name] = value +def test_plugin_defaults_project_to_harbor(): + assert _resolve_project(PluginConfig.from_options()) == ("Harbor", None) + assert _resolve_project(PluginConfig.from_options(project_name="explicit-project")) == ( + "explicit-project", + None, + ) + assert _resolve_project(PluginConfig.from_options(project_id="project-id")) == (None, "project-id") + + def test_harbor_resolves_the_plugin_through_its_entry_point(): # Users select this plugin with `--plugin braintrust`, which Harbor resolves # through the harbor.plugins entry-point group. Nothing else in the test suite