diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd6fc47..048ddaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,33 @@ jobs: - name: Tests run: uv run pytest + # uv.lock pins the newest releases, so the matrix never sees the oldest ones + # pyproject still admits. The floors were once set by hand and were wrong for + # three packages at once. This job installs the oldest direct dependency each + # floor allows and runs the suite on it, so a floor that no longer holds fails + # here, not on a user's machine. + lowest: + name: the oldest dependencies pyproject allows + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + + - name: Resolve the lowest direct dependencies + # uv sync alone keeps a lockfile that already satisfies pyproject, so + # the lowest resolution is locked first and then installed as written. + run: uv lock --resolution lowest-direct + + - name: Sync dependencies + run: uv sync --frozen + + - name: Tests + run: uv run --frozen pytest + # The unit suite has twice now passed while the path a new user takes first # was broken: an adapter that raised a bare TypeError when a setting was # missing, and a local arm that failed every case with the reason visible diff --git a/CHANGELOG.md b/CHANGELOG.md index 0626c50..64cfd3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,19 @@ different event from one that moved because it was wrong. ### Fixed +- The dependency floors in pyproject were never tested, and three were wrong: + scipy 1.14.0 has no wheel for Python 3.13, anthropic before 0.77 lacks the + structured output types the generative adapter uses, and typer before 0.16 + breaks against click 8.2 (#47). The floors are now the oldest releases the + whole suite passes on: numpy 2.1, scipy 1.14.1, typer 0.16, typesafe-sdk + 0.5.7, and anthropic 0.77. The two SDKs are capped below their next breaking + series, httpx is no longer declared (nothing imported it), and a new CI job + runs the suite on the oldest direct dependencies pyproject allows. +- A broken or missing SDK took down every command, the mock included, because + the registry imported all four adapters at startup (#47). An adapter is now + imported the first time it is created, and one that cannot be says which + module is missing; the other adapters, and the adapters list, are unaffected. + - A row whose response named no model was never priced, because pricing looked up only the model reported; the guard, which prices the requested model, and the report then disagreed about the same run (#36). Such a row is diff --git a/pyproject.toml b/pyproject.toml index b593acc..bb52ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,13 +13,15 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] +# Each floor is the oldest release the whole test suite passes on, checked by +# the lowest-direct job in CI. The SDKs are capped below their next breaking +# series, since an adapter depends on their request and response shapes. dependencies = [ "numpy>=2.1", - "scipy>=1.14", - "typer>=0.15", - "httpx>=0.28", - "typesafe-sdk>=0.1", - "anthropic>=0.40", + "scipy>=1.14.1", + "typer>=0.16", + "typesafe-sdk>=0.5.7,<0.8", + "anthropic>=0.77,<2", ] [project.optional-dependencies] @@ -43,6 +45,9 @@ dev = [ "pytest-cov>=6.0", "ruff>=0.8", "mypy>=1.13", + # The CLI tests read stderr apart from stdout, which Click's test runner + # does from 8.2. + "click>=8.2", ] [build-system] diff --git a/src/plumbline/adapters/__init__.py b/src/plumbline/adapters/__init__.py index b0c8a6f..c8774c1 100644 --- a/src/plumbline/adapters/__init__.py +++ b/src/plumbline/adapters/__init__.py @@ -1,11 +1,30 @@ -"""Adapters, organized by transport rather than by vendor.""" +"""Adapters, organized by transport rather than by vendor. + +The adapter classes are imported on first access, not with the package, so a +broken or missing SDK affects only the adapter that needs it. +""" + +from __future__ import annotations + +import importlib +from typing import Any from plumbline.adapters.base import Adapter, check_probability_semantics -from plumbline.adapters.generative import GenerativeAdapter -from plumbline.adapters.local_logits import LocalLogitsAdapter -from plumbline.adapters.mock import MockAdapter from plumbline.adapters.registry import available, create, register +_LAZY = { + "GenerativeAdapter": "plumbline.adapters.generative", + "LocalLogitsAdapter": "plumbline.adapters.local_logits", + "MockAdapter": "plumbline.adapters.mock", +} + + +def __getattr__(name: str) -> Any: + if name in _LAZY: + return getattr(importlib.import_module(_LAZY[name]), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + __all__ = [ "Adapter", "GenerativeAdapter", diff --git a/src/plumbline/adapters/registry.py b/src/plumbline/adapters/registry.py index c216223..25ddbff 100644 --- a/src/plumbline/adapters/registry.py +++ b/src/plumbline/adapters/registry.py @@ -8,33 +8,39 @@ from __future__ import annotations +import importlib import inspect from collections.abc import Callable from typing import Any from plumbline.adapters.base import Adapter -from plumbline.types import UnknownAdapterError +from plumbline.types import PlumblineError, UnknownAdapterError AdapterFactory = Callable[..., Adapter] _REGISTRY: dict[str, AdapterFactory] = {} +#: Built-in adapters by module and class, imported the first time they are +#: created. Importing all four at startup meant one broken or missing SDK took +#: the whole command line down, the mock included. +_BUILTINS: dict[str, tuple[str, str]] = { + "generative": ("plumbline.adapters.generative", "GenerativeAdapter"), + "local_logits": ("plumbline.adapters.local_logits", "LocalLogitsAdapter"), + "mock": ("plumbline.adapters.mock", "MockAdapter"), + "typesafe_wire": ("plumbline.adapters.typesafe_wire", "TypeSafeWireAdapter"), +} + def register(name: str, factory: AdapterFactory) -> None: """Bind ``name`` to a factory. Re-registering the same name is an error.""" - if name in _REGISTRY: + if name in _REGISTRY or name in _BUILTINS: raise ValueError(f"adapter {name!r} is already registered") _REGISTRY[name] = factory def create(name: str, **config: Any) -> Adapter: """Build a configured adapter by registered name.""" - try: - factory = _REGISTRY[name] - except KeyError: - raise UnknownAdapterError( - f"no adapter registered as {name!r}. Available: {available()!r}" - ) from None + factory = _factory(name) missing = _missing_arguments(factory, config) if missing: @@ -63,6 +69,26 @@ def create(name: str, **config: Any) -> Adapter: return factory(**config) +def _factory(name: str) -> AdapterFactory: + """The factory registered as ``name``, importing a built-in on first use.""" + if name in _REGISTRY: + return _REGISTRY[name] + if name not in _BUILTINS: + raise UnknownAdapterError(f"no adapter registered as {name!r}. Available: {available()!r}") + module_name, class_name = _BUILTINS[name] + try: + module = importlib.import_module(module_name) + except ImportError as missing: + raise PlumblineError( + f"the {name} adapter could not be loaded, because a module it needs could not " + f"be imported: {missing}. The other adapters are unaffected. Reinstall " + "plumbline's dependencies (uv sync) to restore it." + ) from missing + factory: AdapterFactory = getattr(module, class_name) + _REGISTRY[name] = factory + return factory + + def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[str]: """Required parameters of ``factory`` that ``config`` does not supply. @@ -85,20 +111,5 @@ def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[ def available() -> tuple[str, ...]: - """Registered adapter names, sorted.""" - return tuple(sorted(_REGISTRY)) - - -def _register_builtins() -> None: - from plumbline.adapters.generative import GenerativeAdapter - from plumbline.adapters.local_logits import LocalLogitsAdapter - from plumbline.adapters.mock import MockAdapter - from plumbline.adapters.typesafe_wire import TypeSafeWireAdapter - - register("generative", GenerativeAdapter) - register("local_logits", LocalLogitsAdapter) - register("mock", MockAdapter) - register("typesafe_wire", TypeSafeWireAdapter) - - -_register_builtins() + """Registered adapter names, sorted, without importing any of them.""" + return tuple(sorted({*_REGISTRY, *_BUILTINS})) diff --git a/tests/test_registry.py b/tests/test_registry.py index 8154225..2c7d20f 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -33,3 +33,29 @@ def test_unknown_name_names_what_is_available() -> None: def test_duplicate_registration_is_an_error() -> None: with pytest.raises(ValueError, match="already registered"): registry.register("mock", MockAdapter) + + +def test_a_broken_optional_sdk_breaks_only_the_adapter_that_needs_it() -> None: + """With anthropic unimportable, the mock must still run and generative must say why (#47). + + Run in a fresh interpreter, so the SDK can be made to fail to import without + disturbing the modules this test session already loaded. + """ + import subprocess + import sys + + script = """ +import sys +sys.modules["anthropic"] = None # makes `import anthropic` raise ImportError +from plumbline.adapters import registry +assert "generative" in registry.available() +registry.create("mock", gold_by_text={"t": "a"}) +try: + registry.create("generative") +except Exception as error: + print(type(error).__name__, "|", error) +""" + done = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + assert done.returncode == 0, done.stderr + assert "PlumblineError" in done.stdout and "anthropic" in done.stdout diff --git a/uv.lock b/uv.lock index 83bb78d..a3380d8 100644 --- a/uv.lock +++ b/uv.lock @@ -1116,7 +1116,6 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "anthropic" }, - { name = "httpx" }, { name = "numpy" }, { name = "scipy" }, { name = "typer" }, @@ -1131,6 +1130,7 @@ local = [ [package.dev-dependencies] dev = [ + { name = "click" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -1139,19 +1139,19 @@ dev = [ [package.metadata] requires-dist = [ - { name = "anthropic", specifier = ">=0.40" }, - { name = "httpx", specifier = ">=0.28" }, + { name = "anthropic", specifier = ">=0.77,<2" }, { name = "numpy", specifier = ">=2.1" }, - { name = "scipy", specifier = ">=1.14" }, + { name = "scipy", specifier = ">=1.14.1" }, { name = "torch", marker = "extra == 'local'", specifier = ">=2.5" }, { name = "transformers", marker = "extra == 'local'", specifier = ">=4.46" }, - { name = "typer", specifier = ">=0.15" }, - { name = "typesafe-sdk", specifier = ">=0.1" }, + { name = "typer", specifier = ">=0.16" }, + { name = "typesafe-sdk", specifier = ">=0.5.7,<0.8" }, ] provides-extras = ["local"] [package.metadata.requires-dev] dev = [ + { name = "click", specifier = ">=8.2" }, { name = "mypy", specifier = ">=1.13" }, { name = "pytest", specifier = ">=8.3" }, { name = "pytest-cov", specifier = ">=6.0" },