Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,33 @@ jobs:
- name: Tests
run: uv run pytest

# uv.lock pins the newest releases, so the matrix never sees the oldest ones
# pyproject still admits. The floors were once set by hand and were wrong for
# three packages at once. This job installs the oldest direct dependency each
# floor allows and runs the suite on it, so a floor that no longer holds fails
# here, not on a user's machine.
lowest:
name: the oldest dependencies pyproject allows
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5

- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"

- name: Resolve the lowest direct dependencies
# uv sync alone keeps a lockfile that already satisfies pyproject, so
# the lowest resolution is locked first and then installed as written.
run: uv lock --resolution lowest-direct

- name: Sync dependencies
run: uv sync --frozen

- name: Tests
run: uv run --frozen pytest

# The unit suite has twice now passed while the path a new user takes first
# was broken: an adapter that raised a bare TypeError when a setting was
# missing, and a local arm that failed every case with the reason visible
Expand Down
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,19 @@ different event from one that moved because it was wrong.

### Fixed

- The dependency floors in pyproject were never tested, and three were wrong:
scipy 1.14.0 has no wheel for Python 3.13, anthropic before 0.77 lacks the
structured output types the generative adapter uses, and typer before 0.16
breaks against click 8.2 (#47). The floors are now the oldest releases the
whole suite passes on: numpy 2.1, scipy 1.14.1, typer 0.16, typesafe-sdk
0.5.7, and anthropic 0.77. The two SDKs are capped below their next breaking
series, httpx is no longer declared (nothing imported it), and a new CI job
runs the suite on the oldest direct dependencies pyproject allows.
- A broken or missing SDK took down every command, the mock included, because
the registry imported all four adapters at startup (#47). An adapter is now
imported the first time it is created, and one that cannot be says which
module is missing; the other adapters, and the adapters list, are unaffected.

- A row whose response named no model was never priced, because pricing
looked up only the model reported; the guard, which prices the requested
model, and the report then disagreed about the same run (#36). Such a row is
Expand Down
15 changes: 10 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ classifiers = [
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
]
# Each floor is the oldest release the whole test suite passes on, checked by
# the lowest-direct job in CI. The SDKs are capped below their next breaking
# series, since an adapter depends on their request and response shapes.
dependencies = [
"numpy>=2.1",
"scipy>=1.14",
"typer>=0.15",
"httpx>=0.28",
"typesafe-sdk>=0.1",
"anthropic>=0.40",
"scipy>=1.14.1",
"typer>=0.16",
"typesafe-sdk>=0.5.7,<0.8",
"anthropic>=0.77,<2",
]

[project.optional-dependencies]
Expand All @@ -43,6 +45,9 @@ dev = [
"pytest-cov>=6.0",
"ruff>=0.8",
"mypy>=1.13",
# The CLI tests read stderr apart from stdout, which Click's test runner
# does from 8.2.
"click>=8.2",
]

[build-system]
Expand Down
27 changes: 23 additions & 4 deletions src/plumbline/adapters/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,30 @@
"""Adapters, organized by transport rather than by vendor."""
"""Adapters, organized by transport rather than by vendor.

The adapter classes are imported on first access, not with the package, so a
broken or missing SDK affects only the adapter that needs it.
"""

from __future__ import annotations

import importlib
from typing import Any

from plumbline.adapters.base import Adapter, check_probability_semantics
from plumbline.adapters.generative import GenerativeAdapter
from plumbline.adapters.local_logits import LocalLogitsAdapter
from plumbline.adapters.mock import MockAdapter
from plumbline.adapters.registry import available, create, register

_LAZY = {
"GenerativeAdapter": "plumbline.adapters.generative",
"LocalLogitsAdapter": "plumbline.adapters.local_logits",
"MockAdapter": "plumbline.adapters.mock",
}


def __getattr__(name: str) -> Any:
if name in _LAZY:
return getattr(importlib.import_module(_LAZY[name]), name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


__all__ = [
"Adapter",
"GenerativeAdapter",
Expand Down
61 changes: 36 additions & 25 deletions src/plumbline/adapters/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,39 @@

from __future__ import annotations

import importlib
import inspect
from collections.abc import Callable
from typing import Any

from plumbline.adapters.base import Adapter
from plumbline.types import UnknownAdapterError
from plumbline.types import PlumblineError, UnknownAdapterError

AdapterFactory = Callable[..., Adapter]

_REGISTRY: dict[str, AdapterFactory] = {}

#: Built-in adapters by module and class, imported the first time they are
#: created. Importing all four at startup meant one broken or missing SDK took
#: the whole command line down, the mock included.
_BUILTINS: dict[str, tuple[str, str]] = {
"generative": ("plumbline.adapters.generative", "GenerativeAdapter"),
"local_logits": ("plumbline.adapters.local_logits", "LocalLogitsAdapter"),
"mock": ("plumbline.adapters.mock", "MockAdapter"),
"typesafe_wire": ("plumbline.adapters.typesafe_wire", "TypeSafeWireAdapter"),
}


def register(name: str, factory: AdapterFactory) -> None:
"""Bind ``name`` to a factory. Re-registering the same name is an error."""
if name in _REGISTRY:
if name in _REGISTRY or name in _BUILTINS:
raise ValueError(f"adapter {name!r} is already registered")
_REGISTRY[name] = factory


def create(name: str, **config: Any) -> Adapter:
"""Build a configured adapter by registered name."""
try:
factory = _REGISTRY[name]
except KeyError:
raise UnknownAdapterError(
f"no adapter registered as {name!r}. Available: {available()!r}"
) from None
factory = _factory(name)

missing = _missing_arguments(factory, config)
if missing:
Expand Down Expand Up @@ -63,6 +69,26 @@ def create(name: str, **config: Any) -> Adapter:
return factory(**config)


def _factory(name: str) -> AdapterFactory:
"""The factory registered as ``name``, importing a built-in on first use."""
if name in _REGISTRY:
return _REGISTRY[name]
if name not in _BUILTINS:
raise UnknownAdapterError(f"no adapter registered as {name!r}. Available: {available()!r}")
module_name, class_name = _BUILTINS[name]
try:
module = importlib.import_module(module_name)
except ImportError as missing:
raise PlumblineError(
f"the {name} adapter could not be loaded, because a module it needs could not "
f"be imported: {missing}. The other adapters are unaffected. Reinstall "
"plumbline's dependencies (uv sync) to restore it."
) from missing
factory: AdapterFactory = getattr(module, class_name)
_REGISTRY[name] = factory
return factory


def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[str]:
"""Required parameters of ``factory`` that ``config`` does not supply.

Expand All @@ -85,20 +111,5 @@ def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[


def available() -> tuple[str, ...]:
"""Registered adapter names, sorted."""
return tuple(sorted(_REGISTRY))


def _register_builtins() -> None:
from plumbline.adapters.generative import GenerativeAdapter
from plumbline.adapters.local_logits import LocalLogitsAdapter
from plumbline.adapters.mock import MockAdapter
from plumbline.adapters.typesafe_wire import TypeSafeWireAdapter

register("generative", GenerativeAdapter)
register("local_logits", LocalLogitsAdapter)
register("mock", MockAdapter)
register("typesafe_wire", TypeSafeWireAdapter)


_register_builtins()
"""Registered adapter names, sorted, without importing any of them."""
return tuple(sorted({*_REGISTRY, *_BUILTINS}))
26 changes: 26 additions & 0 deletions tests/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,29 @@ def test_unknown_name_names_what_is_available() -> None:
def test_duplicate_registration_is_an_error() -> None:
with pytest.raises(ValueError, match="already registered"):
registry.register("mock", MockAdapter)


def test_a_broken_optional_sdk_breaks_only_the_adapter_that_needs_it() -> None:
"""With anthropic unimportable, the mock must still run and generative must say why (#47).

Run in a fresh interpreter, so the SDK can be made to fail to import without
disturbing the modules this test session already loaded.
"""
import subprocess
import sys

script = """
import sys
sys.modules["anthropic"] = None # makes `import anthropic` raise ImportError
from plumbline.adapters import registry
assert "generative" in registry.available()
registry.create("mock", gold_by_text={"t": "a"})
try:
registry.create("generative")
except Exception as error:
print(type(error).__name__, "|", error)
"""
done = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True)

assert done.returncode == 0, done.stderr
assert "PlumblineError" in done.stdout and "anthropic" in done.stdout
12 changes: 6 additions & 6 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading