diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 159ba692..6029807d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,3 +40,20 @@ jobs: - run: pip install build - run: python -m build --wheel - run: bash scripts/smoke-wheel.sh dist/*.whl + + otel-wire: + # The offline suite fakes the SDK, so nothing used to run the real exporter: + # `export_span`/`export_metric` were `# pragma: no cover`, and the fake + # instruments hid a real `Histogram.add` AttributeError. Here the + # `[otel]` extra is installed and one span plus one metric go to a port + # nothing listens on: the refusal must be logged and contained, never raised + # out of `export_run`, and a real export must still return its count. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: pip install -e ".[otel]" pytest + - run: python scripts/otel_wire_smoke.py + - run: AGENTDESCENT_OTEL_WIRE=1 pytest -q tests/test_otel_wire.py tests/test_otel_export.py diff --git a/changelog.d/1083-otel-wire-path.md b/changelog.d/1083-otel-wire-path.md new file mode 100644 index 00000000..05a3eca1 --- /dev/null +++ b/changelog.d/1083-otel-wire-path.md @@ -0,0 +1,14 @@ +Added +An `otel-wire` CI leg installs the `[otel]` extra and exports one span and one +metric to a closed port on `127.0.0.1`, asserting a refused connection is logged +and contained rather than raised out of `export_run`. A test pins that +`export_span` copies `ExportedSpan.attributes` verbatim — it filters and invents +nothing — so the no-content guarantee is `export_run`'s and the caller's for +spans built by hand (#1083). + +The leg also keeps the fail-fast row real where the extra is installed: the SDK +import is halted through the interpreter's own `sys.modules` marker instead of a +probe-and-skip, so construction still has to name the install line, and the +loopback endpoint rows assert the accepted endpoint on a stubbed seam rather than +building a live provider. `docs/metrics.md` now says credentials go in +`OTEL_EXPORTER_OTLP_HEADERS`, never in the URL (#1083). diff --git a/docs/metrics.md b/docs/metrics.md index 8fcf58ef..4322cbe2 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -74,7 +74,10 @@ if exporter is not None: credentials (`https://user:token@host/`) is refused without the value being repeated in the error, and plain `http://` is accepted only for a literal loopback collector (`localhost`, `127.0.0.0/8`, `[::1]`). A remote collector - needs TLS terminated in front of it. + needs TLS terminated in front of it. Credentials belong in + `OTEL_EXPORTER_OTLP_HEADERS`, never in the URL: this module does not echo an + accepted endpoint, but the SDK's httpx errors quote the request URL, so a + token in the path or query would still reach a log. - Delivery is per exporter and explicit. Each `OtlpExporter` owns its tracer and meter providers, so a second exporter in the same process does not export through the first one's endpoint or headers. The SDK batches spans for 5 s and @@ -89,3 +92,12 @@ if exporter is not None: guarantee belongs to `export_run`, while an `ExportedSpan` handed straight to `export_span` has its `attributes` copied verbatim, so a caller building one by hand owns what goes in it. +- The wire path is exercised in CI, not only faked: the `otel-wire` job installs + the `[otel]` extra and runs `scripts/otel_wire_smoke.py`, which exports one + span and one metric to a closed port on `127.0.0.1` and asserts the refusal is + reported and contained — `export_run` still returns its count, `force_flush()` + and `shutdown()` still return, and nothing escapes. +- `export_span` copies `ExportedSpan.attributes` **verbatim**: it neither drops + nor invents a key, so a caller that builds a span by hand owns its contents, + and the fixed attribute set above is `export_run`'s guarantee alone + (`tests/test_otel_wire.py`, `tests/test_otel_export.py`). diff --git a/scripts/otel_wire_smoke.py b/scripts/otel_wire_smoke.py new file mode 100644 index 00000000..c97a4900 --- /dev/null +++ b/scripts/otel_wire_smoke.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Exercise the real OTLP wire path against a closed port on 127.0.0.1 (#1083). + +The offline suite stands a fake SDK in at ``otel_export._load_otel``, so nothing +there proves what the *real* exporter does when the collector is unreachable and +``export_span``/``export_metric`` are marked ``# pragma: no cover``. This script +is run by the ``otel-wire`` CI job with the ``[otel]`` extra installed: + +1. it picks a port on 127.0.0.1 that has no listener, +2. exports one round span, one reward histogram and one decision span through an + ``OtlpExporter`` built the way a host builds it (``exporter_from_env``), +3. and asserts the documented promise: ``export_run`` still returns its count, + no exception escapes, ``force_flush()``/``shutdown()`` do not raise, and the + SDK *logs* the refused connection -- which is what makes this a wire-path + test rather than a no-op. + +Exit status is 0 only when all of that holds; anything else is a promise that +broke and fails the job. +""" + +from __future__ import annotations + +import logging +import socket +import sys +import traceback + +from agentdescent.otel_export import ( + DecisionSample, + RoundSample, + export_run, + exporter_from_env, +) + + +def _closed_port() -> int: + """A port on 127.0.0.1 that is free now and has no listener when we use it.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + +class _Capture(logging.Handler): + """Keep the SDK's own diagnostics, which are the only proof of the attempt.""" + + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + try: + self.messages.append(record.getMessage()) + except Exception: # never let the probe itself fail + self.messages.append(repr(record.msg)) + + +def _mentions_a_refused_connection(messages: list[str]) -> bool: + haystack = "\n".join(messages).lower() + return any(marker in haystack for marker in ( + "connection refused", + "connectionrefused", + "failed to export", + "max retries", + )) + + +def main() -> int: + port = _closed_port() + endpoint = f"http://127.0.0.1:{port}" + capture = _Capture() + logger = logging.getLogger("opentelemetry") + logger.addHandler(capture) + logger.setLevel(logging.DEBUG) + + problems: list[str] = [] + + try: # construction is a run-start step + exporter = exporter_from_env( + {"OTEL_EXPORTER_OTLP_ENDPOINT": endpoint}) + except ImportError as exc: + print(f"the [otel] extra is not installed: {exc}", file=sys.stderr) + print("this job must run `pip install -e \".[otel]\"` first", file=sys.stderr) + return 2 + + try: + exported = export_run( + exporter, + run_id="wire-1", + rounds=[RoundSample(1, reward=0.5, tokens=10, wallclock=1.0, + start=1.0, end=2.0)], + decisions=[DecisionSample("skill", True, calls=1, prompt_tokens=5, + wallclock=0.1, start=2.0, end=2.1)], + ) + flushed = exporter.force_flush(3000) + except Exception: # the promise under test + print("export_run/force_flush raised out:", file=sys.stderr) + traceback.print_exc() + return 1 + finally: + try: + exporter.shutdown() + except Exception: + print("shutdown() raised out:", file=sys.stderr) + traceback.print_exc() + problems.append("shutdown raised") + + # 1 round span + 1 reward histogram + 1 decision span; no aggregate metrics + # were passed, so nothing else is counted. + if exported != 3: + problems.append(f"export_run returned {exported}, expected 3") + if not isinstance(flushed, bool): + problems.append(f"force_flush returned {type(flushed).__name__}, not bool") + if not _mentions_a_refused_connection(capture.messages): + problems.append("the SDK never logged a refused connection: the wire " + "path was not attempted") + if any("traceback" in message.lower() for message in capture.messages): + problems.append("the SDK logged a traceback instead of a handled error") + + print(f"endpoint={endpoint} exported={exported} force_flush={flushed} " + f"sdk-messages={len(capture.messages)}") + for message in capture.messages[:6]: + print(" sdk:", " ".join(message.split())[:200]) + if problems: + for problem in problems: + print(f"FAILED: {problem}", file=sys.stderr) + return 1 + print("ok: the wire path degraded as documented") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_otel_export.py b/tests/test_otel_export.py index a5287d7a..b5c967d4 100644 --- a/tests/test_otel_export.py +++ b/tests/test_otel_export.py @@ -151,21 +151,46 @@ def test_the_exporter_is_off_unless_an_endpoint_is_configured() -> None: assert exporter_from_env({"OTEL_EXPORTER_OTLP_ENDPOINT": ""}) is None -def test_a_configured_endpoint_needs_the_extra_at_construction() -> None: - """Constructing the exporter imports the SDK, so a missing extra fails - before a run starts instead of on the first span written.""" - try: # the same probe the exporter uses - otel_export._load_otel() - except ImportError: - pass - else: - pytest.skip("the [otel] extra is installed here") +@pytest.fixture +def missing_sdk(monkeypatch): + """Make ``import opentelemetry`` fail even when the extra is installed. + + The `otel-wire` leg installs `[otel]`, so a probe-and-skip row would stop + testing the fail-fast contract in exactly the environment that has the SDK + (#1083). ``None`` in ``sys.modules`` is the interpreter's own "import + halted" marker, so this walks the real ImportError path and not a fake + seam: `_load_otel` catches it and re-raises with the install line. + """ + for name in [n for n in sys.modules + if n == "opentelemetry" or n.startswith("opentelemetry.")]: + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.setitem(sys.modules, "opentelemetry", None) + +def test_a_configured_endpoint_needs_the_extra_at_construction( + missing_sdk) -> None: + """Constructing the exporter imports the SDK, so a missing extra fails + before a run starts instead of on the first span written -- asserted with + the SDK absent, whether or not this host installed it.""" with pytest.raises(ImportError) as excinfo: exporter_from_env( {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318"}) assert "agentdescent[otel]" in str(excinfo.value) + with pytest.raises(ImportError) as excinfo: # and the direct entry point + otel_export.OtlpExporter("http://127.0.0.1:4318") + assert "agentdescent[otel]" in str(excinfo.value) + + +def test_the_install_line_is_named_when_the_extra_really_is_absent() -> None: + """One real-import row: skipped only where the SDK is genuinely present.""" + try: + otel_export._load_otel() + except ImportError as exc: + assert "agentdescent[otel]" in str(exc) + else: + pytest.skip("the [otel] extra is installed here") + @pytest.mark.parametrize("endpoint", [ "https://user:token@collector.example:4318", @@ -204,14 +229,16 @@ def test_a_plaintext_endpoint_off_the_loopback_is_refused(endpoint) -> None: "http://127.1.2.3:4318", "http://[::1]:4318", ]) -def test_a_loopback_collector_may_be_plain_http(endpoint) -> None: - """Loopback http stays legal; the SDK missing is the only error here.""" - try: - otel_export.OtlpExporter(endpoint) - except ImportError as exc: - assert "agentdescent[otel]" in str(exc) - else: # the extra is installed here - pytest.skip("the [otel] extra is installed here") +def test_a_loopback_collector_may_be_plain_http(endpoint, monkeypatch) -> None: + """Loopback http stays legal: the endpoint is accepted and kept as given. + + The SDK seam is stubbed, so this row asserts the bridge's own decision and + opens no live provider pointed at `127.0.0.1:4318` -- before #1083 it built + a real exporter on a host with the extra and then skipped itself. + """ + monkeypatch.setattr(otel_export, "_load_otel", _fake_otel) + exporter = otel_export.OtlpExporter(endpoint) + assert exporter.endpoint == endpoint def test_the_environment_path_refuses_a_refused_endpoint_too() -> None: @@ -475,3 +502,24 @@ def test_the_instrument_methods_match_the_sdk(monkeypatch) -> None: "run.id": "run-1"})] assert instruments["agentdescent_rsi_decisions_total"].added == [(2.0, {})] assert instruments["agentdescent_rsi_acceptance_rate"].set_values + + +def test_export_span_copies_attributes_verbatim_and_invents_nothing(monkeypatch) -> None: + """``export_span`` copies what it is given: no filtering, no invented keys. + + So the no-content guarantee is ``export_run``'s (its writer builds a fixed + attribute set), and a caller handing the exporter its own ``ExportedSpan`` + owns every key in it -- load-bearing while ``rsi_telemetry`` spans can feed + the exporter with the redaction question in #947 still open (#1083). + """ + monkeypatch.setattr(otel_export, "_load_otel", _fake_otel) + exporter = otel_export.OtlpExporter("https://a.example:4318") + attributes = {"run.id": "run-1", "caller.note": "free text, spaces, 42"} + span = ExportedSpan("agentdescent.round", attributes, 1.0, 2.0) + + exporter.export_span(span) + + recorded = exporter._tracer.spans[-1] + assert recorded.attributes == attributes # verbatim, nothing dropped + assert recorded.attributes is not attributes # a copy, not an alias + assert "caller.note" in recorded.attributes # nothing invented either diff --git a/tests/test_otel_wire.py b/tests/test_otel_wire.py new file mode 100644 index 00000000..597d76b3 --- /dev/null +++ b/tests/test_otel_wire.py @@ -0,0 +1,97 @@ +"""The real OTLP wire path, and the CI leg that runs it (#1083). + +The offline suite stands a fake SDK in at ``otel_export._load_otel``: it pins the +wiring but cannot pin the SDK's own API or what happens when a collector is +unreachable. Two rows here cover exactly that gap: + +1. with ``AGENTDESCENT_OTEL_WIRE=1`` (set by the ``otel-wire`` CI job) a real + exporter is pointed at a closed port on 127.0.0.1 and the export must be + logged and contained, never raised out of ``export_run``; +2. the workflow leg is asserted here, so deleting the only place the wire path + runs fails a test instead of silently dropping the coverage. + +The socket row is opt-in because the suite is offline by contract; the default +run never opens a socket. +""" + +from __future__ import annotations + +import logging +import os +import pathlib +import socket + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SMOKE_SCRIPT = ROOT / "scripts" / "otel_wire_smoke.py" + + +class _Capture(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.mark.skipif( + os.environ.get("AGENTDESCENT_OTEL_WIRE") != "1", + reason="real socket path; the otel-wire CI leg sets AGENTDESCENT_OTEL_WIRE=1", +) +def test_the_wire_path_degrades_instead_of_raising() -> None: + """A refused connection must be logged, not raised out of ``export_run``.""" + from agentdescent import otel_export + from agentdescent.otel_export import ( + DecisionSample, + RoundSample, + export_run, + exporter_from_env, + ) + + try: + otel_export._load_otel() + except ImportError: + pytest.skip("the [otel] extra is not installed") + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + port = int(probe.getsockname()[1]) # closed again by the context exit + + capture = _Capture() + logger = logging.getLogger("opentelemetry") + logger.addHandler(capture) + logger.setLevel(logging.DEBUG) + try: + exporter = exporter_from_env( + {"OTEL_EXPORTER_OTLP_ENDPOINT": f"http://127.0.0.1:{port}"}) + exported = export_run( + exporter, run_id="wire-1", + rounds=(RoundSample(1, reward=0.5, tokens=10, wallclock=1.0, + start=1.0, end=2.0),), + decisions=(DecisionSample("skill", True, calls=1, prompt_tokens=5, + wallclock=0.1, start=2.0, end=2.1),)) + flushed = exporter.force_flush(3000) + exporter.shutdown() # must not raise either + finally: + logger.removeHandler(capture) + + assert exported == 3 # one round span, one reward, one decision + assert isinstance(flushed, bool) + reported = "\n".join(capture.messages).lower() + assert any(marker in reported for marker in ( + "connection refused", "connectionrefused", "failed to export", + "max retries")), "the SDK never reported the refused connection" + + +def test_the_ci_leg_installs_the_extra_and_runs_the_smoke_script() -> None: + """The wire path has exactly one place that runs it; this pins that place.""" + assert SMOKE_SCRIPT.is_file() + + workflow = (ROOT / ".github" / "workflows" / "tests.yml").read_text( + encoding="utf-8") + assert "otel-wire:" in workflow + assert "otel]" in workflow # pip install -e ".[otel]" + assert "scripts/otel_wire_smoke.py" in workflow + assert "AGENTDESCENT_OTEL_WIRE=1" in workflow