Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,20 @@ jobs:
- run: pip install build
- run: python -m build --wheel
- run: bash scripts/smoke-wheel.sh dist/*.whl

otel-wire:
# The offline suite fakes the SDK, so nothing used to run the real exporter:
# `export_span`/`export_metric` were `# pragma: no cover`, and the fake
# instruments hid a real `Histogram.add` AttributeError. Here the
# `[otel]` extra is installed and one span plus one metric go to a port
# nothing listens on: the refusal must be logged and contained, never raised
# out of `export_run`, and a real export must still return its count.
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- run: pip install -e ".[otel]" pytest
- run: python scripts/otel_wire_smoke.py
- run: AGENTDESCENT_OTEL_WIRE=1 pytest -q tests/test_otel_wire.py tests/test_otel_export.py
14 changes: 14 additions & 0 deletions changelog.d/1083-otel-wire-path.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
Added
An `otel-wire` CI leg installs the `[otel]` extra and exports one span and one
metric to a closed port on `127.0.0.1`, asserting a refused connection is logged
and contained rather than raised out of `export_run`. A test pins that
`export_span` copies `ExportedSpan.attributes` verbatim — it filters and invents
nothing — so the no-content guarantee is `export_run`'s and the caller's for
spans built by hand (#1083).

The leg also keeps the fail-fast row real where the extra is installed: the SDK
import is halted through the interpreter's own `sys.modules` marker instead of a
probe-and-skip, so construction still has to name the install line, and the
loopback endpoint rows assert the accepted endpoint on a stubbed seam rather than
building a live provider. `docs/metrics.md` now says credentials go in
`OTEL_EXPORTER_OTLP_HEADERS`, never in the URL (#1083).
14 changes: 13 additions & 1 deletion docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,10 @@ if exporter is not None:
credentials (`https://user:token@host/`) is refused without the value being
repeated in the error, and plain `http://` is accepted only for a literal
loopback collector (`localhost`, `127.0.0.0/8`, `[::1]`). A remote collector
needs TLS terminated in front of it.
needs TLS terminated in front of it. Credentials belong in
`OTEL_EXPORTER_OTLP_HEADERS`, never in the URL: this module does not echo an
accepted endpoint, but the SDK's httpx errors quote the request URL, so a
token in the path or query would still reach a log.
- Delivery is per exporter and explicit. Each `OtlpExporter` owns its tracer and
meter providers, so a second exporter in the same process does not export
through the first one's endpoint or headers. The SDK batches spans for 5 s and
Expand All @@ -89,3 +92,12 @@ if exporter is not None:
guarantee belongs to `export_run`, while an `ExportedSpan` handed straight to
`export_span` has its `attributes` copied verbatim, so a caller building one
by hand owns what goes in it.
- The wire path is exercised in CI, not only faked: the `otel-wire` job installs
the `[otel]` extra and runs `scripts/otel_wire_smoke.py`, which exports one
span and one metric to a closed port on `127.0.0.1` and asserts the refusal is
reported and contained — `export_run` still returns its count, `force_flush()`
and `shutdown()` still return, and nothing escapes.
- `export_span` copies `ExportedSpan.attributes` **verbatim**: it neither drops
nor invents a key, so a caller that builds a span by hand owns its contents,
and the fixed attribute set above is `export_run`'s guarantee alone
(`tests/test_otel_wire.py`, `tests/test_otel_export.py`).
132 changes: 132 additions & 0 deletions scripts/otel_wire_smoke.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Exercise the real OTLP wire path against a closed port on 127.0.0.1 (#1083).

The offline suite stands a fake SDK in at ``otel_export._load_otel``, so nothing
there proves what the *real* exporter does when the collector is unreachable and
``export_span``/``export_metric`` are marked ``# pragma: no cover``. This script
is run by the ``otel-wire`` CI job with the ``[otel]`` extra installed:

1. it picks a port on 127.0.0.1 that has no listener,
2. exports one round span, one reward histogram and one decision span through an
``OtlpExporter`` built the way a host builds it (``exporter_from_env``),
3. and asserts the documented promise: ``export_run`` still returns its count,
no exception escapes, ``force_flush()``/``shutdown()`` do not raise, and the
SDK *logs* the refused connection -- which is what makes this a wire-path
test rather than a no-op.

Exit status is 0 only when all of that holds; anything else is a promise that
broke and fails the job.
"""

from __future__ import annotations

import logging
import socket
import sys
import traceback

from agentdescent.otel_export import (
DecisionSample,
RoundSample,
export_run,
exporter_from_env,
)


def _closed_port() -> int:
"""A port on 127.0.0.1 that is free now and has no listener when we use it."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", 0))
return int(probe.getsockname()[1])


class _Capture(logging.Handler):
"""Keep the SDK's own diagnostics, which are the only proof of the attempt."""

def __init__(self) -> None:
super().__init__(level=logging.DEBUG)
self.messages: list[str] = []

def emit(self, record: logging.LogRecord) -> None:
try:
self.messages.append(record.getMessage())
except Exception: # never let the probe itself fail
self.messages.append(repr(record.msg))


def _mentions_a_refused_connection(messages: list[str]) -> bool:
haystack = "\n".join(messages).lower()
return any(marker in haystack for marker in (
"connection refused",
"connectionrefused",
"failed to export",
"max retries",
))


def main() -> int:
port = _closed_port()
endpoint = f"http://127.0.0.1:{port}"
capture = _Capture()
logger = logging.getLogger("opentelemetry")
logger.addHandler(capture)
logger.setLevel(logging.DEBUG)

problems: list[str] = []

try: # construction is a run-start step
exporter = exporter_from_env(
{"OTEL_EXPORTER_OTLP_ENDPOINT": endpoint})
except ImportError as exc:
print(f"the [otel] extra is not installed: {exc}", file=sys.stderr)
print("this job must run `pip install -e \".[otel]\"` first", file=sys.stderr)
return 2

try:
exported = export_run(
exporter,
run_id="wire-1",
rounds=[RoundSample(1, reward=0.5, tokens=10, wallclock=1.0,
start=1.0, end=2.0)],
decisions=[DecisionSample("skill", True, calls=1, prompt_tokens=5,
wallclock=0.1, start=2.0, end=2.1)],
)
flushed = exporter.force_flush(3000)
except Exception: # the promise under test
print("export_run/force_flush raised out:", file=sys.stderr)
traceback.print_exc()
return 1
finally:
try:
exporter.shutdown()
except Exception:
print("shutdown() raised out:", file=sys.stderr)
traceback.print_exc()
problems.append("shutdown raised")

# 1 round span + 1 reward histogram + 1 decision span; no aggregate metrics
# were passed, so nothing else is counted.
if exported != 3:
problems.append(f"export_run returned {exported}, expected 3")
if not isinstance(flushed, bool):
problems.append(f"force_flush returned {type(flushed).__name__}, not bool")
if not _mentions_a_refused_connection(capture.messages):
problems.append("the SDK never logged a refused connection: the wire "
"path was not attempted")
if any("traceback" in message.lower() for message in capture.messages):
problems.append("the SDK logged a traceback instead of a handled error")

print(f"endpoint={endpoint} exported={exported} force_flush={flushed} "
f"sdk-messages={len(capture.messages)}")
for message in capture.messages[:6]:
print(" sdk:", " ".join(message.split())[:200])
if problems:
for problem in problems:
print(f"FAILED: {problem}", file=sys.stderr)
return 1
print("ok: the wire path degraded as documented")
return 0


if __name__ == "__main__":
raise SystemExit(main())
82 changes: 65 additions & 17 deletions tests/test_otel_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,21 +151,46 @@ def test_the_exporter_is_off_unless_an_endpoint_is_configured() -> None:
assert exporter_from_env({"OTEL_EXPORTER_OTLP_ENDPOINT": ""}) is None


def test_a_configured_endpoint_needs_the_extra_at_construction() -> None:
"""Constructing the exporter imports the SDK, so a missing extra fails
before a run starts instead of on the first span written."""
try: # the same probe the exporter uses
otel_export._load_otel()
except ImportError:
pass
else:
pytest.skip("the [otel] extra is installed here")
@pytest.fixture
def missing_sdk(monkeypatch):
"""Make ``import opentelemetry`` fail even when the extra is installed.

The `otel-wire` leg installs `[otel]`, so a probe-and-skip row would stop
testing the fail-fast contract in exactly the environment that has the SDK
(#1083). ``None`` in ``sys.modules`` is the interpreter's own "import
halted" marker, so this walks the real ImportError path and not a fake
seam: `_load_otel` catches it and re-raises with the install line.
"""
for name in [n for n in sys.modules
if n == "opentelemetry" or n.startswith("opentelemetry.")]:
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.setitem(sys.modules, "opentelemetry", None)


def test_a_configured_endpoint_needs_the_extra_at_construction(
missing_sdk) -> None:
"""Constructing the exporter imports the SDK, so a missing extra fails
before a run starts instead of on the first span written -- asserted with
the SDK absent, whether or not this host installed it."""
with pytest.raises(ImportError) as excinfo:
exporter_from_env(
{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318"})
assert "agentdescent[otel]" in str(excinfo.value)

with pytest.raises(ImportError) as excinfo: # and the direct entry point
otel_export.OtlpExporter("http://127.0.0.1:4318")
assert "agentdescent[otel]" in str(excinfo.value)


def test_the_install_line_is_named_when_the_extra_really_is_absent() -> None:
"""One real-import row: skipped only where the SDK is genuinely present."""
try:
otel_export._load_otel()
except ImportError as exc:
assert "agentdescent[otel]" in str(exc)
else:
pytest.skip("the [otel] extra is installed here")


@pytest.mark.parametrize("endpoint", [
"https://user:token@collector.example:4318",
Expand Down Expand Up @@ -204,14 +229,16 @@ def test_a_plaintext_endpoint_off_the_loopback_is_refused(endpoint) -> None:
"http://127.1.2.3:4318",
"http://[::1]:4318",
])
def test_a_loopback_collector_may_be_plain_http(endpoint) -> None:
"""Loopback http stays legal; the SDK missing is the only error here."""
try:
otel_export.OtlpExporter(endpoint)
except ImportError as exc:
assert "agentdescent[otel]" in str(exc)
else: # the extra is installed here
pytest.skip("the [otel] extra is installed here")
def test_a_loopback_collector_may_be_plain_http(endpoint, monkeypatch) -> None:
"""Loopback http stays legal: the endpoint is accepted and kept as given.

The SDK seam is stubbed, so this row asserts the bridge's own decision and
opens no live provider pointed at `127.0.0.1:4318` -- before #1083 it built
a real exporter on a host with the extra and then skipped itself.
"""
monkeypatch.setattr(otel_export, "_load_otel", _fake_otel)
exporter = otel_export.OtlpExporter(endpoint)
assert exporter.endpoint == endpoint


def test_the_environment_path_refuses_a_refused_endpoint_too() -> None:
Expand Down Expand Up @@ -475,3 +502,24 @@ def test_the_instrument_methods_match_the_sdk(monkeypatch) -> None:
"run.id": "run-1"})]
assert instruments["agentdescent_rsi_decisions_total"].added == [(2.0, {})]
assert instruments["agentdescent_rsi_acceptance_rate"].set_values


def test_export_span_copies_attributes_verbatim_and_invents_nothing(monkeypatch) -> None:
"""``export_span`` copies what it is given: no filtering, no invented keys.

So the no-content guarantee is ``export_run``'s (its writer builds a fixed
attribute set), and a caller handing the exporter its own ``ExportedSpan``
owns every key in it -- load-bearing while ``rsi_telemetry`` spans can feed
the exporter with the redaction question in #947 still open (#1083).
"""
monkeypatch.setattr(otel_export, "_load_otel", _fake_otel)
exporter = otel_export.OtlpExporter("https://a.example:4318")
attributes = {"run.id": "run-1", "caller.note": "free text, spaces, 42"}
span = ExportedSpan("agentdescent.round", attributes, 1.0, 2.0)

exporter.export_span(span)

recorded = exporter._tracer.spans[-1]
assert recorded.attributes == attributes # verbatim, nothing dropped
assert recorded.attributes is not attributes # a copy, not an alias
assert "caller.note" in recorded.attributes # nothing invented either
97 changes: 97 additions & 0 deletions tests/test_otel_wire.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""The real OTLP wire path, and the CI leg that runs it (#1083).

The offline suite stands a fake SDK in at ``otel_export._load_otel``: it pins the
wiring but cannot pin the SDK's own API or what happens when a collector is
unreachable. Two rows here cover exactly that gap:

1. with ``AGENTDESCENT_OTEL_WIRE=1`` (set by the ``otel-wire`` CI job) a real
exporter is pointed at a closed port on 127.0.0.1 and the export must be
logged and contained, never raised out of ``export_run``;
2. the workflow leg is asserted here, so deleting the only place the wire path
runs fails a test instead of silently dropping the coverage.

The socket row is opt-in because the suite is offline by contract; the default
run never opens a socket.
"""

from __future__ import annotations

import logging
import os
import pathlib
import socket

import pytest

ROOT = pathlib.Path(__file__).resolve().parent.parent
SMOKE_SCRIPT = ROOT / "scripts" / "otel_wire_smoke.py"


class _Capture(logging.Handler):
def __init__(self) -> None:
super().__init__(level=logging.DEBUG)
self.messages: list = []

def emit(self, record: logging.LogRecord) -> None:
self.messages.append(record.getMessage())


@pytest.mark.skipif(
os.environ.get("AGENTDESCENT_OTEL_WIRE") != "1",
reason="real socket path; the otel-wire CI leg sets AGENTDESCENT_OTEL_WIRE=1",
)
def test_the_wire_path_degrades_instead_of_raising() -> None:
"""A refused connection must be logged, not raised out of ``export_run``."""
from agentdescent import otel_export
from agentdescent.otel_export import (
DecisionSample,
RoundSample,
export_run,
exporter_from_env,
)

try:
otel_export._load_otel()
except ImportError:
pytest.skip("the [otel] extra is not installed")

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", 0))
port = int(probe.getsockname()[1]) # closed again by the context exit

capture = _Capture()
logger = logging.getLogger("opentelemetry")
logger.addHandler(capture)
logger.setLevel(logging.DEBUG)
try:
exporter = exporter_from_env(
{"OTEL_EXPORTER_OTLP_ENDPOINT": f"http://127.0.0.1:{port}"})
exported = export_run(
exporter, run_id="wire-1",
rounds=(RoundSample(1, reward=0.5, tokens=10, wallclock=1.0,
start=1.0, end=2.0),),
decisions=(DecisionSample("skill", True, calls=1, prompt_tokens=5,
wallclock=0.1, start=2.0, end=2.1),))
flushed = exporter.force_flush(3000)
exporter.shutdown() # must not raise either
finally:
logger.removeHandler(capture)

assert exported == 3 # one round span, one reward, one decision
assert isinstance(flushed, bool)
reported = "\n".join(capture.messages).lower()
assert any(marker in reported for marker in (
"connection refused", "connectionrefused", "failed to export",
"max retries")), "the SDK never reported the refused connection"


def test_the_ci_leg_installs_the_extra_and_runs_the_smoke_script() -> None:
"""The wire path has exactly one place that runs it; this pins that place."""
assert SMOKE_SCRIPT.is_file()

workflow = (ROOT / ".github" / "workflows" / "tests.yml").read_text(
encoding="utf-8")
assert "otel-wire:" in workflow
assert "otel]" in workflow # pip install -e ".[otel]"
assert "scripts/otel_wire_smoke.py" in workflow
assert "AGENTDESCENT_OTEL_WIRE=1" in workflow
Loading