Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/MCP_AND_TOOLS.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ def book_flight(

`agentctl validate` warns about any tool parameter without a description (or
with a blank one). It does not fail validation yet.
Parameters LangChain fills in at runtime, such as `config: RunnableConfig` or
`InjectedToolArg` annotations, are never sent to the model and are not checked.

Run `agentctl generate` to create tool stubs. The engine loads each tool once at
graph-build time and wraps it as a LangChain `StructuredTool`. At runtime, only
Expand Down
2 changes: 1 addition & 1 deletion docs/mcp-and-tools.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ def get_order_status(
...
```

`agentctl validate` warns about any tool parameter without a description, or with a blank one. It does not fail validation yet.
`agentctl validate` warns about any tool parameter without a description, or with a blank one. It does not fail validation yet. Parameters LangChain fills in at runtime, such as `config: RunnableConfig` or `InjectedToolArg` annotations, are never sent to the model and are not checked.

<Tip>
Write clear descriptions and typed parameters. The LLM reads both when deciding how to call a tool. Vague descriptions lead to wrong or missed tool calls.
Expand Down
54 changes: 54 additions & 0 deletions src/agent_engine/core/tool_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ def get_customer(customer_code: Annotated[str, "The unique customer code."]) ->
(``ast``), never executed. Anything the scan cannot positively identify — a
missing or unparseable file, a function it cannot find — is skipped and left
to the existing plugin checks and runtime loaders to report.

Parameters LangChain fills in at runtime are left out of the schema sent to the
model, so they are not checked. Being static, the scan recognizes them by the
names they are written with (see ``_is_runtime_injected``).
"""

from __future__ import annotations
Expand Down Expand Up @@ -63,6 +67,8 @@ def _check_tool(path: Path, tool_id: str) -> list[str]:
return []
warnings: list[str] = []
for arg in (*fn.args.posonlyargs, *fn.args.args, *fn.args.kwonlyargs):
if _is_runtime_injected(arg):
continue
description = _annotated_description(arg.annotation)
if description is None:
warnings.append(f'tool "{tool_id}": parameter "{arg.arg}" is missing a description')
Expand All @@ -71,6 +77,54 @@ def _check_tool(path: Path, tool_id: str) -> list[str]:
return warnings


# LangChain drops these parameter names from every tool schema (``FILTERED_ARGS``).
_RUNTIME_ARG_NAMES = frozenset({"callbacks", "run_manager"})


def _is_runtime_injected(arg: ast.arg) -> bool:
"""True for a parameter LangChain fills in itself and never shows the model.

Mirrors ``langchain_core`` tool-schema filtering:

- ``callbacks`` / ``run_manager``, by name;
- ``config: RunnableConfig`` — only the bare type; ``RunnableConfig | None``
is still shown to the model;
- ``ToolRuntime`` or ``ToolRuntime[...]``;
- ``Annotated[T, Injected...]``: ``InjectedToolArg`` and its subclasses
(``InjectedToolCallId``, LangGraph's ``InjectedState`` / ``InjectedStore``),
bare or called.
"""
if arg.arg in _RUNTIME_ARG_NAMES:
return True
annotation = arg.annotation
if annotation is None:
return False
if _type_name(annotation) in {"RunnableConfig", "ToolRuntime"}:
return True
if isinstance(annotation, ast.Subscript):
if _type_name(annotation.value) == "ToolRuntime":
return True
if _is_annotated(annotation.value) and isinstance(annotation.slice, ast.Tuple):
return any(_is_injected_marker(item) for item in annotation.slice.elts[1:])
return False


def _is_injected_marker(node: ast.expr) -> bool:
"""``InjectedState``, ``InjectedState("messages")``, ``tools.InjectedToolArg()``…"""
target = node.func if isinstance(node, ast.Call) else node
name = _type_name(target)
return name is not None and name.startswith("Injected")


def _type_name(node: ast.expr) -> str | None:
"""The bare name of ``X`` or ``module.X``; None for anything else."""
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
return node.attr
return None


def _annotated_description(annotation: ast.expr | None) -> str | None:
"""The first string in ``Annotated[T, ...]`` metadata, or None if there is none."""
if not isinstance(annotation, ast.Subscript) or not _is_annotated(annotation.value):
Expand Down
16 changes: 16 additions & 0 deletions tests/cli/test_validate_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,22 @@ def test_described_tool_parameter_has_no_warnings(tmp_path: Path) -> None:
assert "warnings:" not in CliRunner().invoke(cli, ["validate", spec]).output


def test_runtime_injected_parameter_is_not_reported(tmp_path: Path) -> None:
"""LangChain fills ``config`` in at runtime; the model never sees it."""
spec = _spec_with_tool(
tmp_path,
"from typing import Annotated\n"
"from langchain_core.runnables import RunnableConfig\n\n"
"def lookup(query: Annotated[str, 'What to look up.'], config: RunnableConfig) -> str:\n"
" return query\n",
)

result = validate_spec(spec)

assert result.ok, result.errors
assert result.warnings == []


def test_warnings_are_shown_alongside_errors(tmp_path: Path) -> None:
spec = _spec_with_tool(tmp_path, "def lookup(query: str) -> str:\n return query\n")
# Break the spec so validation fails for an unrelated reason.
Expand Down
95 changes: 95 additions & 0 deletions tests/core/test_tool_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,101 @@ def test_star_args_and_kwargs_are_ignored(tmp_path: Path) -> None:
assert warnings == []


# -- runtime-injected parameters: LangChain fills these in; the model never sees them --


def test_runnable_config_parameter_is_not_reported(tmp_path: Path) -> None:
"""The case from the #156 review: the model's schema only exposes ``query``."""
warnings = scan_one(
tmp_path,
"""\
from typing import Annotated
from langchain_core.runnables import RunnableConfig

def get_customer(query: Annotated[str, "What to look up."], config: RunnableConfig) -> str:
return query
""",
)

assert warnings == []


def test_optional_runnable_config_is_still_reported(tmp_path: Path) -> None:
"""LangChain only hides a bare ``RunnableConfig``; an optional one reaches the model."""
warnings = scan_one(
tmp_path,
"""\
from langchain_core.runnables import RunnableConfig

def get_customer(config: RunnableConfig | None = None) -> str:
return ""
""",
)

assert warnings == ['tool "get_customer": parameter "config" is missing a description']


def test_callback_parameters_are_not_reported(tmp_path: Path) -> None:
warnings = scan_one(
tmp_path,
"def get_customer(callbacks=None, run_manager=None) -> str:\n return ''\n",
)

assert warnings == []


def test_tool_runtime_is_not_reported(tmp_path: Path) -> None:
warnings = scan_one(
tmp_path,
"""\
from langchain.tools import ToolRuntime

def get_customer(runtime: ToolRuntime, other: ToolRuntime[dict, dict]) -> str:
return ""
""",
)

assert warnings == []


def test_injected_annotations_are_not_reported(tmp_path: Path) -> None:
warnings = scan_one(
tmp_path,
"""\
from typing import Annotated
from langchain_core import tools
from langchain_core.tools import InjectedToolArg, InjectedToolCallId
from langgraph.prebuilt import InjectedState, InjectedStore

def get_customer(
call_id: Annotated[str, InjectedToolCallId],
arg: Annotated[str, InjectedToolArg()],
qualified: Annotated[str, tools.InjectedToolArg],
state: Annotated[dict, InjectedState],
messages: Annotated[list, InjectedState("messages")],
store: Annotated[object, InjectedStore()],
) -> str:
return ""
""",
)

assert warnings == []


def test_injected_parameters_do_not_hide_undescribed_model_parameters(tmp_path: Path) -> None:
warnings = scan_one(
tmp_path,
"""\
from langchain_core.runnables import RunnableConfig

def get_customer(query: str, config: RunnableConfig) -> str:
return query
""",
)

assert warnings == ['tool "get_customer": parameter "query" is missing a description']


def test_async_tool_is_scanned(tmp_path: Path) -> None:
warnings = scan_one(
tmp_path,
Expand Down
Loading