diff --git a/docs/MCP_AND_TOOLS.md b/docs/MCP_AND_TOOLS.md index 737289cf..627fd263 100644 --- a/docs/MCP_AND_TOOLS.md +++ b/docs/MCP_AND_TOOLS.md @@ -149,6 +149,8 @@ def book_flight( `agentctl validate` warns about any tool parameter without a description (or with a blank one). It does not fail validation yet. +Parameters LangChain fills in at runtime, such as `config: RunnableConfig` or +`InjectedToolArg` annotations, are never sent to the model and are not checked. Run `agentctl generate` to create tool stubs. The engine loads each tool once at graph-build time and wraps it as a LangChain `StructuredTool`. At runtime, only diff --git a/docs/mcp-and-tools.mdx b/docs/mcp-and-tools.mdx index 69465c25..edc58690 100644 --- a/docs/mcp-and-tools.mdx +++ b/docs/mcp-and-tools.mdx @@ -163,7 +163,7 @@ def get_order_status( ... ``` -`agentctl validate` warns about any tool parameter without a description, or with a blank one. It does not fail validation yet. +`agentctl validate` warns about any tool parameter without a description, or with a blank one. It does not fail validation yet. Parameters LangChain fills in at runtime, such as `config: RunnableConfig` or `InjectedToolArg` annotations, are never sent to the model and are not checked. Write clear descriptions and typed parameters. The LLM reads both when deciding how to call a tool. Vague descriptions lead to wrong or missed tool calls. diff --git a/src/agent_engine/core/tool_metadata.py b/src/agent_engine/core/tool_metadata.py index d5415e47..94665b3b 100644 --- a/src/agent_engine/core/tool_metadata.py +++ b/src/agent_engine/core/tool_metadata.py @@ -17,6 +17,10 @@ def get_customer(customer_code: Annotated[str, "The unique customer code."]) -> (``ast``), never executed. Anything the scan cannot positively identify — a missing or unparseable file, a function it cannot find — is skipped and left to the existing plugin checks and runtime loaders to report. + +Parameters LangChain fills in at runtime are left out of the schema sent to the +model, so they are not checked. Being static, the scan recognizes them by the +names they are written with (see ``_is_runtime_injected``). """ from __future__ import annotations @@ -63,6 +67,8 @@ def _check_tool(path: Path, tool_id: str) -> list[str]: return [] warnings: list[str] = [] for arg in (*fn.args.posonlyargs, *fn.args.args, *fn.args.kwonlyargs): + if _is_runtime_injected(arg): + continue description = _annotated_description(arg.annotation) if description is None: warnings.append(f'tool "{tool_id}": parameter "{arg.arg}" is missing a description') @@ -71,6 +77,54 @@ def _check_tool(path: Path, tool_id: str) -> list[str]: return warnings +# LangChain drops these parameter names from every tool schema (``FILTERED_ARGS``). +_RUNTIME_ARG_NAMES = frozenset({"callbacks", "run_manager"}) + + +def _is_runtime_injected(arg: ast.arg) -> bool: + """True for a parameter LangChain fills in itself and never shows the model. + + Mirrors ``langchain_core`` tool-schema filtering: + + - ``callbacks`` / ``run_manager``, by name; + - ``config: RunnableConfig`` — only the bare type; ``RunnableConfig | None`` + is still shown to the model; + - ``ToolRuntime`` or ``ToolRuntime[...]``; + - ``Annotated[T, Injected...]``: ``InjectedToolArg`` and its subclasses + (``InjectedToolCallId``, LangGraph's ``InjectedState`` / ``InjectedStore``), + bare or called. + """ + if arg.arg in _RUNTIME_ARG_NAMES: + return True + annotation = arg.annotation + if annotation is None: + return False + if _type_name(annotation) in {"RunnableConfig", "ToolRuntime"}: + return True + if isinstance(annotation, ast.Subscript): + if _type_name(annotation.value) == "ToolRuntime": + return True + if _is_annotated(annotation.value) and isinstance(annotation.slice, ast.Tuple): + return any(_is_injected_marker(item) for item in annotation.slice.elts[1:]) + return False + + +def _is_injected_marker(node: ast.expr) -> bool: + """``InjectedState``, ``InjectedState("messages")``, ``tools.InjectedToolArg()``…""" + target = node.func if isinstance(node, ast.Call) else node + name = _type_name(target) + return name is not None and name.startswith("Injected") + + +def _type_name(node: ast.expr) -> str | None: + """The bare name of ``X`` or ``module.X``; None for anything else.""" + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + def _annotated_description(annotation: ast.expr | None) -> str | None: """The first string in ``Annotated[T, ...]`` metadata, or None if there is none.""" if not isinstance(annotation, ast.Subscript) or not _is_annotated(annotation.value): diff --git a/tests/cli/test_validate_command.py b/tests/cli/test_validate_command.py index 9f393b1a..e021648e 100644 --- a/tests/cli/test_validate_command.py +++ b/tests/cli/test_validate_command.py @@ -319,6 +319,22 @@ def test_described_tool_parameter_has_no_warnings(tmp_path: Path) -> None: assert "warnings:" not in CliRunner().invoke(cli, ["validate", spec]).output +def test_runtime_injected_parameter_is_not_reported(tmp_path: Path) -> None: + """LangChain fills ``config`` in at runtime; the model never sees it.""" + spec = _spec_with_tool( + tmp_path, + "from typing import Annotated\n" + "from langchain_core.runnables import RunnableConfig\n\n" + "def lookup(query: Annotated[str, 'What to look up.'], config: RunnableConfig) -> str:\n" + " return query\n", + ) + + result = validate_spec(spec) + + assert result.ok, result.errors + assert result.warnings == [] + + def test_warnings_are_shown_alongside_errors(tmp_path: Path) -> None: spec = _spec_with_tool(tmp_path, "def lookup(query: str) -> str:\n return query\n") # Break the spec so validation fails for an unrelated reason. diff --git a/tests/core/test_tool_metadata.py b/tests/core/test_tool_metadata.py index 48339803..79855f88 100644 --- a/tests/core/test_tool_metadata.py +++ b/tests/core/test_tool_metadata.py @@ -171,6 +171,101 @@ def test_star_args_and_kwargs_are_ignored(tmp_path: Path) -> None: assert warnings == [] +# -- runtime-injected parameters: LangChain fills these in; the model never sees them -- + + +def test_runnable_config_parameter_is_not_reported(tmp_path: Path) -> None: + """The case from the #156 review: the model's schema only exposes ``query``.""" + warnings = scan_one( + tmp_path, + """\ + from typing import Annotated + from langchain_core.runnables import RunnableConfig + + def get_customer(query: Annotated[str, "What to look up."], config: RunnableConfig) -> str: + return query + """, + ) + + assert warnings == [] + + +def test_optional_runnable_config_is_still_reported(tmp_path: Path) -> None: + """LangChain only hides a bare ``RunnableConfig``; an optional one reaches the model.""" + warnings = scan_one( + tmp_path, + """\ + from langchain_core.runnables import RunnableConfig + + def get_customer(config: RunnableConfig | None = None) -> str: + return "" + """, + ) + + assert warnings == ['tool "get_customer": parameter "config" is missing a description'] + + +def test_callback_parameters_are_not_reported(tmp_path: Path) -> None: + warnings = scan_one( + tmp_path, + "def get_customer(callbacks=None, run_manager=None) -> str:\n return ''\n", + ) + + assert warnings == [] + + +def test_tool_runtime_is_not_reported(tmp_path: Path) -> None: + warnings = scan_one( + tmp_path, + """\ + from langchain.tools import ToolRuntime + + def get_customer(runtime: ToolRuntime, other: ToolRuntime[dict, dict]) -> str: + return "" + """, + ) + + assert warnings == [] + + +def test_injected_annotations_are_not_reported(tmp_path: Path) -> None: + warnings = scan_one( + tmp_path, + """\ + from typing import Annotated + from langchain_core import tools + from langchain_core.tools import InjectedToolArg, InjectedToolCallId + from langgraph.prebuilt import InjectedState, InjectedStore + + def get_customer( + call_id: Annotated[str, InjectedToolCallId], + arg: Annotated[str, InjectedToolArg()], + qualified: Annotated[str, tools.InjectedToolArg], + state: Annotated[dict, InjectedState], + messages: Annotated[list, InjectedState("messages")], + store: Annotated[object, InjectedStore()], + ) -> str: + return "" + """, + ) + + assert warnings == [] + + +def test_injected_parameters_do_not_hide_undescribed_model_parameters(tmp_path: Path) -> None: + warnings = scan_one( + tmp_path, + """\ + from langchain_core.runnables import RunnableConfig + + def get_customer(query: str, config: RunnableConfig) -> str: + return query + """, + ) + + assert warnings == ['tool "get_customer": parameter "query" is missing a description'] + + def test_async_tool_is_scanned(tmp_path: Path) -> None: warnings = scan_one( tmp_path,