From 1d7d6218d646e458a38c9b3d090f1a9446537bf7 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 24 Oct 2026 00:34:57 -0500 Subject: [PATCH 1/2] server: tail_tools are callable without entering the rendered head A request may carry `tail_tools` beside `tools`: the chat template renders only `tools` (so the prompt bytes, and the KV cached for them, are unchanged), while the format, parser, lazy grammar, triggers and stops are built over tools + tail_tools. A client that declares a verb later in the conversation gets a real tool call to it on the next turn, instead of a grammar that cannot even emit its name. Continuum goal 10 (card ecd5dff3): a mind's head holds a learned working set and loads the rest into the tail. test-chat: test_tail_tools (Qwen3.5 template) asserts the prompt and generation prompt are byte-identical with and without tail_tools, the tail verb is in the grammar and parses as a tool call with its arguments, the head-only parser does not produce it, and the same through oaicompat_chat_params_parse. Disabling the tail branch fails the test (checked). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/chat.cpp | 20 ++++++++++++ common/chat.h | 4 +++ tests/test-chat.cpp | 60 ++++++++++++++++++++++++++++++++++ tools/server/server-common.cpp | 3 +- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index 743ecde0a77e..009c27a2a753 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3816,6 +3816,26 @@ static common_chat_params common_chat_templates_apply_legacy(const struct common common_chat_params common_chat_templates_apply(const struct common_chat_templates * tmpls, const struct common_chat_templates_inputs & inputs) { GGML_ASSERT(tmpls != nullptr); + if (inputs.use_jinja && !inputs.tail_tools.empty()) { + // The prompt is what the head tools alone render; everything that reads the model's + // output (format, parser, grammar, triggers, stops) is built over head + tail, so a + // verb declared later in the conversation is a real tool call, not text. + common_chat_templates_inputs head = inputs; + head.tail_tools.clear(); + common_chat_templates_inputs callable = head; + for (const auto & tool : inputs.tail_tools) { + const bool in_head = std::any_of(callable.tools.begin(), callable.tools.end(), + [&](const common_chat_tool & held) { return held.name == tool.name; }); + if (!in_head) { + callable.tools.push_back(tool); + } + } + auto rendered = common_chat_templates_apply_jinja(tmpls, head); + auto params = common_chat_templates_apply_jinja(tmpls, callable); + params.prompt = std::move(rendered.prompt); + params.generation_prompt = std::move(rendered.generation_prompt); + return params; + } return inputs.use_jinja ? common_chat_templates_apply_jinja(tmpls, inputs) : common_chat_templates_apply_legacy(tmpls, inputs); } diff --git a/common/chat.h b/common/chat.h index cb39e3458f44..6e564ec16eff 100644 --- a/common/chat.h +++ b/common/chat.h @@ -255,6 +255,10 @@ struct common_chat_templates_inputs { bool use_jinja = true; // Parameters below only supported when use_jinja is true std::vector tools; + // Callable but not in the head: the parser and the lazy grammar accept these, the template + // never renders them. The caller declares each one later in the conversation, so loading a + // verb leaves the rendered prefix (and the KV cached for it) byte-identical. + std::vector tail_tools; common_chat_tool_choice tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; bool parallel_tool_calls = false; common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking" diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 41f019680677..abe97c431fa0 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -7102,6 +7102,65 @@ static void test_learning_closed_transcript() { } } +// A verb loaded into the TAIL is callable without moving the head (continuum goal 10): the +// rendered prompt is byte-identical to the head-only render, and the parser and the lazy grammar +// accept the tail verb as a real tool call. Without `tail_tools` the same output is no call to +// it at all, which is the gap this closes. +static void test_tail_tools() { + LOG_DBG("%s\n", __func__); + auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja"); + common_chat_templates_inputs head; + head.messages = { message_user }; + head.tools = { special_function_tool }; + head.enable_thinking = false; + head.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + common_chat_templates_inputs loaded = head; + loaded.tail_tools = { python_tool, special_function_tool }; + + const auto head_params = common_chat_templates_apply(tmpls.get(), head); + const auto loaded_params = common_chat_templates_apply(tmpls.get(), loaded); + assert_equals(head_params.prompt, loaded_params.prompt); + assert_equals(head_params.generation_prompt, loaded_params.generation_prompt); + assert_not_contains(loaded_params.prompt, python_tool.description); + assert_not_contains(head_params.grammar, "python"); + assert_contains(loaded_params.grammar, "python"); + + const std::string call = + "\n\n\nprint(1)\n\n\n"; + const auto parsed = make_peg_parser(tmpls.get(), loaded).parse(call, false); + assert_equals(size_t(1), parsed.tool_calls.size()); + assert_equals(std::string("python"), parsed.tool_calls[0].name); + assert_equals(renormalize_json(R"J({"code":"print(1)"})J"), renormalize_json(parsed.tool_calls[0].arguments)); + bool head_called_python = false; + try { + const auto unloaded = make_peg_parser(tmpls.get(), head).parse(call, false); + head_called_python = std::any_of(unloaded.tool_calls.begin(), unloaded.tool_calls.end(), + [](const common_chat_tool_call & c) { return c.name == "python"; }); + } catch (const std::exception &) { + head_called_python = false; // the head-only parser refuses the call outright: also no call + } + if (head_called_python) { + throw std::runtime_error("a verb absent from tools and tail_tools parsed as a call"); + } + + // The same through the server's request parsing: the body's `tail_tools` reach the parser + // and grammar and leave the prompt bytes alone. + server_chat_params opt{}; + opt.tmpls = read_templates("models/templates/Qwen3.5-4B.jinja"); + opt.use_jinja = true; + json body = {{"messages", json::array({json{{"role", "user"}, {"content", "run it"}}})}, + {"tools", common_chat_tools_to_json_oaicompat({ special_function_tool })}, + {"chat_template_kwargs", {{"enable_thinking", false}}}}; + std::vector files; + const json served = oaicompat_chat_params_parse(body, opt, files); + json with_tail = body; + with_tail["tail_tools"] = common_chat_tools_to_json_oaicompat({ python_tool }); + const json served_tail = oaicompat_chat_params_parse(with_tail, opt, files); + assert_equals(served.at("prompt").get(), served_tail.at("prompt").get()); + assert_contains(served_tail.at("grammar").get(), "python"); + assert_equals(true, served_tail.at("parse_tool_calls").get()); +} + static void test_reasoning_budget_tokens_per_request() { LOG_DBG("%s\n", __func__); // Use Qwen3 template which has ... reasoning markers. @@ -7522,6 +7581,7 @@ int main(int argc, char ** argv) { test_template_generation_prompt(); test_reasoning_effort_caps(); test_learning_closed_transcript(); + test_tail_tools(); test_reasoning_budget_tokens_per_request(); test_reasoning_budget_message_per_request(); test_template_output_peg_parsers(detailed_debug); diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 3269f86b2199..ef717d557e46 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1279,6 +1279,7 @@ json oaicompat_chat_params_parse( common_chat_templates_inputs inputs; inputs.messages = common_chat_msgs_parse_oaicompat(messages); inputs.tools = common_chat_tools_parse_oaicompat(tools); + inputs.tail_tools = common_chat_tools_parse_oaicompat(json_value(body, "tail_tools", json())); inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice); inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump(); inputs.grammar = grammar; @@ -1310,7 +1311,7 @@ json oaicompat_chat_params_parse( inputs.reasoning_format = common_reasoning_format_from_name(body.at("reasoning_format").get()); } inputs.enable_thinking = opt.enable_thinking; - if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + if ((!inputs.tools.empty() || !inputs.tail_tools.empty()) && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { if (body.contains("grammar")) { throw std::invalid_argument("Cannot use custom grammar constraints with tools."); } From ca53e26b3995f42acb6de1703d8a0eef9625794f Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 26 Oct 2026 14:36:12 -0500 Subject: [PATCH 2/2] server: a tail-only request obeys every tools boundary; tail coverage for Qwen2.5, empty head, tool_choice none and streaming Codex's review of #71: has_tools read only `tools`, so a request whose verbs were all in `tail_tools` passed the requires-Jinja rejection on a non-Jinja server (the legacy renderer then dropped the tail while parse_tool_calls was set) and skipped the tools + stream format guard. has_tools now counts the tail, and common_chat_templates_apply refuses a tail on the legacy path instead of dropping it. test_tail_tools now covers, per template family (Qwen2.5 JSON , Qwen3.5 XML ) and for a populated and an empty head: prompt bytes equal the head-only render, the tail verb's call parses through streamed deltas with the lazy grammar accepting it, and with tool_choice "none" the tail adds nothing to the grammar and the verb does not parse as a call. Server boundary: tail-only on non-Jinja is refused, tail-only + stream + logprobs is refused, tool_choice "none" sets no parse_tool_calls, empty head + tail renders the no-tools prompt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/chat.cpp | 6 +- tests/test-chat.cpp | 114 +++++++++++++++++++++++++++++++++ tools/server/server-common.cpp | 8 ++- 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 009c27a2a753..78d0f62a7d20 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3816,7 +3816,11 @@ static common_chat_params common_chat_templates_apply_legacy(const struct common common_chat_params common_chat_templates_apply(const struct common_chat_templates * tmpls, const struct common_chat_templates_inputs & inputs) { GGML_ASSERT(tmpls != nullptr); - if (inputs.use_jinja && !inputs.tail_tools.empty()) { + if (!inputs.use_jinja && !inputs.tail_tools.empty()) { + // The legacy renderer has no tools at all: a tail it cannot make callable is refused, never dropped. + throw std::invalid_argument("tail_tools require the jinja template path"); + } + if (!inputs.tail_tools.empty()) { // The prompt is what the head tools alone render; everything that reads the model's // output (format, parser, grammar, triggers, stops) is built over head + tail, so a // verb declared later in the conversation is a real tool call, not text. diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index abe97c431fa0..e9aaadd0d210 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -7159,6 +7159,120 @@ static void test_tail_tools() { assert_equals(served.at("prompt").get(), served_tail.at("prompt").get()); assert_contains(served_tail.at("grammar").get(), "python"); assert_equals(true, served_tail.at("parse_tool_calls").get()); + + // Each template family a node serves, with a populated and an EMPTY head: the prompt bytes + // are the head's alone; a tail verb's call streams (every prefix parses, the deltas add up to + // the final call, the lazy grammar accepts it); tool_choice "none" leaves it uncallable. + const auto calls_python = [](common_chat_templates * t, const common_chat_templates_inputs & in, + const std::string & text) { + try { + const auto msg = make_peg_parser(t, in).parse(text, false); + return std::any_of(msg.tool_calls.begin(), msg.tool_calls.end(), + [](const common_chat_tool_call & c) { return c.name == "python"; }); + } catch (const std::exception &) { + return false; // refused outright: also no call + } + }; + struct family { + const char * path; + std::string call; + }; + const family families[] = { + // Qwen2.5 (enrico serves qwen2.5-coder-1.5b): JSON inside . + { "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja", + "\n{\"name\": \"python\", \"arguments\": {\"code\": \"print(1)\"}}\n" }, + // Qwen3.5 / Qwen3-coder: XML parameters inside . + { "models/templates/Qwen3.5-4B.jinja", call }, + }; + for (const auto & fam : families) { + for (const auto & head_tools : { std::vector{ special_function_tool }, + std::vector{} }) { + LOG_DBG("%s: %s, head of %zu\n", __func__, fam.path, head_tools.size()); + auto t = read_templates(fam.path); + common_chat_templates_inputs h; + h.messages = { message_user }; + h.tools = head_tools; + h.enable_thinking = false; + h.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + common_chat_templates_inputs l = h; + l.tail_tools = { python_tool }; + + const auto hp = common_chat_templates_apply(t.get(), h); + const auto lp = common_chat_templates_apply(t.get(), l); + assert_equals(hp.prompt, lp.prompt); + assert_equals(hp.generation_prompt, lp.generation_prompt); + assert_not_contains(lp.prompt, python_tool.description); + assert_contains(lp.grammar, "python"); + assert_equals(true, calls_python(t.get(), l, fam.call)); + assert_equals(false, calls_python(t.get(), h, fam.call)); + + test_peg_parser(t.get(), [&](peg_test_case & tc) { + tc.params = l; + tc.input = fam.call; + tc.expect = message_with_tool_calls("python", R"J({"code": "print(1)"})J"); + }, false); + + common_chat_templates_inputs hn = h; + hn.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE; + common_chat_templates_inputs ln = l; + ln.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE; + const auto hnp = common_chat_templates_apply(t.get(), hn); + const auto lnp = common_chat_templates_apply(t.get(), ln); + assert_equals(hnp.prompt, lnp.prompt); + assert_equals(hnp.grammar, lnp.grammar); + assert_not_contains(lnp.grammar, "python"); + assert_equals(false, calls_python(t.get(), ln, fam.call)); + } + } + + // The request boundary treats a tail exactly like a head: tail-only on a non-Jinja server is + // refused (the legacy renderer cannot supply a callable verb), tail-only obeys the tools + + // stream format guard, tool_choice "none" asks for no tool-call parsing, and an empty head + // with a tail renders the no-tools prompt yet parses tool calls. + json tail_only = {{"messages", json::array({json{{"role", "user"}, {"content", "run it"}}})}, + {"tail_tools", common_chat_tools_to_json_oaicompat({ python_tool })}}; + const auto refuses = [&](json req, const server_chat_params & o) { + try { + oaicompat_chat_params_parse(req, o, files); + } catch (const std::exception &) { + return true; + } + return false; + }; + server_chat_params legacy{}; + legacy.tmpls = read_templates("models/templates/Qwen-Qwen2.5-7B-Instruct.jinja"); + legacy.use_jinja = false; + assert_equals(true, refuses(tail_only, legacy)); + bool legacy_dropped_tail = true; + try { + common_chat_templates_inputs legacy_in; + legacy_in.messages = { message_user }; + legacy_in.use_jinja = false; + legacy_in.tail_tools = { python_tool }; + common_chat_templates_apply(legacy.tmpls.get(), legacy_in); + } catch (const std::invalid_argument &) { + legacy_dropped_tail = false; // the library refuses it too, below the server + } + assert_equals(false, legacy_dropped_tail); + + server_chat_params q25{}; + q25.tmpls = read_templates("models/templates/Qwen-Qwen2.5-7B-Instruct.jinja"); + q25.use_jinja = true; + json streamed = tail_only; + streamed["stream"] = true; + streamed["logprobs"] = true; + assert_equals(true, refuses(streamed, q25)); + + json none = tail_only; + none["tool_choice"] = "none"; + const json served_none = oaicompat_chat_params_parse(none, q25, files); + assert_equals(false, served_none.value("parse_tool_calls", false)); + + json no_tools = {{"messages", tail_only.at("messages")}}; + const json served_no_tools = oaicompat_chat_params_parse(no_tools, q25, files); + const json served_tail_only = oaicompat_chat_params_parse(tail_only, q25, files); + assert_equals(served_no_tools.at("prompt").get(), served_tail_only.at("prompt").get()); + assert_equals(true, served_tail_only.at("parse_tool_calls").get()); } static void test_reasoning_budget_tokens_per_request() { diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index ef717d557e46..fb1c12019685 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1152,13 +1152,15 @@ json oaicompat_chat_params_parse( json llama_params; auto tools = json_value(body, "tools", json()); - auto has_tools = tools.is_array() && !tools.empty(); + auto tail_tools = json_value(body, "tail_tools", json()); + // A tail verb is callable like a head one, so every tools boundary below holds for it too. + auto has_tools = (tools.is_array() && !tools.empty()) || (tail_tools.is_array() && !tail_tools.empty()); auto stream = json_value(body, "stream", false); auto tool_choice = json_value(body, "tool_choice", std::string("auto")); if (!opt.use_jinja) { if (has_tools) { - throw std::runtime_error("tools param requires --jinja flag"); + throw std::runtime_error("tools and tail_tools params require --jinja flag"); } if (tool_choice != "auto") { throw std::runtime_error("tool_choice param requires --jinja flag"); @@ -1279,7 +1281,7 @@ json oaicompat_chat_params_parse( common_chat_templates_inputs inputs; inputs.messages = common_chat_msgs_parse_oaicompat(messages); inputs.tools = common_chat_tools_parse_oaicompat(tools); - inputs.tail_tools = common_chat_tools_parse_oaicompat(json_value(body, "tail_tools", json())); + inputs.tail_tools = common_chat_tools_parse_oaicompat(tail_tools); inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice); inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump(); inputs.grammar = grammar;