From f577a4c9c6dd1c70081ae4c8a1209bd42bfa7316 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 18:52:28 -0600 Subject: [PATCH 1/3] Every memory reports its cells: llama_memory_cell_usage, per-slot kv positions, GET /cells Continuum's admission reads free cells from the engine, never from core arithmetic (ONE-BUDGET 3b, continuum card a156ee65): on --kv-unified the only hard constraint is the cells in use within n_ctx. - llama_memory_i::cell_usage(): each holder's {used, size}; a composite prefixes its children (iswa base/swa, hybrid attn/recr, dsa mla/lid, dsa-iswa dsa/swa, dsv4 raw/csa/hca/lid, msa base/idx). Nothing is summed across holders of different kinds. DSv4's compressed states are fixed rows per sequence, not cells, and are not reported. - C API llama_memory_cell_usage(mem, out, n_max). - /slots: kv_pos_min / kv_pos_max per slot, the engine's own positions for its sequence. - GET /cells: the holders and the slots' positions, read in one main-loop pass (the SLOT_GET task). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- include/llama.h | 15 +++++++++ src/llama-context.cpp | 17 +++++++++++ src/llama-kv-cache-dsa-iswa.cpp | 7 +++++ src/llama-kv-cache-dsa-iswa.h | 1 + src/llama-kv-cache-dsa.cpp | 7 +++++ src/llama-kv-cache-dsa.h | 1 + src/llama-kv-cache-dsv4.cpp | 11 +++++++ src/llama-kv-cache-dsv4.h | 1 + src/llama-kv-cache-iswa.cpp | 7 +++++ src/llama-kv-cache-iswa.h | 1 + src/llama-kv-cache-msa.cpp | 7 +++++ src/llama-kv-cache-msa.h | 1 + src/llama-kv-cache.cpp | 10 ++++++ src/llama-kv-cache.h | 1 + src/llama-memory-hybrid-iswa.cpp | 7 +++++ src/llama-memory-hybrid-iswa.h | 1 + src/llama-memory-hybrid.cpp | 7 +++++ src/llama-memory-hybrid.h | 1 + src/llama-memory-recurrent.cpp | 4 +++ src/llama-memory-recurrent.h | 1 + src/llama-memory.h | 23 ++++++++++++++ tests/test-batch-alloc.cpp | 1 + tools/server/server-context.cpp | 52 ++++++++++++++++++++++++++++++++ tools/server/server-context.h | 1 + tools/server/server-task.h | 3 ++ tools/server/server.cpp | 2 ++ 26 files changed, 190 insertions(+) diff --git a/include/llama.h b/include/llama.h index 277fc9d881cb..447c147c2187 100644 --- a/include/llama.h +++ b/include/llama.h @@ -897,6 +897,21 @@ extern "C" { // Check if the memory supports shifting LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + // One cell holder's occupancy (Continuum fork): cells holding at least one sequence, of the cells + // it has. A composite memory reports each holder it owns by name ("attn.base", "recr"). + struct llama_memory_cells_report { + char holder[32]; + uint32_t used; + uint32_t size; + }; + + // Write up to n_max holders' occupancy into out and return how many holders the memory has (a + // return above n_max means out was too small). Read it on the thread that decodes. + LLAMA_API int32_t llama_memory_cell_usage( + llama_memory_t mem, + struct llama_memory_cells_report * out, + int32_t n_max); + // // State / sessions // diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 654d386b63f3..578142e4fbd8 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3871,6 +3871,23 @@ llama_pos llama_memory_seq_pos_max( return mem->seq_pos_max(seq_id); } +int32_t llama_memory_cell_usage( + llama_memory_t mem, + struct llama_memory_cells_report * out, + int32_t n_max) { + if (!mem) { + return 0; + } + const auto holders = mem->cell_usage(); + for (int32_t i = 0; i < (int32_t) holders.size() && i < n_max; ++i) { + const auto & h = holders[i]; + snprintf(out[i].holder, sizeof(out[i].holder), "%s", h.holder.c_str()); + out[i].used = h.used; + out[i].size = h.size; + } + return (int32_t) holders.size(); +} + bool llama_memory_can_shift(llama_memory_t mem) { if (!mem) { return false; diff --git a/src/llama-kv-cache-dsa-iswa.cpp b/src/llama-kv-cache-dsa-iswa.cpp index 83d8e6cd3528..90842dcd3236 100644 --- a/src/llama-kv-cache-dsa-iswa.cpp +++ b/src/llama-kv-cache-dsa-iswa.cpp @@ -346,3 +346,10 @@ const char * llama_kv_cache_dsa_iswa::training_error(const llama_batch & batch, } return kv_dsa->training_error(batch, flash_attn); } + +std::vector llama_kv_cache_dsa_iswa::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "dsa", kv_dsa->cell_usage()); + llama_memory_cells_append(out, "swa", kv_swa->cell_usage()); + return out; +} diff --git a/src/llama-kv-cache-dsa-iswa.h b/src/llama-kv-cache-dsa-iswa.h index 08e270acf838..890b324e9477 100644 --- a/src/llama-kv-cache-dsa-iswa.h +++ b/src/llama-kv-cache-dsa-iswa.h @@ -59,6 +59,7 @@ class llama_kv_cache_dsa_iswa : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp index 54a9b4b440f7..163f21c45619 100644 --- a/src/llama-kv-cache-dsa.cpp +++ b/src/llama-kv-cache-dsa.cpp @@ -264,3 +264,10 @@ const llama_kv_cache_context * llama_kv_cache_dsa_context::get_lid() const { const char * llama_kv_cache_dsa::training_error(const llama_batch &, bool) const { return "DSA cache row gradients are not supported"; } + +std::vector llama_kv_cache_dsa::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "mla", kv_mla->cell_usage()); + llama_memory_cells_append(out, "lid", kv_lid->cell_usage()); + return out; +} diff --git a/src/llama-kv-cache-dsa.h b/src/llama-kv-cache-dsa.h index 0fe3b538e8b9..e6e21eb4f73b 100644 --- a/src/llama-kv-cache-dsa.h +++ b/src/llama-kv-cache-dsa.h @@ -61,6 +61,7 @@ class llama_kv_cache_dsa : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index e6b2465cffad..d9ff820f7871 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -2257,3 +2257,14 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ const char * llama_kv_cache_dsv4::training_error(const llama_batch &, bool) const { return "DSV4 cache row gradients are not supported"; } + +// The compressed states (csa_state, hca_state, lid_state) are fixed rows per sequence, not cells +// that a prompt fills; only the caches below are reported. +std::vector llama_kv_cache_dsv4::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "raw", kv_raw->cell_usage()); + llama_memory_cells_append(out, "csa", kv_csa->cell_usage()); + llama_memory_cells_append(out, "hca", kv_hca->cell_usage()); + llama_memory_cells_append(out, "lid", kv_lid->cell_usage()); + return out; +} diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index da5af159720f..293307434f13 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -134,6 +134,7 @@ class llama_kv_cache_dsv4 : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cpp index 42bd02a59782..4ac04ae50ce7 100644 --- a/src/llama-kv-cache-iswa.cpp +++ b/src/llama-kv-cache-iswa.cpp @@ -368,3 +368,10 @@ const char * llama_kv_cache_iswa::training_error(const llama_batch & batch, bool } return kv_swa->training_error(batch, flash_attn); } + +std::vector llama_kv_cache_iswa::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "base", kv_base->cell_usage()); + llama_memory_cells_append(out, "swa", kv_swa->cell_usage()); + return out; +} diff --git a/src/llama-kv-cache-iswa.h b/src/llama-kv-cache-iswa.h index 7f7364174f40..546b8325d8ad 100644 --- a/src/llama-kv-cache-iswa.h +++ b/src/llama-kv-cache-iswa.h @@ -79,6 +79,7 @@ class llama_kv_cache_iswa : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-kv-cache-msa.cpp b/src/llama-kv-cache-msa.cpp index 32dc7c91116d..d4de229a9b81 100644 --- a/src/llama-kv-cache-msa.cpp +++ b/src/llama-kv-cache-msa.cpp @@ -397,3 +397,10 @@ void llama_kv_cache_msa_context::set_input_pos_mask(ggml_tensor * dst, const lla const char * llama_kv_cache_msa::training_error(const llama_batch &, bool) const { return "MSA cache row gradients are not supported"; } + +std::vector llama_kv_cache_msa::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "base", kv_base->cell_usage()); + llama_memory_cells_append(out, "idx", kv_idx->cell_usage()); + return out; +} diff --git a/src/llama-kv-cache-msa.h b/src/llama-kv-cache-msa.h index cb93a00b034f..04fce2d66671 100644 --- a/src/llama-kv-cache-msa.h +++ b/src/llama-kv-cache-msa.h @@ -58,6 +58,7 @@ class llama_kv_cache_msa : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index b35ca7817c27..674ece212e60 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -2709,3 +2709,13 @@ const char * llama_kv_cache::training_error(const llama_batch & batch, bool flas } return nullptr; } + +std::vector llama_kv_cache::cell_usage() const { + uint32_t used = 0; + uint32_t size = 0; + for (const auto & cells : v_cells) { + used += cells.get_used(); + size += cells.size(); + } + return { { "", used, size } }; +} diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 696db3546492..7d9f5d7f7c3d 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -147,6 +147,7 @@ class llama_kv_cache : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-memory-hybrid-iswa.cpp b/src/llama-memory-hybrid-iswa.cpp index 21c39149a358..9ab3d6f70b18 100644 --- a/src/llama-memory-hybrid-iswa.cpp +++ b/src/llama-memory-hybrid-iswa.cpp @@ -290,3 +290,10 @@ const char * llama_memory_hybrid_iswa::training_error(const llama_batch & batch, } return mem_recr->training_error(batch, flash_attn); } + +std::vector llama_memory_hybrid_iswa::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "attn", mem_attn->cell_usage()); + llama_memory_cells_append(out, "recr", mem_recr->cell_usage()); + return out; +} diff --git a/src/llama-memory-hybrid-iswa.h b/src/llama-memory-hybrid-iswa.h index 693782a1c2ce..a6b9d84c1ecb 100644 --- a/src/llama-memory-hybrid-iswa.h +++ b/src/llama-memory-hybrid-iswa.h @@ -72,6 +72,7 @@ class llama_memory_hybrid_iswa : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp index dfb4e9b947f5..a80e1b8470be 100644 --- a/src/llama-memory-hybrid.cpp +++ b/src/llama-memory-hybrid.cpp @@ -284,3 +284,10 @@ const char * llama_memory_hybrid::training_error(const llama_batch & batch, bool } return mem_recr->training_error(batch, flash_attn); } + +std::vector llama_memory_hybrid::cell_usage() const { + std::vector out; + llama_memory_cells_append(out, "attn", mem_attn->cell_usage()); + llama_memory_cells_append(out, "recr", mem_recr->cell_usage()); + return out; +} diff --git a/src/llama-memory-hybrid.h b/src/llama-memory-hybrid.h index 2a92bdbaa81a..50d14d5ba8ab 100644 --- a/src/llama-memory-hybrid.h +++ b/src/llama-memory-hybrid.h @@ -72,6 +72,7 @@ class llama_memory_hybrid : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; // state write/load diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 93c3ca873c12..29a613b92d55 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -1279,3 +1279,7 @@ const char * llama_memory_recurrent::training_error(const llama_batch & batch, b } return nullptr; } + +std::vector llama_memory_recurrent::cell_usage() const { + return { { "", used, (uint32_t) cells.size() } }; +} diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index 46b65f5dfa67..33071f5fd57e 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -55,6 +55,7 @@ class llama_memory_recurrent : public llama_memory_i { llama_pos seq_pos_max(llama_seq_id seq_id) const override; std::map memory_breakdown() const override; + std::vector cell_usage() const override; bool prepare(const std::vector & ubatches); diff --git a/src/llama-memory.h b/src/llama-memory.h index bc4935ede0a5..121adbcf70d8 100644 --- a/src/llama-memory.h +++ b/src/llama-memory.h @@ -4,6 +4,8 @@ #include "llama-graph.h" #include +#include +#include #include #include @@ -70,6 +72,24 @@ using llama_memory_context_ptr = std::unique_ptr; // general concept of LLM memory // the KV cache is a type of LLM memory, but there can be other types +// One cell holder's occupancy: the cells holding at least one sequence, of the cells it has. A +// composite memory reports each holder it owns, its name prefixed ("attn.base", "recr"), so a +// reader sees which holder bounds what; nothing is summed across holders of different kinds. +struct llama_memory_cells { + std::string holder; + uint32_t used; + uint32_t size; +}; + +// Prefix every holder in `children` with `name` ("kv" -> "attn.kv" is never produced: a plain cache +// reports holder "", which becomes `name`). +inline void llama_memory_cells_append(std::vector & out, const char * name, std::vector children) { + for (auto & c : children) { + c.holder = c.holder.empty() ? std::string(name) : std::string(name) + "." + c.holder; + out.push_back(std::move(c)); + } +} + struct llama_memory_i { // this callback is used to filter out layers that should not be included in the cache using layer_filter_cb = std::function; @@ -121,6 +141,9 @@ struct llama_memory_i { virtual std::map memory_breakdown() const = 0; + // the occupancy of every cell holder this memory owns (see llama_memory_cells) + virtual std::vector cell_usage() const = 0; + // // state write/read // diff --git a/tests/test-batch-alloc.cpp b/tests/test-batch-alloc.cpp index 963fb1a06b36..6ce1179981b4 100644 --- a/tests/test-batch-alloc.cpp +++ b/tests/test-batch-alloc.cpp @@ -44,6 +44,7 @@ struct mock_memory : public llama_memory_i { } std::map memory_breakdown() const override { return {}; } + std::vector cell_usage() const override { return {}; } // holds no cells void state_write(llama_io_write_i &, llama_seq_id, llama_state_seq_flags) const override { GGML_ASSERT(false && "not implemented"); } void state_read (llama_io_read_i &, llama_seq_id, llama_state_seq_flags) override { GGML_ASSERT(false && "not implemented"); } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a6202ca09a41..632afd8cc16a 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -681,6 +681,14 @@ struct server_slot { {"is_processing", is_processing()}, }; + // The engine's own positions for this slot's sequence (Continuum fork): what its cache holds + // now, in flight or idle. -1 when the sequence is empty. + { + llama_memory_t mem = llama_get_memory(ctx_tgt); + res["kv_pos_min"] = llama_memory_seq_pos_min(mem, id); + res["kv_pos_max"] = llama_memory_seq_pos_max(mem, id); + } + const auto & ptask = task ? task : task_prev; if (ptask) { @@ -2672,6 +2680,22 @@ struct server_context_impl { res->id = task.id; res->slots_data = std::move(slots_data); res->n_idle_slots = n_idle_slots; + // Every cell holder's occupancy, read in the same main-loop pass as the slots so + // the two agree (Continuum fork, GET /cells). + { + llama_memory_t mem = llama_get_memory(ctx_tgt); + std::vector holders(8); + int32_t n = llama_memory_cell_usage(mem, holders.data(), (int32_t) holders.size()); + if (n > (int32_t) holders.size()) { + holders.resize(n); + n = llama_memory_cell_usage(mem, holders.data(), n); + } + json cells = json::array(); + for (int32_t i = 0; i < n; ++i) { + cells.push_back({ {"holder", holders[i].holder}, {"used", holders[i].used}, {"size", holders[i].size} }); + } + res->cells = std::move(cells); + } queue_results.send(std::move(res)); } break; @@ -5453,6 +5477,34 @@ void server_routes::init_routes() { return res; }; + // GET /cells (Continuum fork): every cell holder's occupancy, by the engine's own count, with the + // slots' positions read in the same pass. Admission reads free cells here, never from arithmetic. + this->get_cells = [this](const server_http_req & req) { + auto res = create_response(); + { + server_task task(SERVER_TASK_TYPE_SLOT_GET); + task.id = res->rd.get_new_id(); + res->rd.post_task(std::move(task), true); // high-priority task + } + auto result = res->rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + auto * res_task = dynamic_cast(result.get()); + GGML_ASSERT(res_task != nullptr); + json slots = json::array(); + for (const auto & slot : res_task->slots_data) { + slots.push_back({ {"id", slot.at("id")}, {"kv_pos_min", slot.at("kv_pos_min")}, {"kv_pos_max", slot.at("kv_pos_max")}, {"is_processing", slot.at("is_processing")} }); + } + res->ok({ {"holders", res_task->cells}, {"slots", slots} }); + return res; + }; + this->post_slots = [this](const server_http_req & req) { auto res = create_response(); if (params.slot_save_path.empty()) { diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 6deb3a13cc06..848495d40604 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -144,6 +144,7 @@ struct server_routes { server_http_context::handler_t get_health; server_http_context::handler_t get_metrics; server_http_context::handler_t get_slots; + server_http_context::handler_t get_cells; server_http_context::handler_t post_slots; server_http_context::handler_t get_props; server_http_context::handler_t post_props; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 3966e6a7742d..0b275141b34c 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -609,6 +609,9 @@ struct server_task_result_slots : server_task_result { // therefore, we use json to temporarily store the slot.to_json() result json slots_data = json::array(); + // every cell holder's occupancy, read in the same pass (Continuum fork, GET /cells) + json cells = json::array(); + virtual json to_json() override; }; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index f449dc00f11e..8fb272dc879c 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -221,6 +221,7 @@ int llama_server(common_params & params, int argc, char ** argv) { routes.post_lora_adapters_unload = models_routes->proxy_post; routes.post_learn = models_routes->proxy_post; routes.get_slots = models_routes->proxy_get; + routes.get_cells = models_routes->proxy_get; routes.post_slots = models_routes->proxy_post; // custom routes for router @@ -283,6 +284,7 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.post("/train/resume", ex_wrapper(routes.post_train_resume)); // Save & load slots ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); + ctx_http.get ("/cells", ex_wrapper(routes.get_cells)); ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); // resumable streaming: a child binds the local session factories, the router binds From eca3a0ca227b5b038e0ba9247b714b6e32d1420f Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 18:53:48 -0600 Subject: [PATCH 2/3] POST /v1/chat/completions/cells: the cells a chat request adds on one slot, by the server's own template, tokenizer and that slot's cache The admission half of continuum card a156ee65 (ONE-BUDGET 3b): a request is sized by the engine, never by the caller's estimate. Same body as /v1/chat/completions plus a required integer id_slot; tokenized exactly as the completion would be (oaicompat_chat_params_parse, then tokenize_mixed or the mtmd prompt); a SERVER_TASK_TYPE_CELLS_NEEDED task on the main loop compares the tokens with the slot's cached prompt (get_common_prefix). Returns {id_slot, n_tokens, n_cached, cells_needed}; the reply reserve (max_tokens) is the caller's. Read-only, so it is declined while a decode yields and offered again after, like the other slot tasks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 55 +++++++++++++++++++++++++++++++++ tools/server/server-context.h | 1 + tools/server/server-task.cpp | 9 ++++++ tools/server/server-task.h | 12 +++++++ tools/server/server.cpp | 2 ++ 5 files changed, 79 insertions(+) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 632afd8cc16a..0a80a0def477 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2697,6 +2697,21 @@ struct server_context_impl { res->cells = std::move(cells); } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_CELLS_NEEDED: + { + // read-only: the slot's cached prompt against the request's tokens + server_slot * slot = get_slot_by_id(task.slot_action.id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = slot->id; + res->n_tokens = task.tokens.size(); + res->n_cached = slot->prompt.tokens.get_common_prefix(task.tokens); queue_results.send(std::move(res)); } break; case SERVER_TASK_TYPE_SLOT_SAVE: @@ -5680,6 +5695,46 @@ void server_routes::init_routes() { TASK_RESPONSE_TYPE_OAI_CHAT); }; + // POST /v1/chat/completions/cells (Continuum fork): the cells a chat request would add to the KV + // pool on slot `id_slot` (required), by this server's own template and tokenizer and that slot's + // cache. Admission sizes a request from this, never from the caller's estimate. + this->post_chat_completions_cells = [this](const server_http_req & req) { + auto res = create_response(); + std::vector files; + json body = json::parse(req.body); + if (!body.contains("id_slot") || !body.at("id_slot").is_number_integer()) { + res->error(format_error_response("\"id_slot\" (integer) is required", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + const int id_slot = body.at("id_slot").get(); + json body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); + json prompt = body_parsed.at("prompt"); + server_task task(SERVER_TASK_TYPE_CELLS_NEEDED); + if (ctx_server.mctx != nullptr) { + if (!prompt.is_string()) { + res->error(format_error_response("for mtmd, input prompt must be a string", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + task.tokens = process_mtmd_prompt(ctx_server.mctx, prompt.get(), files, true); + } else { + task.tokens = server_tokens(tokenize_mixed(ctx_server.vocab, prompt, true, true), false); + } + task.id = res->rd.get_new_id(); + task.slot_action.id_slot = id_slot; + res->rd.post_task(std::move(task), true); // high-priority task + auto result = res->rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + res->ok(result->to_json()); + return res; + }; + this->post_chat_completions_tok = [this](const server_http_req & req) { return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_CHAT); }; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 848495d40604..76ec41689db6 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -145,6 +145,7 @@ struct server_routes { server_http_context::handler_t get_metrics; server_http_context::handler_t get_slots; server_http_context::handler_t get_cells; + server_http_context::handler_t post_chat_completions_cells; server_http_context::handler_t post_slots; server_http_context::handler_t get_props; server_http_context::handler_t post_props; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 746c3a1f98c8..7b4e5692ccce 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1539,6 +1539,15 @@ json server_task_result_error::to_json() { // // server_task_result_metrics // +json server_task_result_cells_needed::to_json() { + return json { + {"id_slot", id_slot}, + {"n_tokens", n_tokens}, + {"n_cached", n_cached}, + {"cells_needed", n_tokens - n_cached}, + }; +} + json server_task_result_slots::to_json() { return slots_data; } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 0b275141b34c..2d866581c0cd 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -53,6 +53,7 @@ enum server_task_type { SERVER_TASK_TYPE_NEXT_RESPONSE, SERVER_TASK_TYPE_METRICS, SERVER_TASK_TYPE_SLOT_GET, + SERVER_TASK_TYPE_CELLS_NEEDED, // Continuum fork: a prompt's new cells on one slot SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE, @@ -615,6 +616,17 @@ struct server_task_result_slots : server_task_result { virtual json to_json() override; }; +// Continuum fork: what a chat prompt would take of the KV pool on one slot. Its tokens by this +// server's own template and tokenizer, the prefix that slot's cache already holds, and the +// difference: the cells the request adds (its reply reserve, max_tokens, is the caller's). +struct server_task_result_cells_needed : server_task_result { + int id_slot = -1; + size_t n_tokens = 0; + size_t n_cached = 0; + + virtual json to_json() override; +}; + struct server_task_result_slot_save_load : server_task_result { std::string filename; bool is_save; // true = save, false = load diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 8fb272dc879c..b3414ac24bbe 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -214,6 +214,7 @@ int llama_server(common_params & params, int argc, char ** argv) { routes.post_detokenize = models_routes->proxy_post; routes.post_apply_template = models_routes->proxy_post; routes.post_chat_completions_tok = models_routes->proxy_post; + routes.post_chat_completions_cells = models_routes->proxy_post; routes.post_responses_tok_oai = models_routes->proxy_post; routes.get_lora_adapters = models_routes->proxy_get; routes.post_lora_adapters = models_routes->proxy_post; @@ -267,6 +268,7 @@ int llama_server(common_params & params, int argc, char ** argv) { // token counting ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); + ctx_http.post("/v1/chat/completions/cells", ex_wrapper(routes.post_chat_completions_cells)); ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting From bd43e2aec3714b270b9ce6ab896f9354a2e6bca9 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:16:25 -0600 Subject: [PATCH 3/3] Cells report per stream, and a cached prefix is clamped to what the sequence's cells hold (Cormac on #92) - llama_kv_cache::cell_usage reports one holder per stream (s0, s1, ...); a unified cache is one stream, one unnamed holder. Streams are never summed: a sequence draws only from its own stream. - cells_needed's n_cached is min(prefix with the remembered prompt, seq_pos_max + 1): a page-out can empty the sequence and keep prompt.tokens, which would over-admit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- src/llama-kv-cache.cpp | 13 +++++++------ tools/server/server-context.cpp | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 674ece212e60..bb5c0629ec81 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -2710,12 +2710,13 @@ const char * llama_kv_cache::training_error(const llama_batch & batch, bool flas return nullptr; } +// One holder per stream: a sequence draws only from its own stream's cells, so streams are never +// summed (a sum would report cells a request cannot use). A unified cache is one stream, one holder. std::vector llama_kv_cache::cell_usage() const { - uint32_t used = 0; - uint32_t size = 0; - for (const auto & cells : v_cells) { - used += cells.get_used(); - size += cells.size(); + std::vector out; + for (size_t s = 0; s < v_cells.size(); ++s) { + const auto & cells = v_cells[s]; + out.push_back({ v_cells.size() == 1 ? std::string() : "s" + std::to_string(s), cells.get_used(), cells.size() }); } - return { { "", used, size } }; + return out; } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 0a80a0def477..9dd7a34d11b6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2711,7 +2711,10 @@ struct server_context_impl { res->id = task.id; res->id_slot = slot->id; res->n_tokens = task.tokens.size(); - res->n_cached = slot->prompt.tokens.get_common_prefix(task.tokens); + // The prefix the sequence's CELLS still hold, never more than its remembered prompt: + // a page-out can empty the sequence and keep prompt.tokens (Cormac on #92). + const llama_pos held = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot->id) + 1; + res->n_cached = std::min(slot->prompt.tokens.get_common_prefix(task.tokens), (size_t) std::max(held, 0)); queue_results.send(std::move(res)); } break; case SERVER_TASK_TYPE_SLOT_SAVE: