Skip to content

server: /reconfigure changes the context geometry without reloading the model (S2) #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joelteply wants to merge 3 commits into
base: fable/ram-pages
Choose a base branch
from
Open

server: /reconfigure changes the context geometry without reloading the model (S2) #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
971962d
server: a slot pages to named host RAM and back; checkpoints keep a b…
joelteply Oct 24, 2026
e08b11e
server: page RAM ceiling counts every live page; page files publish w…
joelteply Oct 24, 2026
77cfa0b
server: /reconfigure changes the context geometry without reloading t…
joelteply Oct 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 34 additions & 5 deletions common/arg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ const std::vector<ggml_type> kv_cache_types = {
GGML_TYPE_Q5_1,
};

static ggml_type kv_cache_type_from_str(const std::string & s) {
ggml_type common_kv_cache_type_from_str(const std::string & s) {
for (const auto & type : kv_cache_types) {
if (ggml_type_name(type) == s) {
return type;
Expand All @@ -330,6 +330,18 @@ static std::string get_all_kv_cache_types() {
return msg.str();
}

// an exact byte count: the caller passes a measured number, so no unit suffixes are guessed
static uint64_t parse_byte_count(const std::string & value, const char * flag) {
if (value.empty() || value.size() > 20 || value.find_first_not_of("0123456789") != std::string::npos) {
throw std::invalid_argument(std::string(flag) + " takes an unsigned byte count, got: " + value);
}
try {
return std::stoull(value);
} catch (const std::exception &) {
throw std::invalid_argument(std::string(flag) + " is out of range: " + value);
}
}

static bool parse_bool_value(const std::string & value) {
if (is_truthy(value)) {
return true;
Expand Down Expand Up @@ -1691,6 +1703,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.n_ctx_checkpoints = value;
}
).set_env("LLAMA_ARG_CTX_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
{"--ctx-checkpoints-bytes"}, "BYTES",
"byte budget for the context checkpoints of all slots together, in place of the per-slot count "
"(default: 0 = use --ctx-checkpoints); when a new checkpoint would exceed it, the slot holding the "
"most checkpoint bytes drops its oldest checkpoint after its first, so each slot keeps its head and its newest",
[](common_params & params, const std::string & value) {
params.ctx_checkpoints_bytes = parse_byte_count(value, "--ctx-checkpoints-bytes");
}
).set_env("LLAMA_ARG_CTX_CHECKPOINTS_BYTES").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--page-ram-bytes"}, "BYTES",
"ceiling of the host-RAM page store that /slots page_out fills (default: 0 = no ceiling); "
"a page_out that would exceed it is refused, so the caller spills a page to disk first",
[](common_params & params, const std::string & value) {
params.page_ram_bytes = parse_byte_count(value, "--page-ram-bytes");
}
).set_env("LLAMA_ARG_PAGE_RAM_BYTES").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-cms", "--checkpoint-min-step"}, "N",
string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum)", params.checkpoint_min_step),
Expand Down Expand Up @@ -2432,7 +2461,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
ggml_type_name(params.cache_type_k)
),
[](common_params & params, const std::string & value) {
params.cache_type_k = kv_cache_type_from_str(value);
params.cache_type_k = common_kv_cache_type_from_str(value);
}
).set_env("LLAMA_ARG_CACHE_TYPE_K"));
add_opt(common_arg(
Expand All @@ -2445,7 +2474,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
ggml_type_name(params.cache_type_v)
),
[](common_params & params, const std::string & value) {
params.cache_type_v = kv_cache_type_from_str(value);
params.cache_type_v = common_kv_cache_type_from_str(value);
}
).set_env("LLAMA_ARG_CACHE_TYPE_V"));
add_opt(common_arg(
Expand Down Expand Up @@ -4056,7 +4085,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
ggml_type_name(params.speculative.draft.cache_type_k)
),
[](common_params & params, const std::string & value) {
params.speculative.draft.cache_type_k = kv_cache_type_from_str(value);
params.speculative.draft.cache_type_k = common_kv_cache_type_from_str(value);
}
).set_env("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K"));
add_opt(common_arg(
Expand All @@ -4069,7 +4098,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
ggml_type_name(params.speculative.draft.cache_type_v)
),
[](common_params & params, const std::string & value) {
params.speculative.draft.cache_type_v = kv_cache_type_from_str(value);
params.speculative.draft.cache_type_v = common_kv_cache_type_from_str(value);
}
).set_env("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V"));
add_opt(common_arg(
Expand Down
3 changes: 3 additions & 0 deletions common/arg.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e
// load all backends and print the list of available (non-CPU) devices to stdout
void common_print_available_devices();

// a KV cache type by its ggml name (the --cache-type-k/-v values); throws std::runtime_error otherwise
ggml_type common_kv_cache_type_from_str(const std::string & s);

// parse input arguments from CLI into a map
bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<common_arg, std::string> & out_map);

Expand Down
121 changes: 86 additions & 35 deletions common/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1271,6 +1271,35 @@ static void common_init_sampler_from_model(
get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA), sparams.mirostat_eta, common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA);
}

// the per-context state a context takes from `params`: control vectors and the adapter set
static bool common_context_attach(llama_context * lctx, llama_model * model, common_params & params) {
if (!params.control_vectors.empty()) {
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);

const auto cvec = common_control_vector_load(params.control_vectors);
if (cvec.n_embd == -1) {
return false;
}

int err = llama_set_adapter_cvec(
lctx,
cvec.data.data(),
cvec.data.size(),
cvec.n_embd,
params.control_vector_layer_start,
params.control_vector_layer_end);
if (err) {
return false;
}
}

if (!params.lora_init_without_apply) {
common_set_adapter_lora(lctx, params.lora_adapters);
}
return true;
}

struct common_init_result::impl {
impl() = default;
~impl() = default;
Expand All @@ -1286,6 +1315,9 @@ struct common_init_result::impl {

std::vector<common_sampler_ptr> samplers;
std::vector<llama_sampler_seq_config> samplers_seq_config;

// the backend samplers (one per sequence) and a context on `model`; replaces any context held
llama_context * make_context(common_params & params, llama_context_params cparams);
};

common_init_result::common_init_result(common_params & params, bool model_only) :
Expand Down Expand Up @@ -1387,31 +1419,65 @@ common_init_result::common_init_result(common_params & params, bool model_only)
params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end());
}

llama_context * lctx = pimpl->make_context(params, cparams);
if (lctx == NULL) {
return;
}

set_process_priority(params.cpuparams.priority);

pimpl->threadpools.init(lctx, params);
}

llama_context * common_init_result::impl::make_context(common_params & params, llama_context_params cparams) {
llama_model * mdl = model.get();

// init the backend samplers as part of the context creation
pimpl->samplers.resize(cparams.n_seq_max);
pimpl->samplers_seq_config.resize(cparams.n_seq_max);
samplers.clear();
samplers.resize(cparams.n_seq_max);
samplers_seq_config.resize(cparams.n_seq_max);

for (int i = 0; i < (int) cparams.n_seq_max; ++i) {
pimpl->samplers[i].reset(common_sampler_init(model, params.sampling));
pimpl->samplers_seq_config[i] = { i, common_sampler_get(pimpl->samplers[i].get()) };
samplers[i].reset(common_sampler_init(mdl, params.sampling));
samplers_seq_config[i] = { i, common_sampler_get(samplers[i].get()) };
}

if (params.sampling.backend_sampling) {
cparams.samplers = pimpl->samplers_seq_config.data();
cparams.n_samplers = pimpl->samplers_seq_config.size();
cparams.samplers = samplers_seq_config.data();
cparams.n_samplers = samplers_seq_config.size();
}

llama_context * lctx = llama_init_from_model(model, cparams);
llama_context * lctx = llama_init_from_model(mdl, cparams);
if (lctx == NULL) {
COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str());
return;
return nullptr;
}

pimpl->context.reset(lctx);
context.reset(lctx);
return lctx;
}

set_process_priority(params.cpuparams.priority);
void common_init_result::free_context() {
pimpl->context.reset();
}

pimpl->threadpools.init(lctx, params);
llama_context * common_init_result::reinit_context(common_params & params) {
GGML_ASSERT(pimpl->model);

// free first: the old and the new context never hold memory at the same time
pimpl->context.reset();

llama_context * lctx = pimpl->make_context(params, common_context_params_to_llama(params));
if (lctx == nullptr) {
return nullptr;
}
pimpl->threadpools.attach(lctx);

if (!common_context_attach(lctx, pimpl->model.get(), params)) {
pimpl->context.reset();
return nullptr;
}
return lctx;
}

llama_model * common_init_result::model() {
Expand Down Expand Up @@ -1465,25 +1531,8 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
params.ctx_shift = false;
}

if (!params.control_vectors.empty()) {
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);

const auto cvec = common_control_vector_load(params.control_vectors);
if (cvec.n_embd == -1) {
return res;
}

int err = llama_set_adapter_cvec(
lctx,
cvec.data.data(),
cvec.data.size(),
cvec.n_embd,
params.control_vector_layer_start,
params.control_vector_layer_end);
if (err) {
return res;
}
if (!common_context_attach(lctx, model, params)) {
return res;
}

if (llama_pooling_type(lctx) == LLAMA_POOLING_TYPE_RANK) {
Expand All @@ -1510,10 +1559,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
}
}

if (!params.lora_init_without_apply) {
common_set_adapter_lora(lctx, params.lora_adapters);
}

if (params.warmup) {
COM_TRC("%s", "warming up the model with an empty run - please wait ... (--no-warmup to disable)\n");

Expand Down Expand Up @@ -1829,7 +1874,13 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
return;
}

llama_attach_threadpool(ctx, threadpool, threadpool_batch);
attach(ctx);
}

void common_threadpools::attach(llama_context * ctx) const {
if (threadpool != nullptr) {
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
}
}

//
Expand Down
15 changes: 15 additions & 0 deletions common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,8 @@ struct common_params {
bool cache_idle_slots = true; // save and clear idle slots upon starting a new task
int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
uint64_t ctx_checkpoints_bytes = 0; // checkpoint byte budget across all slots; 0 = the count above governs
uint64_t page_ram_bytes = 0; // ceiling of the named host-RAM page store; 0 = no ceiling
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.

std::string hostname = "127.0.0.1";
Expand Down Expand Up @@ -929,6 +931,15 @@ struct common_init_result {

std::vector<llama_adapter_lora_ptr> & lora();

// Continuum fork: frees the context and builds one from `params` (n_ctx, n_parallel, KV types)
// on the SAME model: its weights, mmap and adapters stay loaded. Returns nullptr when the new
// context cannot be created; the result then holds no context and the caller may call it
// again with the old geometry.
llama_context * reinit_context(common_params & params);

// Continuum fork: frees the context only; the model and adapters stay (reinit_context builds the next)
void free_context();

private:
struct impl;
std::unique_ptr<impl> pimpl;
Expand Down Expand Up @@ -965,6 +976,10 @@ struct common_threadpools {

void init(llama_context * ctx, const common_params & params);

// Continuum fork: hands the pools init() made to another context of the same process (a
// context rebuilt with a new geometry keeps its CPU threads)
void attach(llama_context * ctx) const;

private:
ggml_threadpool * threadpool = nullptr;
ggml_threadpool * threadpool_batch = nullptr;
Expand Down
53 changes: 42 additions & 11 deletions common/speculative.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2375,17 +2375,8 @@ struct common_speculative_init_result::impl {
llama_context_ptr context;
};

common_speculative_init_result::common_speculative_init_result(
common_params & params,
llama_model * model_tgt,
llama_context * ctx_tgt) :
pimpl(new impl{}) {
const bool has_draft = params.speculative.has_dft();
const bool spec_mtp = std::find(params.speculative.types.begin(),
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();

auto mparams = common_model_params_to_llama(params);
// the draft context's parameters: as many tokens per sequence as the target context, which it follows
static llama_context_params common_speculative_cparams(common_params & params, llama_context * ctx_tgt, bool spec_mtp) {
auto cparams = common_context_params_to_llama(params);

if (spec_mtp) {
Expand All @@ -2399,6 +2390,24 @@ common_speculative_init_result::common_speculative_init_result(
// the extra memory for small models is likely negligible?
cparams.n_rs_seq = 0;
cparams.ctx_other = ctx_tgt;
return cparams;
}

static bool common_speculative_is_mtp(const common_params & params) {
return std::find(params.speculative.types.begin(), params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
}

common_speculative_init_result::common_speculative_init_result(
common_params & params,
llama_model * model_tgt,
llama_context * ctx_tgt) :
pimpl(new impl{}) {
const bool has_draft = params.speculative.has_dft();
const bool spec_mtp = common_speculative_is_mtp(params);

auto mparams = common_model_params_to_llama(params);
auto cparams = common_speculative_cparams(params, ctx_tgt, spec_mtp);

std::string model_path;
if (has_draft) {
Expand Down Expand Up @@ -2435,6 +2444,28 @@ common_speculative_init_result::common_speculative_init_result(
}
}

void common_speculative_init_result::free_context() {
pimpl->context.reset();
}

llama_context * common_speculative_init_result::reinit_context(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt) {
pimpl->context.reset();

const bool spec_mtp = common_speculative_is_mtp(params);
llama_model * model = pimpl->model ? pimpl->model.get() : (spec_mtp ? model_tgt : nullptr);
if (model == nullptr) {
return nullptr;
}

llama_context * ctx_dft = llama_init_from_model(model, common_speculative_cparams(params, ctx_tgt, spec_mtp));
if (ctx_dft == nullptr) {
LOG_ERR("%s: failed to create the draft context\n", __func__);
return nullptr;
}
pimpl->context.reset(ctx_dft);
return ctx_dft;
}

common_speculative_init_result::~common_speculative_init_result() = default;

llama_model * common_speculative_init_result::model() {
Expand Down
Loading
Loading