From 2f624ce53f2013d10ba15e54a6d867670020099a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 28 Oct 2026 16:53:33 -0500 Subject: [PATCH 1/5] server: create dormant dream adapters and publish immutable snapshots --- tools/server/server-adapter-file.h | 37 +++++++++ tools/server/server-context.cpp | 95 ++++++++++++++++++++++ tools/server/server-context.h | 2 + tools/server/server-task.h | 3 + tools/server/server.cpp | 4 + tools/server/tests/dream-adapter-routes.py | 67 +++++++++++++++ 6 files changed, 208 insertions(+) create mode 100644 tools/server/server-adapter-file.h create mode 100644 tools/server/tests/dream-adapter-routes.py diff --git a/tools/server/server-adapter-file.h b/tools/server/server-adapter-file.h new file mode 100644 index 000000000000..2b46b1e968ae --- /dev/null +++ b/tools/server/server-adapter-file.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +// Publish a complete adapter without replacing an existing snapshot. The private +// directory reserves this destination across concurrent fresh/save requests. +// A crash leaves a named reservation for diagnosis, never a partial public GGUF. +inline std::string server_publish_adapter(const std::string & path, + const std::function & write) { + namespace fs = std::filesystem; + const fs::path staging = path + ".pending"; + std::error_code ec; + if (!fs::create_directory(staging, ec)) { + return "adapter publication already reserved or unavailable: " + staging.string(); + } + struct cleanup { + fs::path dir; + ~cleanup() { + std::error_code ignored; + fs::remove(dir / "adapter.gguf", ignored); + fs::remove(dir, ignored); + } + } guard{staging}; + const auto temporary = staging / "adapter.gguf"; + if (!write(temporary.string())) { + return "adapter serialization failed"; + } + // Unlike rename on POSIX, hard-link creation cannot replace a destination. + // Both paths live on the train-dir filesystem. Unsupported filesystems refuse. + fs::create_hard_link(temporary, path, ec); + if (ec) { + return "immutable adapter publication refused: " + ec.message(); + } + return {}; +} diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 87672928aeb3..4155e530d71f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2,6 +2,7 @@ #include "server-context.h" #include "../../src/llama-ext.h" #include "server-train.h" +#include "server-adapter-file.h" #include "server-chat.h" #include "server-common.h" #include "server-http.h" @@ -31,6 +32,7 @@ #include #include #include +#include // Continuum fork: an adapter file's revision (size, mtime in ns since the epoch; (0, 0) when it // cannot be read), stored in a saved slot's `.lora` sidecar so a restore refuses a page computed @@ -3102,6 +3104,29 @@ struct server_context_impl { res->id = task.id; queue_results.send(std::move(res)); } break; + case SERVER_TASK_TYPE_SAVE_LORA: + { + const auto & loras = params_base.lora_adapters; + const int32_t id = task.save_lora; + if (id < 0 || (size_t) id >= loras.size() || loras[id].ptr == nullptr) { + send_error(task, "no loaded LoRA adapter with id " + std::to_string(id), ERROR_TYPE_INVALID_REQUEST); + break; + } + // Same queue as learn and unload: the snapshot cannot observe a + // partial optimizer update or an adapter freed by another request. + const auto error = server_publish_adapter(task.load_lora_path, [&](const std::string & temporary) { + return llama_adapter_lora_save(loras[id].ptr, temporary.c_str()) == 0; + }); + if (!error.empty()) { + send_error(task, error, ERROR_TYPE_INVALID_REQUEST); + break; + } + auto res = std::make_unique(); + res->id = task.id; + res->lora_id = id; + res->path = task.load_lora_path; + queue_results.send(std::move(res)); + } break; case SERVER_TASK_TYPE_LOAD_LORA: { // Read from disk off this loop by the HTTP handler; appending it here, @@ -5968,6 +5993,76 @@ void server_routes::init_routes() { return res; }; + this->post_lora_adapters_save = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); + const auto path = confine_out(params.train_dir, json_value(body, "name", std::string())); + if (params.train_dir.empty() || path.empty() || !body.contains("id") || !body.at("id").is_number_integer() || + body.at("id").get() < 0 || body.at("id").get() > INT32_MAX) { + res->error(format_error_response("save requires an integer id and a bare .gguf name under --train-dir", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + auto & rd = res->rd; + server_task task(SERVER_TASK_TYPE_SAVE_LORA); + task.id = rd.get_new_id(); + task.save_lora = body.at("id").get(); + task.load_lora_path = path; + rd.post_task(std::move(task)); + auto result = rd.next(req.should_stop); + if (!result) { + return res; // queued snapshot remains owned by the loop after disconnect + } + if (result->is_error()) { + res->error(result->to_json()); + } else { + res->ok(result->to_json()); + } + return res; + }; + + this->post_lora_adapters_fresh = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); + const auto path = confine_out(params.train_dir, json_value(body, "name", std::string())); + if (params.train_dir.empty() || path.empty() || ctx_server.model_tgt == nullptr) { + res->error(format_error_response("fresh requires a loaded model and a bare .gguf name under --train-dir", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + const double rank = body.value("rank", 8.0); + const double alpha = body.value("alpha", 16.0); + const double top = body.value("top_layers", 0.0); + const double seed = body.value("seed", 42.0); + if (!std::isfinite(rank) || rank < 1 || rank > 256 || std::floor(rank) != rank || + !std::isfinite(alpha) || alpha <= 0 || alpha > std::numeric_limits::max() || + !std::isfinite(top) || top < 0 || top > llama_model_n_layer(ctx_server.model_tgt) || std::floor(top) != top || + !std::isfinite(seed) || seed < 0 || seed > UINT32_MAX || std::floor(seed) != seed) { + res->error(format_error_response("invalid rank, alpha, top_layers or seed", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + std::vector targets; + std::stringstream parts(body.value("targets", std::string("attn_q,attn_v"))); + std::string target; + while (std::getline(parts, target, ',')) { + if (!target.empty()) { + targets.push_back(target); + } + } + if (targets.empty()) { + res->error(format_error_response("targets must name at least one module", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + const auto error = server_publish_adapter(path, [&](const std::string & temporary) { + return common_lora_write_fresh(ctx_server.model_tgt, temporary, (int32_t) rank, (float) alpha, + targets, (uint32_t) seed, (int32_t) top); + }); + if (!error.empty()) { + res->error(format_error_response(error, ERROR_TYPE_INVALID_REQUEST)); + return res; + } + // Reuse load's ownership transfer and dormant scale-zero installation. + return post_lora_adapters_load(req); + }; + this->post_lora_adapters_load = [this](const server_http_req & req) { auto res = create_response(); const json body = json::parse(req.body); diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 6deb3a13cc06..0341bb45f52c 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -168,6 +168,8 @@ struct server_routes { server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; server_http_context::handler_t post_lora_adapters_load; + server_http_context::handler_t post_lora_adapters_fresh; + server_http_context::handler_t post_lora_adapters_save; server_http_context::handler_t post_lora_adapters_unload; server_http_context::handler_t post_learn; server_http_context::handler_t get_train; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 3966e6a7742d..838ecd1e85b9 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -59,6 +59,7 @@ enum server_task_type { SERVER_TASK_TYPE_GET_LORA, SERVER_TASK_TYPE_SET_LORA, SERVER_TASK_TYPE_LOAD_LORA, + SERVER_TASK_TYPE_SAVE_LORA, SERVER_TASK_TYPE_UNLOAD_LORA, SERVER_TASK_TYPE_LEARN, }; @@ -260,6 +261,8 @@ struct server_task { // server loop; the loop only appends it (dormant, scale 0) to the adapter list llama_adapter_lora * load_lora = nullptr; std::string load_lora_path; + // SAVE_LORA runs between decode/learn steps; path is confined by the route. + int32_t save_lora = -1; // used by SERVER_TASK_TYPE_UNLOAD_LORA: the adapter id to retire int32_t unload_lora = -1; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index f449dc00f11e..2984260bed6c 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -218,6 +218,8 @@ int llama_server(common_params & params, int argc, char ** argv) { routes.get_lora_adapters = models_routes->proxy_get; routes.post_lora_adapters = models_routes->proxy_post; routes.post_lora_adapters_load = models_routes->proxy_post; + routes.post_lora_adapters_fresh = models_routes->proxy_post; + routes.post_lora_adapters_save = models_routes->proxy_post; routes.post_lora_adapters_unload = models_routes->proxy_post; routes.post_learn = models_routes->proxy_post; routes.get_slots = models_routes->proxy_get; @@ -273,6 +275,8 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters)); ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters)); ctx_http.post("/lora-adapters/load", ex_wrapper(routes.post_lora_adapters_load)); + ctx_http.post("/lora-adapters/fresh", ex_wrapper(routes.post_lora_adapters_fresh)); + ctx_http.post("/lora-adapters/save", ex_wrapper(routes.post_lora_adapters_save)); ctx_http.post("/lora-adapters/unload", ex_wrapper(routes.post_lora_adapters_unload)); ctx_http.post("/learn", ex_wrapper(routes.post_learn)); ctx_http.get ("/train", ex_wrapper(routes.get_train)); diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py new file mode 100644 index 000000000000..26a3cf71f28a --- /dev/null +++ b/tools/server/tests/dream-adapter-routes.py @@ -0,0 +1,67 @@ +"""D1 live gate. Run against an idle test server with --train-dir, never production. + +python dream-adapter-routes.py http://127.0.0.1:8080 +Checks dormant and enabled zero-B parity, snapshot reload parity, immutable names, +bad ids, path confinement, and competing publication of the same name. +""" +import concurrent.futures +import json +import sys +import urllib.error +import urllib.request +import uuid + +url = sys.argv[1].rstrip("/") +prefix = "d1-" + uuid.uuid4().hex + + +def post(route, body): + req = urllib.request.Request(url + route, json.dumps(body).encode(), + {"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=120) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.load(error) + + +def ok(route, body): + status, result = post(route, body) + assert status == 200, (route, status, result) + return result + + +def complete(adapter=None): + body = {"prompt": "The first four prime numbers are", "n_predict": 24, + "temperature": 0, "seed": 17, "cache_prompt": False, + "lora": [] if adapter is None else [{"id": adapter, "scale": 1.0}]} + return ok("/completion", body)["content"] + + +baseline = complete() +fresh_name = prefix + "-fresh.gguf" +fresh = ok("/lora-adapters/fresh", {"name": fresh_name, "rank": 8, + "alpha": 16, "targets": "attn_q,attn_v", "top_layers": 1, "seed": 42}) +assert complete() == baseline, "dormant shadow changed base output" +assert complete(fresh["id"]) == baseline, "zero-B shadow changed enabled output" +learned = ok("/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, + "n_predict": 1, "temperature": 0, + "learn": {"reply": "Topeka is the capital of Kansas.", + "lora_id": fresh["id"], "lr": 0.001}}) +assert learned["learn"]["success"], learned +trained_text = complete(fresh["id"]) +snapshot_name = prefix + "-snapshot.gguf" +snapshot = ok("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name}) +loaded = ok("/lora-adapters/load", {"name": snapshot_name}) +assert complete(loaded["id"]) == trained_text, "trained snapshot reload changed output" +assert post("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})[0] >= 400 +assert post("/lora-adapters/fresh", {"name": fresh_name})[0] >= 400 +for bad_id in (-1, 2**32, 2**63): + assert post("/lora-adapters/save", {"id": bad_id, "name": prefix + "-bad.gguf"})[0] >= 400 +for route in ("/lora-adapters/fresh", "/lora-adapters/save"): + assert post(route, {"id": fresh["id"], "name": "../escape.gguf"})[0] >= 400 +with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: post("/lora-adapters/save", + {"id": fresh["id"], "name": prefix + "-race.gguf"})[0], range(2))) +assert sorted(status == 200 for status in results) == [False, True], results +print(json.dumps({"gate": "D1", "fresh": fresh, "snapshot": snapshot, "passed": True})) From 956e1015697733b9ded4b5d1d21f1e393eca933b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 28 Oct 2026 17:00:47 -0500 Subject: [PATCH 2/5] server: declare common response type for fresh adapter handler --- tools/server/server-context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 4155e530d71f..15717a2c45df 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6020,7 +6020,7 @@ void server_routes::init_routes() { return res; }; - this->post_lora_adapters_fresh = [this](const server_http_req & req) { + this->post_lora_adapters_fresh = [this](const server_http_req & req) -> server_http_res_ptr { auto res = create_response(); const json body = json::parse(req.body); const auto path = confine_out(params.train_dir, json_value(body, "name", std::string())); From 3349756132f11f820c55331980d3ea8cb0879275 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 28 Oct 2026 17:11:03 -0500 Subject: [PATCH 3/5] ci: exercise dream adapter creation and trained snapshot reload --- .github/workflows/server.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 93370e51c64f..75e55a9760a4 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -96,6 +96,28 @@ jobs: - name: Prefill scheduling test run: ctest --test-dir build -C Release -R ^test-server-prefill$ --output-on-failure --no-tests=error + - name: Dream adapter creation and snapshot parity + timeout-minutes: 5 + shell: bash + run: | + set -euo pipefail + fixture=$(mktemp -d) + curl --fail --location --retry 3 --max-time 90 \ + https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf \ + -o "$fixture/model.gguf" + mkdir "$fixture/train" + build/bin/llama-server -m "$fixture/model.gguf" --port 58199 \ + -np 1 -c 512 -b 512 -ub 512 -ngl 0 -t 2 --train-dir "$fixture/train" \ + > "$fixture/server.log" 2>&1 & + server_pid=$! + trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" || true; cat "$fixture/server.log"' EXIT + for attempt in $(seq 1 60); do + if curl --fail --silent http://127.0.0.1:58199/health > /dev/null; then break; fi + kill -0 "$server_pid" + sleep 1 + done + python3 tools/server/tests/dream-adapter-routes.py http://127.0.0.1:58199 + - name: Python setup id: setup_python uses: actions/setup-python@v6 From b7478978d13efcf19a7b65fe77fab10ecc07fe28 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 28 Oct 2026 17:20:42 -0500 Subject: [PATCH 4/5] ci: use standard CPU weight layout for dream backward test --- .github/workflows/server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 75e55a9760a4..149b48f5be38 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -107,7 +107,7 @@ jobs: -o "$fixture/model.gguf" mkdir "$fixture/train" build/bin/llama-server -m "$fixture/model.gguf" --port 58199 \ - -np 1 -c 512 -b 512 -ub 512 -ngl 0 -t 2 --train-dir "$fixture/train" \ + -np 1 -c 512 -b 512 -ub 512 -ngl 0 --no-repack -t 2 --train-dir "$fixture/train" \ > "$fixture/server.log" 2>&1 & server_pid=$! trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" || true; cat "$fixture/server.log"' EXIT From 1d7333325307aa8c87be6784b25672f38b373565 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 28 Oct 2026 18:11:36 -0500 Subject: [PATCH 5/5] test: require a learned adapter to differ from its initial snapshot --- .github/workflows/server.yml | 2 +- tools/server/tests/dream-adapter-routes.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 149b48f5be38..764b4f8a0acf 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -116,7 +116,7 @@ jobs: kill -0 "$server_pid" sleep 1 done - python3 tools/server/tests/dream-adapter-routes.py http://127.0.0.1:58199 + python3 tools/server/tests/dream-adapter-routes.py http://127.0.0.1:58199 "$fixture/train" - name: Python setup id: setup_python diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index 26a3cf71f28a..172a8d5dd514 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -1,17 +1,20 @@ """D1 live gate. Run against an idle test server with --train-dir, never production. -python dream-adapter-routes.py http://127.0.0.1:8080 -Checks dormant and enabled zero-B parity, snapshot reload parity, immutable names, +python dream-adapter-routes.py http://127.0.0.1:8080 /path/to/server/train-dir +Checks dormant and enabled zero-B parity, changed weights, snapshot reload parity, immutable names, bad ids, path confinement, and competing publication of the same name. """ import concurrent.futures +import hashlib import json +from pathlib import Path import sys import urllib.error import urllib.request import uuid url = sys.argv[1].rstrip("/") +train_dir = Path(sys.argv[2]) prefix = "d1-" + uuid.uuid4().hex @@ -44,6 +47,12 @@ def complete(adapter=None): "alpha": 16, "targets": "attn_q,attn_v", "top_layers": 1, "seed": 42}) assert complete() == baseline, "dormant shadow changed base output" assert complete(fresh["id"]) == baseline, "zero-B shadow changed enabled output" +before_name = prefix + "-before.gguf" +ok("/lora-adapters/save", {"id": fresh["id"], "name": before_name}) +before = hashlib.sha256((train_dir / before_name).read_bytes()).hexdigest() +repeat_name = prefix + "-repeat.gguf" +ok("/lora-adapters/save", {"id": fresh["id"], "name": repeat_name}) +assert hashlib.sha256((train_dir / repeat_name).read_bytes()).hexdigest() == before, "save bytes are unstable without learning" learned = ok("/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, "n_predict": 1, "temperature": 0, "learn": {"reply": "Topeka is the capital of Kansas.", @@ -52,6 +61,8 @@ def complete(adapter=None): trained_text = complete(fresh["id"]) snapshot_name = prefix + "-snapshot.gguf" snapshot = ok("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name}) +after = hashlib.sha256((train_dir / snapshot_name).read_bytes()).hexdigest() +assert after != before, "successful learn did not change saved adapter weights" loaded = ok("/lora-adapters/load", {"name": snapshot_name}) assert complete(loaded["id"]) == trained_text, "trained snapshot reload changed output" assert post("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})[0] >= 400 @@ -64,4 +75,5 @@ def complete(adapter=None): results = list(pool.map(lambda _: post("/lora-adapters/save", {"id": fresh["id"], "name": prefix + "-race.gguf"})[0], range(2))) assert sorted(status == 200 for status in results) == [False, True], results -print(json.dumps({"gate": "D1", "fresh": fresh, "snapshot": snapshot, "passed": True})) +print(json.dumps({"gate": "D1", "fresh": fresh, "snapshot": snapshot, + "before_sha256": before, "after_sha256": after, "passed": True}))