server: create dormant dream adapters and save immutable snapshots

.github/workflows/server.yml

@@ -96,6 +96,28 @@ jobs:
       - name: Prefill scheduling test
         run: ctest --test-dir build -C Release -R ^test-server-prefill$ --output-on-failure --no-tests=error
 
+      - name: Dream adapter creation and snapshot parity
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          set -euo pipefail
+          fixture=$(mktemp -d)
+          curl --fail --location --retry 3 --max-time 90 \
+            https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf \
+            -o "$fixture/model.gguf"
+          mkdir "$fixture/train"
+          build/bin/llama-server -m "$fixture/model.gguf" --port 58199 \
+            -np 1 -c 512 -b 512 -ub 512 -ngl 0 --no-repack -t 2 --train-dir "$fixture/train" \
+            > "$fixture/server.log" 2>&1 &
+          server_pid=$!
+          trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" || true; cat "$fixture/server.log"' EXIT
+          for attempt in $(seq 1 60); do
+            if curl --fail --silent http://127.0.0.1:58199/health > /dev/null; then break; fi
+            kill -0 "$server_pid"
+            sleep 1
+          done
+          python3 tools/server/tests/dream-adapter-routes.py http://127.0.0.1:58199 "$fixture/train"
+
       - name: Python setup
         id: setup_python
         uses: actions/setup-python@v6

tools/server/server-adapter-file.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <filesystem>
+#include <functional>
+#include <string>
+
+// Publish a complete adapter without replacing an existing snapshot. The private
+// directory reserves this destination across concurrent fresh/save requests.
+// A crash leaves a named reservation for diagnosis, never a partial public GGUF.
+inline std::string server_publish_adapter(const std::string & path,
+                                         const std::function<bool(const std::string &)> & write) {
+    namespace fs = std::filesystem;
+    const fs::path staging = path + ".pending";
+    std::error_code ec;
+    if (!fs::create_directory(staging, ec)) {
+        return "adapter publication already reserved or unavailable: " + staging.string();
+    }
+    struct cleanup {
+        fs::path dir;
+        ~cleanup() {
+            std::error_code ignored;
+            fs::remove(dir / "adapter.gguf", ignored);
+            fs::remove(dir, ignored);
+        }
+    } guard{staging};
+    const auto temporary = staging / "adapter.gguf";
+    if (!write(temporary.string())) {
+        return "adapter serialization failed";
+    }
+    // Unlike rename on POSIX, hard-link creation cannot replace a destination.
+    // Both paths live on the train-dir filesystem. Unsupported filesystems refuse.
+    fs::create_hard_link(temporary, path, ec);
+    if (ec) {
+        return "immutable adapter publication refused: " + ec.message();
+    }
+    return {};
+}

tools/server/server-context.cpp

@@ -2,6 +2,7 @@
 #include "server-context.h"
 #include "../../src/llama-ext.h"
 #include "server-train.h"
+#include "server-adapter-file.h"
 #include "server-chat.h"
 #include "server-common.h"
 #include "server-http.h"
@@ -31,6 +32,7 @@
 #include <filesystem>
 #include <utility>
 #include <fstream>
+#include <sstream>
 
 // Continuum fork: an adapter file's revision (size, mtime in ns since the epoch; (0, 0) when it
 // cannot be read), stored in a saved slot's `.lora` sidecar so a restore refuses a page computed
@@ -3102,6 +3104,29 @@ struct server_context_impl {
                     res->id = task.id;
                     queue_results.send(std::move(res));
                 } break;
+            case SERVER_TASK_TYPE_SAVE_LORA:
+                {
+                    const auto & loras = params_base.lora_adapters;
+                    const int32_t id = task.save_lora;
+                    if (id < 0 || (size_t) id >= loras.size() || loras[id].ptr == nullptr) {
+                        send_error(task, "no loaded LoRA adapter with id " + std::to_string(id), ERROR_TYPE_INVALID_REQUEST);
+                        break;
+                    }
+                    // Same queue as learn and unload: the snapshot cannot observe a
+                    // partial optimizer update or an adapter freed by another request.
+                    const auto error = server_publish_adapter(task.load_lora_path, [&](const std::string & temporary) {
+                        return llama_adapter_lora_save(loras[id].ptr, temporary.c_str()) == 0;
+                    });
+                    if (!error.empty()) {
+                        send_error(task, error, ERROR_TYPE_INVALID_REQUEST);
+                        break;
+                    }
+                    auto res = std::make_unique<server_task_result_lora_change>();
+                    res->id = task.id;
+                    res->lora_id = id;
+                    res->path = task.load_lora_path;
+                    queue_results.send(std::move(res));
+                } break;
             case SERVER_TASK_TYPE_LOAD_LORA:
                 {
                     // Read from disk off this loop by the HTTP handler; appending it here,
@@ -5968,6 +5993,76 @@ void server_routes::init_routes() {
         return res;
     };
 
+    this->post_lora_adapters_save = [this](const server_http_req & req) {
+        auto res = create_response();
+        const json body = json::parse(req.body);
+        const auto path = confine_out(params.train_dir, json_value(body, "name", std::string()));
+        if (params.train_dir.empty() || path.empty() || !body.contains("id") || !body.at("id").is_number_integer() ||
+            body.at("id").get<double>() < 0 || body.at("id").get<double>() > INT32_MAX) {
+            res->error(format_error_response("save requires an integer id and a bare .gguf name under --train-dir", ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+        auto & rd = res->rd;
+        server_task task(SERVER_TASK_TYPE_SAVE_LORA);
+        task.id = rd.get_new_id();
+        task.save_lora = body.at("id").get<int32_t>();
+        task.load_lora_path = path;
+        rd.post_task(std::move(task));
+        auto result = rd.next(req.should_stop);
+        if (!result) {
+            return res; // queued snapshot remains owned by the loop after disconnect
+        }
+        if (result->is_error()) {
+            res->error(result->to_json());
+        } else {
+            res->ok(result->to_json());
+        }
+        return res;
+    };
+
+    this->post_lora_adapters_fresh = [this](const server_http_req & req) -> server_http_res_ptr {
+        auto res = create_response();
+        const json body = json::parse(req.body);
+        const auto path = confine_out(params.train_dir, json_value(body, "name", std::string()));
+        if (params.train_dir.empty() || path.empty() || ctx_server.model_tgt == nullptr) {
+            res->error(format_error_response("fresh requires a loaded model and a bare .gguf name under --train-dir", ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+        const double rank = body.value("rank", 8.0);
+        const double alpha = body.value("alpha", 16.0);
+        const double top = body.value("top_layers", 0.0);
+        const double seed = body.value("seed", 42.0);
+        if (!std::isfinite(rank) || rank < 1 || rank > 256 || std::floor(rank) != rank ||
+            !std::isfinite(alpha) || alpha <= 0 || alpha > std::numeric_limits<float>::max() ||
+            !std::isfinite(top) || top < 0 || top > llama_model_n_layer(ctx_server.model_tgt) || std::floor(top) != top ||
+            !std::isfinite(seed) || seed < 0 || seed > UINT32_MAX || std::floor(seed) != seed) {
+            res->error(format_error_response("invalid rank, alpha, top_layers or seed", ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+        std::vector<std::string> targets;
+        std::stringstream parts(body.value("targets", std::string("attn_q,attn_v")));
+        std::string target;
+        while (std::getline(parts, target, ',')) {
+            if (!target.empty()) {
+                targets.push_back(target);
+            }
+        }
+        if (targets.empty()) {
+            res->error(format_error_response("targets must name at least one module", ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+        const auto error = server_publish_adapter(path, [&](const std::string & temporary) {
+            return common_lora_write_fresh(ctx_server.model_tgt, temporary, (int32_t) rank, (float) alpha,
+                                           targets, (uint32_t) seed, (int32_t) top);
+        });
+        if (!error.empty()) {
+            res->error(format_error_response(error, ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+        // Reuse load's ownership transfer and dormant scale-zero installation.
+        return post_lora_adapters_load(req);
+    };
+
     this->post_lora_adapters_load = [this](const server_http_req & req) {
         auto res = create_response();
         const json body = json::parse(req.body);

tools/server/server-context.h

@@ -168,6 +168,8 @@ struct server_routes {
     server_http_context::handler_t get_lora_adapters;
     server_http_context::handler_t post_lora_adapters;
     server_http_context::handler_t post_lora_adapters_load;
+    server_http_context::handler_t post_lora_adapters_fresh;
+    server_http_context::handler_t post_lora_adapters_save;
     server_http_context::handler_t post_lora_adapters_unload;
     server_http_context::handler_t post_learn;
     server_http_context::handler_t get_train;

tools/server/server-task.h

@@ -59,6 +59,7 @@ enum server_task_type {
     SERVER_TASK_TYPE_GET_LORA,
     SERVER_TASK_TYPE_SET_LORA,
     SERVER_TASK_TYPE_LOAD_LORA,
+    SERVER_TASK_TYPE_SAVE_LORA,
     SERVER_TASK_TYPE_UNLOAD_LORA,
     SERVER_TASK_TYPE_LEARN,
 };
@@ -260,6 +261,8 @@ struct server_task {
     // server loop; the loop only appends it (dormant, scale 0) to the adapter list
     llama_adapter_lora * load_lora = nullptr;
     std::string          load_lora_path;
+    // SAVE_LORA runs between decode/learn steps; path is confined by the route.
+    int32_t save_lora = -1;
 
     // used by SERVER_TASK_TYPE_UNLOAD_LORA: the adapter id to retire
     int32_t unload_lora = -1;

tools/server/server.cpp

@@ -218,6 +218,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
         routes.get_lora_adapters           = models_routes->proxy_get;
         routes.post_lora_adapters          = models_routes->proxy_post;
         routes.post_lora_adapters_load     = models_routes->proxy_post;
+        routes.post_lora_adapters_fresh    = models_routes->proxy_post;
+        routes.post_lora_adapters_save     = models_routes->proxy_post;
         routes.post_lora_adapters_unload   = models_routes->proxy_post;
         routes.post_learn                  = models_routes->proxy_post;
         routes.get_slots                   = models_routes->proxy_get;
@@ -273,6 +275,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
     ctx_http.get ("/lora-adapters",            ex_wrapper(routes.get_lora_adapters));
     ctx_http.post("/lora-adapters",            ex_wrapper(routes.post_lora_adapters));
     ctx_http.post("/lora-adapters/load",       ex_wrapper(routes.post_lora_adapters_load));
+    ctx_http.post("/lora-adapters/fresh",      ex_wrapper(routes.post_lora_adapters_fresh));
+    ctx_http.post("/lora-adapters/save",       ex_wrapper(routes.post_lora_adapters_save));
     ctx_http.post("/lora-adapters/unload",     ex_wrapper(routes.post_lora_adapters_unload));
     ctx_http.post("/learn",                    ex_wrapper(routes.post_learn));
     ctx_http.get ("/train",                    ex_wrapper(routes.get_train));

tools/server/tests/dream-adapter-routes.py

@@ -0,0 +1,79 @@
+"""D1 live gate. Run against an idle test server with --train-dir, never production.
+
+python dream-adapter-routes.py http://127.0.0.1:8080 /path/to/server/train-dir
+Checks dormant and enabled zero-B parity, changed weights, snapshot reload parity, immutable names,
+bad ids, path confinement, and competing publication of the same name.
+"""
+import concurrent.futures
+import hashlib
+import json
+from pathlib import Path
+import sys
+import urllib.error
+import urllib.request
+import uuid
+
+url = sys.argv[1].rstrip("/")
+train_dir = Path(sys.argv[2])
+prefix = "d1-" + uuid.uuid4().hex
+
+
+def post(route, body):
+    req = urllib.request.Request(url + route, json.dumps(body).encode(),
+                                 {"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as response:
+            return response.status, json.load(response)
+    except urllib.error.HTTPError as error:
+        return error.code, json.load(error)
+
+
+def ok(route, body):
+    status, result = post(route, body)
+    assert status == 200, (route, status, result)
+    return result
+
+
+def complete(adapter=None):
+    body = {"prompt": "The first four prime numbers are", "n_predict": 24,
+            "temperature": 0, "seed": 17, "cache_prompt": False,
+            "lora": [] if adapter is None else [{"id": adapter, "scale": 1.0}]}
+    return ok("/completion", body)["content"]
+
+
+baseline = complete()
+fresh_name = prefix + "-fresh.gguf"
+fresh = ok("/lora-adapters/fresh", {"name": fresh_name, "rank": 8,
+           "alpha": 16, "targets": "attn_q,attn_v", "top_layers": 1, "seed": 42})
+assert complete() == baseline, "dormant shadow changed base output"
+assert complete(fresh["id"]) == baseline, "zero-B shadow changed enabled output"
+before_name = prefix + "-before.gguf"
+ok("/lora-adapters/save", {"id": fresh["id"], "name": before_name})
+before = hashlib.sha256((train_dir / before_name).read_bytes()).hexdigest()
+repeat_name = prefix + "-repeat.gguf"
+ok("/lora-adapters/save", {"id": fresh["id"], "name": repeat_name})
+assert hashlib.sha256((train_dir / repeat_name).read_bytes()).hexdigest() == before, "save bytes are unstable without learning"
+learned = ok("/completion", {"prompt": "The capital of Kansas is", "id_slot": 0,
+             "n_predict": 1, "temperature": 0,
+             "learn": {"reply": "Topeka is the capital of Kansas.",
+                       "lora_id": fresh["id"], "lr": 0.001}})
+assert learned["learn"]["success"], learned
+trained_text = complete(fresh["id"])
+snapshot_name = prefix + "-snapshot.gguf"
+snapshot = ok("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})
+after = hashlib.sha256((train_dir / snapshot_name).read_bytes()).hexdigest()
+assert after != before, "successful learn did not change saved adapter weights"
+loaded = ok("/lora-adapters/load", {"name": snapshot_name})
+assert complete(loaded["id"]) == trained_text, "trained snapshot reload changed output"
+assert post("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})[0] >= 400
+assert post("/lora-adapters/fresh", {"name": fresh_name})[0] >= 400
+for bad_id in (-1, 2**32, 2**63):
+    assert post("/lora-adapters/save", {"id": bad_id, "name": prefix + "-bad.gguf"})[0] >= 400
+for route in ("/lora-adapters/fresh", "/lora-adapters/save"):
+    assert post(route, {"id": fresh["id"], "name": "../escape.gguf"})[0] >= 400
+with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+    results = list(pool.map(lambda _: post("/lora-adapters/save",
+                   {"id": fresh["id"], "name": prefix + "-race.gguf"})[0], range(2)))
+assert sorted(status == 200 for status in results) == [False, True], results
+print(json.dumps({"gate": "D1", "fresh": fresh, "snapshot": snapshot,
+                  "before_sha256": before, "after_sha256": after, "passed": True}))