Skip to content

ggml: CLAMP has a backward, and a training graph the gradient cannot pass through refuses by name instead of aborting the server #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joelteply merged 1 commit into from
Nov 1, 2026
Merged

ggml: CLAMP has a backward, and a training graph the gradient cannot pass through refuses by name instead of aborting the server #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2923,6 +2923,10 @@ extern "C" {
// that cannot: an op without a backward, or an in-place write)
GGML_API bool ggml_node_has_backward(const struct ggml_tensor * tensor);

// The first node the loss's gradient cannot pass through (no backward, an in-place write, or a
// non-float source it must differentiate), with why, else NULL: the graphs on which
// ggml_build_backward_expand would abort, for a caller that must refuse instead.
GGML_API struct ggml_tensor * ggml_graph_backward_refusal(const struct ggml_cgraph * cgraph, const char ** why);
GGML_API void ggml_build_backward_expand(
struct ggml_context * ctx, // context for gradient computation
struct ggml_cgraph * cgraph,
Expand Down
33 changes: 27 additions & 6 deletions ggml/src/ggml-opt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,7 +545,8 @@ static int64_t ggml_opt_output_rows(ggml_opt_context_t opt_ctx) {
return n;
}

static void ggml_opt_build(ggml_opt_context_t opt_ctx) {
// false (with refusal set) when the loss's gradient cannot pass through the graph
static bool ggml_opt_build(ggml_opt_context_t opt_ctx) {
GGML_ASSERT(opt_ctx->ctx_compute && "no compute context set, either use static graphs or set one with ggml_opt_prepare_alloc");
GGML_ASSERT((!opt_ctx->static_graphs || opt_ctx->inputs->data) && "when using static graphs the inputs must be allocated statically");

Expand Down Expand Up @@ -731,12 +732,25 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) {

if (opt_ctx->buf_static) {
if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_FORWARD) {
return;
return true;
}
} else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_FORWARD) {
opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(
opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));
return;
return true;
}

// Refused by the op's name before anything is built: ggml_build_backward_expand aborts on such a
// graph, and the M5's serving engine went down with it (the 35B-A3B's first /train met the MoE
// router's CLAMP, 2026-11-01).
{
const char * why = nullptr;
if (const ggml_tensor * node = ggml_graph_backward_refusal(opt_ctx->gf, &why)) {
opt_ctx->refusal = std::string("the training graph has no backward through ") + ggml_op_desc(node)
+ " (node \"" + node->name + "\"): " + why;
GGML_LOG_ERROR("%s: %s\n", __func__, opt_ctx->refusal.c_str());
return false;
}
}

if (!opt_ctx->accumulators_created) {
Expand Down Expand Up @@ -801,7 +815,7 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) {

if (opt_ctx->buf_static) {
if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_GRAD) {
return;
return true;
}
} else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_GRAD) {
opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));
Expand Down Expand Up @@ -854,6 +868,7 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) {
}

opt_ctx->buf_cpu = ggml_backend_alloc_ctx_tensors_from_buft(opt_ctx->ctx_cpu, ggml_backend_cpu_buffer_type());
return true;
}

ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) {
Expand Down Expand Up @@ -887,7 +902,11 @@ ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) {
result->gf = ggml_new_graph_custom(result->ctx_compute, GGML_DEFAULT_GRAPH_SIZE, /*grads =*/ true); // Forward pass.
ggml_build_forward_expand(result->gf, result->outputs);

ggml_opt_build(result);
// static graphs are built here, where no request can be refused: a graph without a backward
// stays the caller's abort, with its reason
if (!ggml_opt_build(result)) {
GGML_ABORT("%s", result->refusal.c_str());
}

return result;
}
Expand Down Expand Up @@ -1547,7 +1566,9 @@ bool ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward) {
}

if (!opt_ctx->static_graphs) {
ggml_opt_build(opt_ctx);
if (!ggml_opt_build(opt_ctx)) {
return false;
}

// Graphs built per step keep their gradient accumulators in ctx_static across graphs,
// and the reset above found no graph to reset (gb_grad is rebuilt every step). Without
Expand Down
134 changes: 100 additions & 34 deletions ggml/src/ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7081,7 +7081,6 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) {
case GGML_OP_REPEAT_BACK:
case GGML_OP_RMS_NORM:
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
case GGML_OP_SCALE:
case GGML_OP_SET:
case GGML_OP_CPY:
Expand All @@ -7094,6 +7093,7 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) {
case GGML_OP_GET_ROWS:
case GGML_OP_DIAG_MASK_INF:
case GGML_OP_DIAG_MASK_ZERO:
case GGML_OP_CLAMP:
case GGML_OP_SOFT_MAX:
case GGML_OP_ROPE:
case GGML_OP_IM2COL:
Expand Down Expand Up @@ -7124,13 +7124,27 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) {
default:
return false;
}
case GGML_OP_MUL_MAT_ID:
return tensor->src[3] == NULL; // no backward through the pointer-table gather
case GGML_OP_GLU:
return ggml_get_glu_op(tensor) == GGML_GLU_OP_SWIGLU;
return ggml_get_glu_op(tensor) == GGML_GLU_OP_SWIGLU && tensor->src[1] != NULL; // split swiglu only
default:
return false;
}
}

// Whether a node's backward gives its source j a gradient: false for the sources whose gradient
// ggml_compute_backward asserts it never computes.
static bool ggml_backward_reaches_src(const struct ggml_tensor * node, int j) {
switch (node->op) {
case GGML_OP_MUL_MAT_ID: return j != 0; // the experts train frozen
case GGML_OP_SOFT_MAX: return j != 1; // the mask
case GGML_OP_ROPE: return j != 2; // the frequency factors
case GGML_OP_CROSS_ENTROPY_LOSS: return j != 1; // the labels
default: return true;
}
}

static void ggml_compute_backward(
struct ggml_context * ctx, struct ggml_cgraph * cgraph, int i, const bool * grads_needed) {
struct ggml_tensor * tensor = cgraph->nodes[i];
Expand Down Expand Up @@ -7497,6 +7511,19 @@ static void ggml_compute_backward(
ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false));
}
} break;
case GGML_OP_CLAMP: {
// 1 strictly inside (min, max), 0 where the value was clamped (an infinite bound
// never clamps: x - (-inf) and inf - x are +inf, whose step is 1). The MoE router
// normalizes its expert weights by clamp(sum, eps, inf) (llama-graph.cpp), so every
// LoRA on a MoE base trains through it.
if (src0_needs_grads) {
const float min = ggml_get_op_params_f32(tensor, 0);
const float max = ggml_get_op_params_f32(tensor, 1);
struct ggml_tensor * above = ggml_step(ctx, ggml_scale_bias(ctx, src0, 1.0f, -min));
struct ggml_tensor * below = ggml_step(ctx, ggml_scale_bias(ctx, src0, -1.0f, max));
ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, ggml_mul(ctx, above, below), grad));
}
} break;
case GGML_OP_SOFT_MAX: {
if (src0_needs_grads) {
float scale = 1.0f;
Expand Down Expand Up @@ -7880,33 +7907,12 @@ void ggml_build_forward_order(struct ggml_cgraph * cgraph, struct ggml_tensor *
ggml_build_forward_impl(cgraph, tensor, true, false);
}

void ggml_build_backward_expand(
struct ggml_context * ctx,
struct ggml_cgraph * cgraph,
struct ggml_tensor ** grad_accs) {
GGML_ASSERT(cgraph->n_nodes > 0);
GGML_ASSERT(cgraph->grads);
GGML_ASSERT(cgraph->grad_accs);

const int n_nodes_f = cgraph->n_nodes;

memset(cgraph->grads, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
memset(cgraph->grad_accs, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool));

{
bool any_params = false;
bool any_loss = false;
for (int i = 0; i < n_nodes_f; ++i) {
struct ggml_tensor * node = cgraph->nodes[i];
any_params = any_params || (node->flags & GGML_TENSOR_FLAG_PARAM);
any_loss = any_loss || (node->flags & GGML_TENSOR_FLAG_LOSS);
}
GGML_ASSERT(any_params && "no trainable parameters found, did you forget to call ggml_set_param?");
GGML_ASSERT(any_loss && "no training loss found, did you forget to call ggml_set_loss?");
}

for (int i = 0; i < n_nodes_f; ++i) {
// Marks in grads_needed every node the loss's gradient passes through, in forward order, and
// returns the first one it cannot pass through (with why), else NULL. ggml_build_backward_expand
// and ggml_graph_backward_refusal share it, so a caller that must not abort (a training request
// on a serving process) refuses exactly the graphs the build would.
static struct ggml_tensor * ggml_backward_mark(const struct ggml_cgraph * cgraph, bool * grads_needed, const char ** why) {
for (int i = 0; i < cgraph->n_nodes; ++i) {
struct ggml_tensor * node = cgraph->nodes[i];

if (node->type == GGML_TYPE_I32) {
Expand Down Expand Up @@ -7956,8 +7962,11 @@ void ggml_build_backward_expand(
// that attends to a q8_0 cache in place, as serving does); every other op needs float.
const bool shapes_only = node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE ||
node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_FLASH_ATTN_EXT;
GGML_ASSERT(node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16 ||
(shapes_only && ggml_is_quantized(node->src[j]->type)));
if (!(node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16 ||
(shapes_only && ggml_is_quantized(node->src[j]->type)))) {
*why = "a source it must differentiate is neither F32 nor F16";
return node;
}
node_needs_grad = true;
break;
}
Expand All @@ -7966,13 +7975,71 @@ void ggml_build_backward_expand(
}

if (!ggml_node_has_backward(node)) {
GGML_ABORT("%s: no backward through %s (%s): its op has none, or it is an in-place write", __func__,
ggml_op_name(node->op), ggml_op_desc(node));
*why = "its op has no backward, or it is an in-place write";
return node;
}
for (int j = 0; j < GGML_MAX_SRC; ++j) {
if (node->src[j] && !ignore_src[j] && grads_needed[ggml_hash_find(&cgraph->visited_hash_set, node->src[j])] &&
!ggml_backward_reaches_src(node, j)) {
*why = "its backward gives no gradient to a source that needs one (a MoE's experts, a softmax mask, "
"rope frequency factors or loss labels)";
return node;
}
}

const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node);
GGML_ASSERT(ihash != GGML_HASHSET_FULL);
GGML_ASSERT(ggml_bitset_get(cgraph->visited_hash_set.used, ihash));
grads_needed[ihash] = true;
}
return NULL;
}

struct ggml_tensor * ggml_graph_backward_refusal(const struct ggml_cgraph * cgraph, const char ** why) {
bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool));
struct ggml_tensor * refused = ggml_backward_mark(cgraph, grads_needed, why);
free(grads_needed);
return refused;
}

void ggml_build_backward_expand(
struct ggml_context * ctx,
struct ggml_cgraph * cgraph,
struct ggml_tensor ** grad_accs) {
GGML_ASSERT(cgraph->n_nodes > 0);
GGML_ASSERT(cgraph->grads);
GGML_ASSERT(cgraph->grad_accs);

const int n_nodes_f = cgraph->n_nodes;

memset(cgraph->grads, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
memset(cgraph->grad_accs, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool));

{
bool any_params = false;
bool any_loss = false;
for (int i = 0; i < n_nodes_f; ++i) {
struct ggml_tensor * node = cgraph->nodes[i];
any_params = any_params || (node->flags & GGML_TENSOR_FLAG_PARAM);
any_loss = any_loss || (node->flags & GGML_TENSOR_FLAG_LOSS);
}
GGML_ASSERT(any_params && "no trainable parameters found, did you forget to call ggml_set_param?");
GGML_ASSERT(any_loss && "no training loss found, did you forget to call ggml_set_loss?");
}

const char * why = NULL;
const struct ggml_tensor * refused = ggml_backward_mark(cgraph, grads_needed, &why);
if (refused) {
GGML_ABORT("%s: no backward through %s (%s): %s", __func__, ggml_op_name(refused->op), ggml_op_desc(refused), why);
}

for (int i = 0; i < n_nodes_f; ++i) {
struct ggml_tensor * node = cgraph->nodes[i];
const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node);
if (!grads_needed[ihash]) {
continue;
}
if (grad_accs && grad_accs[i]) {
cgraph->grad_accs[ihash] = grad_accs[i];
cgraph->grads[ihash] = cgraph->grad_accs[ihash];
Expand All @@ -7981,7 +8048,6 @@ void ggml_build_backward_expand(
cgraph->grad_accs[ihash] = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);
cgraph->grads[ihash] = cgraph->grad_accs[ihash];
}
grads_needed[ihash] = true;
}

for (int i = n_nodes_f - 1; i >= 0; --i) {
Expand Down
38 changes: 17 additions & 21 deletions tests/test-backend-ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
#include <thread>
#include <vector>
#include <unordered_map>
#include <unordered_set>

#ifdef __EMSCRIPTEN__
# define N_THREADS 1
Expand Down Expand Up @@ -337,7 +336,11 @@ static double mean_abs_asymm(const float * a, const float * b, const size_t n, c
}
}

const float asymm = (a[i] - b[i]) / (a[i] + b[i]);
// two equal gradients agree exactly; (a - b)/(a + b) at a = b = 0 is NaN, and one NaN made
// the mean NaN, which no `err > max` refuses: every case with a zero gradient anywhere
// passed whatever the backward computed (a CLAMP backward that let clamped elements
// through passed, 2026-11-01)
const float asymm = a[i] == b[i] ? 0.0f : (a[i] - b[i]) / (a[i] + b[i]);

sum += fabsf(asymm);
nvalid++;
Expand Down Expand Up @@ -1771,26 +1774,14 @@ struct test_case {
ggml_set_loss(out);

ggml_build_forward_expand(gf, out);
// A param-reachable node whose op has no backward is not supported, not an abort: the
// full grad suite must reach its summary (ggml_node_has_backward is the one list).
// A graph the loss's gradient cannot pass through is not supported, not an abort: the full
// grad suite must reach its summary. ggml_graph_backward_refusal is the build's own walk.
{
std::unordered_set<const ggml_tensor *> reach;
for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
ggml_tensor * node = ggml_graph_node(gf, i);
bool reached = false;
for (int k = 0; k < GGML_MAX_SRC; ++k) {
const ggml_tensor * s = node->src[k];
reached = reached || (s != nullptr && ((s->flags & GGML_TENSOR_FLAG_PARAM) || reach.count(s)));
}
if (!reached) {
continue;
}
reach.insert(node);
if (!ggml_node_has_backward(node)) {
output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend),
test_status_t::NOT_SUPPORTED, std::string("no backward for ") + ggml_op_desc(node)));
return true;
}
const char * why = nullptr;
if (const ggml_tensor * node = ggml_graph_backward_refusal(gf, &why)) {
output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend),
test_status_t::NOT_SUPPORTED, std::string("no backward through ") + ggml_op_desc(node) + ": " + why));
return true;
}
}
ggml_graph_cpy(gf, gb);
Expand Down Expand Up @@ -5514,6 +5505,7 @@ struct test_clamp : public test_case {

ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
ggml_set_param(a);
ggml_set_name(a, "a");

ggml_tensor * out = ggml_clamp(ctx, a, min, max);
Expand Down Expand Up @@ -10240,6 +10232,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_cos (type, {7, 1, 5, 3}));
test_cases.emplace_back(new test_cos (type, {1024, 1024, 1, 1}));
test_cases.emplace_back(new test_clamp (type, {7, 1, 5, 3}));
// the MoE router's normalization: clamp(weights_sum, eps, inf), an infinite bound. Fifty
// elements: half pass the bound, and a loss summing 600 of them (~150) loses the central
// difference to f32 rounding (MAA 1.3e-4 against 1e-4 on CPU, 2026-11-01)
test_cases.emplace_back(new test_clamp (type, {10, 5, 1, 1}, 6.103515625e-5f, INFINITY));
test_cases.emplace_back(new test_clamp (type, {1024, 1024, 1, 1}));
test_cases.emplace_back(new test_leaky_relu(type, {7, 1, 5, 3}));
test_cases.emplace_back(new test_leaky_relu(type, {1024, 1024, 1, 1}));
Expand Down
Loading
Loading