diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 093d727195f6..e78b04560112 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -2923,6 +2923,10 @@ extern "C" { // that cannot: an op without a backward, or an in-place write) GGML_API bool ggml_node_has_backward(const struct ggml_tensor * tensor); + // The first node the loss's gradient cannot pass through (no backward, an in-place write, or a + // non-float source it must differentiate), with why, else NULL: the graphs on which + // ggml_build_backward_expand would abort, for a caller that must refuse instead. + GGML_API struct ggml_tensor * ggml_graph_backward_refusal(const struct ggml_cgraph * cgraph, const char ** why); GGML_API void ggml_build_backward_expand( struct ggml_context * ctx, // context for gradient computation struct ggml_cgraph * cgraph, diff --git a/ggml/src/ggml-opt.cpp b/ggml/src/ggml-opt.cpp index ac787e16b9d9..1ade72b90bf2 100644 --- a/ggml/src/ggml-opt.cpp +++ b/ggml/src/ggml-opt.cpp @@ -545,7 +545,8 @@ static int64_t ggml_opt_output_rows(ggml_opt_context_t opt_ctx) { return n; } -static void ggml_opt_build(ggml_opt_context_t opt_ctx) { +// false (with refusal set) when the loss's gradient cannot pass through the graph +static bool ggml_opt_build(ggml_opt_context_t opt_ctx) { GGML_ASSERT(opt_ctx->ctx_compute && "no compute context set, either use static graphs or set one with ggml_opt_prepare_alloc"); GGML_ASSERT((!opt_ctx->static_graphs || opt_ctx->inputs->data) && "when using static graphs the inputs must be allocated statically"); @@ -731,12 +732,25 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) { if (opt_ctx->buf_static) { if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_FORWARD) { - return; + return true; } } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_FORWARD) { opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors( opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0)); - return; + return true; + } + + // Refused by the op's name before anything is built: ggml_build_backward_expand aborts on such a + // graph, and the M5's serving engine went down with it (the 35B-A3B's first /train met the MoE + // router's CLAMP, 2026-11-01). + { + const char * why = nullptr; + if (const ggml_tensor * node = ggml_graph_backward_refusal(opt_ctx->gf, &why)) { + opt_ctx->refusal = std::string("the training graph has no backward through ") + ggml_op_desc(node) + + " (node \"" + node->name + "\"): " + why; + GGML_LOG_ERROR("%s: %s\n", __func__, opt_ctx->refusal.c_str()); + return false; + } } if (!opt_ctx->accumulators_created) { @@ -801,7 +815,7 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) { if (opt_ctx->buf_static) { if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_GRAD) { - return; + return true; } } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_GRAD) { opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0)); @@ -854,6 +868,7 @@ static void ggml_opt_build(ggml_opt_context_t opt_ctx) { } opt_ctx->buf_cpu = ggml_backend_alloc_ctx_tensors_from_buft(opt_ctx->ctx_cpu, ggml_backend_cpu_buffer_type()); + return true; } ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) { @@ -887,7 +902,11 @@ ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) { result->gf = ggml_new_graph_custom(result->ctx_compute, GGML_DEFAULT_GRAPH_SIZE, /*grads =*/ true); // Forward pass. ggml_build_forward_expand(result->gf, result->outputs); - ggml_opt_build(result); + // static graphs are built here, where no request can be refused: a graph without a backward + // stays the caller's abort, with its reason + if (!ggml_opt_build(result)) { + GGML_ABORT("%s", result->refusal.c_str()); + } return result; } @@ -1547,7 +1566,9 @@ bool ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward) { } if (!opt_ctx->static_graphs) { - ggml_opt_build(opt_ctx); + if (!ggml_opt_build(opt_ctx)) { + return false; + } // Graphs built per step keep their gradient accumulators in ctx_static across graphs, // and the reset above found no graph to reset (gb_grad is rebuilt every step). Without diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 8869f9cedb54..e76bc254f5b1 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -7081,7 +7081,6 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) { case GGML_OP_REPEAT_BACK: case GGML_OP_RMS_NORM: case GGML_OP_MUL_MAT: - case GGML_OP_MUL_MAT_ID: case GGML_OP_SCALE: case GGML_OP_SET: case GGML_OP_CPY: @@ -7094,6 +7093,7 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) { case GGML_OP_GET_ROWS: case GGML_OP_DIAG_MASK_INF: case GGML_OP_DIAG_MASK_ZERO: + case GGML_OP_CLAMP: case GGML_OP_SOFT_MAX: case GGML_OP_ROPE: case GGML_OP_IM2COL: @@ -7124,13 +7124,27 @@ bool ggml_node_has_backward(const struct ggml_tensor * tensor) { default: return false; } + case GGML_OP_MUL_MAT_ID: + return tensor->src[3] == NULL; // no backward through the pointer-table gather case GGML_OP_GLU: - return ggml_get_glu_op(tensor) == GGML_GLU_OP_SWIGLU; + return ggml_get_glu_op(tensor) == GGML_GLU_OP_SWIGLU && tensor->src[1] != NULL; // split swiglu only default: return false; } } +// Whether a node's backward gives its source j a gradient: false for the sources whose gradient +// ggml_compute_backward asserts it never computes. +static bool ggml_backward_reaches_src(const struct ggml_tensor * node, int j) { + switch (node->op) { + case GGML_OP_MUL_MAT_ID: return j != 0; // the experts train frozen + case GGML_OP_SOFT_MAX: return j != 1; // the mask + case GGML_OP_ROPE: return j != 2; // the frequency factors + case GGML_OP_CROSS_ENTROPY_LOSS: return j != 1; // the labels + default: return true; + } +} + static void ggml_compute_backward( struct ggml_context * ctx, struct ggml_cgraph * cgraph, int i, const bool * grads_needed) { struct ggml_tensor * tensor = cgraph->nodes[i]; @@ -7497,6 +7511,19 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false)); } } break; + case GGML_OP_CLAMP: { + // 1 strictly inside (min, max), 0 where the value was clamped (an infinite bound + // never clamps: x - (-inf) and inf - x are +inf, whose step is 1). The MoE router + // normalizes its expert weights by clamp(sum, eps, inf) (llama-graph.cpp), so every + // LoRA on a MoE base trains through it. + if (src0_needs_grads) { + const float min = ggml_get_op_params_f32(tensor, 0); + const float max = ggml_get_op_params_f32(tensor, 1); + struct ggml_tensor * above = ggml_step(ctx, ggml_scale_bias(ctx, src0, 1.0f, -min)); + struct ggml_tensor * below = ggml_step(ctx, ggml_scale_bias(ctx, src0, -1.0f, max)); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, ggml_mul(ctx, above, below), grad)); + } + } break; case GGML_OP_SOFT_MAX: { if (src0_needs_grads) { float scale = 1.0f; @@ -7880,33 +7907,12 @@ void ggml_build_forward_order(struct ggml_cgraph * cgraph, struct ggml_tensor * ggml_build_forward_impl(cgraph, tensor, true, false); } -void ggml_build_backward_expand( - struct ggml_context * ctx, - struct ggml_cgraph * cgraph, - struct ggml_tensor ** grad_accs) { - GGML_ASSERT(cgraph->n_nodes > 0); - GGML_ASSERT(cgraph->grads); - GGML_ASSERT(cgraph->grad_accs); - - const int n_nodes_f = cgraph->n_nodes; - - memset(cgraph->grads, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *)); - memset(cgraph->grad_accs, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *)); - bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool)); - - { - bool any_params = false; - bool any_loss = false; - for (int i = 0; i < n_nodes_f; ++i) { - struct ggml_tensor * node = cgraph->nodes[i]; - any_params = any_params || (node->flags & GGML_TENSOR_FLAG_PARAM); - any_loss = any_loss || (node->flags & GGML_TENSOR_FLAG_LOSS); - } - GGML_ASSERT(any_params && "no trainable parameters found, did you forget to call ggml_set_param?"); - GGML_ASSERT(any_loss && "no training loss found, did you forget to call ggml_set_loss?"); - } - - for (int i = 0; i < n_nodes_f; ++i) { +// Marks in grads_needed every node the loss's gradient passes through, in forward order, and +// returns the first one it cannot pass through (with why), else NULL. ggml_build_backward_expand +// and ggml_graph_backward_refusal share it, so a caller that must not abort (a training request +// on a serving process) refuses exactly the graphs the build would. +static struct ggml_tensor * ggml_backward_mark(const struct ggml_cgraph * cgraph, bool * grads_needed, const char ** why) { + for (int i = 0; i < cgraph->n_nodes; ++i) { struct ggml_tensor * node = cgraph->nodes[i]; if (node->type == GGML_TYPE_I32) { @@ -7956,8 +7962,11 @@ void ggml_build_backward_expand( // that attends to a q8_0 cache in place, as serving does); every other op needs float. const bool shapes_only = node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_FLASH_ATTN_EXT; - GGML_ASSERT(node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16 || - (shapes_only && ggml_is_quantized(node->src[j]->type))); + if (!(node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16 || + (shapes_only && ggml_is_quantized(node->src[j]->type)))) { + *why = "a source it must differentiate is neither F32 nor F16"; + return node; + } node_needs_grad = true; break; } @@ -7966,13 +7975,71 @@ void ggml_build_backward_expand( } if (!ggml_node_has_backward(node)) { - GGML_ABORT("%s: no backward through %s (%s): its op has none, or it is an in-place write", __func__, - ggml_op_name(node->op), ggml_op_desc(node)); + *why = "its op has no backward, or it is an in-place write"; + return node; + } + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (node->src[j] && !ignore_src[j] && grads_needed[ggml_hash_find(&cgraph->visited_hash_set, node->src[j])] && + !ggml_backward_reaches_src(node, j)) { + *why = "its backward gives no gradient to a source that needs one (a MoE's experts, a softmax mask, " + "rope frequency factors or loss labels)"; + return node; + } } const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node); GGML_ASSERT(ihash != GGML_HASHSET_FULL); GGML_ASSERT(ggml_bitset_get(cgraph->visited_hash_set.used, ihash)); + grads_needed[ihash] = true; + } + return NULL; +} + +struct ggml_tensor * ggml_graph_backward_refusal(const struct ggml_cgraph * cgraph, const char ** why) { + bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool)); + struct ggml_tensor * refused = ggml_backward_mark(cgraph, grads_needed, why); + free(grads_needed); + return refused; +} + +void ggml_build_backward_expand( + struct ggml_context * ctx, + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs) { + GGML_ASSERT(cgraph->n_nodes > 0); + GGML_ASSERT(cgraph->grads); + GGML_ASSERT(cgraph->grad_accs); + + const int n_nodes_f = cgraph->n_nodes; + + memset(cgraph->grads, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *)); + memset(cgraph->grad_accs, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *)); + bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool)); + + { + bool any_params = false; + bool any_loss = false; + for (int i = 0; i < n_nodes_f; ++i) { + struct ggml_tensor * node = cgraph->nodes[i]; + any_params = any_params || (node->flags & GGML_TENSOR_FLAG_PARAM); + any_loss = any_loss || (node->flags & GGML_TENSOR_FLAG_LOSS); + } + GGML_ASSERT(any_params && "no trainable parameters found, did you forget to call ggml_set_param?"); + GGML_ASSERT(any_loss && "no training loss found, did you forget to call ggml_set_loss?"); + } + + const char * why = NULL; + const struct ggml_tensor * refused = ggml_backward_mark(cgraph, grads_needed, &why); + if (refused) { + GGML_ABORT("%s: no backward through %s (%s): %s", __func__, ggml_op_name(refused->op), ggml_op_desc(refused), why); + } + + for (int i = 0; i < n_nodes_f; ++i) { + struct ggml_tensor * node = cgraph->nodes[i]; + const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node); + if (!grads_needed[ihash]) { + continue; + } if (grad_accs && grad_accs[i]) { cgraph->grad_accs[ihash] = grad_accs[i]; cgraph->grads[ihash] = cgraph->grad_accs[ihash]; @@ -7981,7 +8048,6 @@ void ggml_build_backward_expand( cgraph->grad_accs[ihash] = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne); cgraph->grads[ihash] = cgraph->grad_accs[ihash]; } - grads_needed[ihash] = true; } for (int i = n_nodes_f - 1; i >= 0; --i) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 097f5f1a6e42..c799e0052b5c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #ifdef __EMSCRIPTEN__ # define N_THREADS 1 @@ -337,7 +336,11 @@ static double mean_abs_asymm(const float * a, const float * b, const size_t n, c } } - const float asymm = (a[i] - b[i]) / (a[i] + b[i]); + // two equal gradients agree exactly; (a - b)/(a + b) at a = b = 0 is NaN, and one NaN made + // the mean NaN, which no `err > max` refuses: every case with a zero gradient anywhere + // passed whatever the backward computed (a CLAMP backward that let clamped elements + // through passed, 2026-11-01) + const float asymm = a[i] == b[i] ? 0.0f : (a[i] - b[i]) / (a[i] + b[i]); sum += fabsf(asymm); nvalid++; @@ -1771,26 +1774,14 @@ struct test_case { ggml_set_loss(out); ggml_build_forward_expand(gf, out); - // A param-reachable node whose op has no backward is not supported, not an abort: the - // full grad suite must reach its summary (ggml_node_has_backward is the one list). + // A graph the loss's gradient cannot pass through is not supported, not an abort: the full + // grad suite must reach its summary. ggml_graph_backward_refusal is the build's own walk. { - std::unordered_set reach; - for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { - ggml_tensor * node = ggml_graph_node(gf, i); - bool reached = false; - for (int k = 0; k < GGML_MAX_SRC; ++k) { - const ggml_tensor * s = node->src[k]; - reached = reached || (s != nullptr && ((s->flags & GGML_TENSOR_FLAG_PARAM) || reach.count(s))); - } - if (!reached) { - continue; - } - reach.insert(node); - if (!ggml_node_has_backward(node)) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), - test_status_t::NOT_SUPPORTED, std::string("no backward for ") + ggml_op_desc(node))); - return true; - } + const char * why = nullptr; + if (const ggml_tensor * node = ggml_graph_backward_refusal(gf, &why)) { + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, std::string("no backward through ") + ggml_op_desc(node) + ": " + why)); + return true; } } ggml_graph_cpy(gf, gb); @@ -5514,6 +5505,7 @@ struct test_clamp : public test_case { ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); ggml_set_name(a, "a"); ggml_tensor * out = ggml_clamp(ctx, a, min, max); @@ -10240,6 +10232,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cos (type, {7, 1, 5, 3})); test_cases.emplace_back(new test_cos (type, {1024, 1024, 1, 1})); test_cases.emplace_back(new test_clamp (type, {7, 1, 5, 3})); + // the MoE router's normalization: clamp(weights_sum, eps, inf), an infinite bound. Fifty + // elements: half pass the bound, and a loss summing 600 of them (~150) loses the central + // difference to f32 rounding (MAA 1.3e-4 against 1e-4 on CPU, 2026-11-01) + test_cases.emplace_back(new test_clamp (type, {10, 5, 1, 1}, 6.103515625e-5f, INFINITY)); test_cases.emplace_back(new test_clamp (type, {1024, 1024, 1, 1})); test_cases.emplace_back(new test_leaky_relu(type, {7, 1, 5, 3})); test_cases.emplace_back(new test_leaky_relu(type, {1024, 1024, 1, 1})); diff --git a/tests/test-opt-dynamic-accum.cpp b/tests/test-opt-dynamic-accum.cpp index f72dce6ea54d..52bf1116eb78 100644 --- a/tests/test-opt-dynamic-accum.cpp +++ b/tests/test-opt-dynamic-accum.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #define CHECK(cond) do { if (!(cond)) { fprintf(stderr, "FAILED %s:%d: %s\n", __FILE__, __LINE__, #cond); exit(1); } } while (0) @@ -374,11 +375,67 @@ static void run_label_projection(bool projected) { ggml_backend_sched_free(sched); ggml_backend_free(cpu); } +// what this catches: a training graph the loss's gradient cannot pass through (here an in-place +// clamp, a view with no backward) must refuse ggml_opt_alloc by the op's name, not abort: the +// abort took the M5's serving engine down on the 35B-A3B's first /train (2026-11-01). The same +// context then trains a graph it can differentiate. +static void run_refuses_graph_without_backward() { + ggml_backend_t cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + CHECK(cpu != nullptr); + ggml_backend_t backends[] = { cpu }; + ggml_backend_sched_t sched = ggml_backend_sched_new(backends, nullptr, 1, GGML_DEFAULT_GRAPH_SIZE, false, true); + + ggml_init_params sp = { 8 * ggml_tensor_overhead(), nullptr, true }; + ggml_context * ctx_static = ggml_init(sp); + ggml_tensor * w = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, 1); + ggml_set_param(w); + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_static, cpu); + const float w0 = 1.0f; + ggml_backend_tensor_set(w, &w0, 0, sizeof(float)); + + ggml_opt_params params = ggml_opt_default_params(sched, GGML_OPT_LOSS_TYPE_SUM); + params.optimizer = GGML_OPT_OPTIMIZER_TYPE_SGD; + params.get_opt_pars = sgd_pars; + ggml_opt_context_t opt_ctx = ggml_opt_init(params); + + for (const bool differentiable : { false, true }) { + ggml_init_params cp = { GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + 4 * ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), nullptr, true }; + ggml_context * ctx_compute = ggml_init(cp); + ggml_tensor * x = ggml_new_tensor_1d(ctx_compute, GGML_TYPE_F32, 1); + ggml_tensor * wx = ggml_mul(ctx_compute, w, x); + ggml_tensor * out = differentiable ? ggml_clamp(ctx_compute, wx, -8.0f, 8.0f) : ggml_clamp_inplace(ctx_compute, wx, -8.0f, 8.0f); + ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute, GGML_DEFAULT_GRAPH_SIZE, true); + ggml_build_forward_expand(gf, out); + ggml_opt_prepare_alloc(opt_ctx, ctx_compute, gf, x, out); + if (!differentiable) { + CHECK(!ggml_opt_alloc(opt_ctx, true)); + CHECK(std::strstr(ggml_opt_refusal(opt_ctx), "no backward through CLAMP") != nullptr); + } else { + CHECK(ggml_opt_alloc(opt_ctx, true)); + ggml_backend_tensor_set(x, &X, 0, sizeof(float)); + ggml_opt_eval(opt_ctx, nullptr); + CHECK(ggml_opt_refusal(opt_ctx)[0] == '\0'); + float w_after; + ggml_backend_tensor_get(w, &w_after, 0, sizeof(float)); + // |w x| = 2 is inside the clamp, so its gradient is x and SGD moves w by -lr x + CHECK(std::fabs(w_after - (w0 - LR * X)) < 1e-6f); + } + ggml_free(ctx_compute); + } + + ggml_opt_free(opt_ctx); + ggml_backend_buffer_free(buf); + ggml_free(ctx_static); + ggml_backend_sched_free(sched); + ggml_backend_free(cpu); +} + int main() { ggml_backend_load_all(); run(1); run(2); run_manual(); + run_refuses_graph_without_backward(); run_label_projection(false); run_label_projection(true); for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_Q8_0 }) {