producers: refuse a message nested past MaxNestingDepth

CHANGELOG.md

@@ -20,6 +20,7 @@ format changes.
 
 ### Added
 
+- **Producers refuse a message nested past `MaxNestingDepth`** (#167; HARDENING.md § Recursion, protowire#502). `Pxf.marshal` counts each `{` and `[` it writes, as a decoder of the text counts them. `Pb.marshal` counts each submessage and map entry. Past the limit, `Pxf.marshal` throws `PxfException` and `Pb.marshal` throws `IOException`, both with the message `nesting depth exceeds MaxNestingDepth=100`. Before, they wrote a document that a reader at the same limit refuses, and a reference cycle in a `Pb` struct overflowed the stack. The limit is per call: `MarshalOptions.withMaxNestingDepth` in `pxf`, and the new `org.protowire.pb.MarshalOptions` with `Pb.marshal(Object, MarshalOptions)`. An Any whose body would go past the limit is still written, as `type_url` and `value`. `check-decode --format pb-render` now reads its input with the stock reader's limit raised, so the corpus's 101-level rows reach the producer.
 - **`check-decode` and `check-decode-android` gain `--format pxf-pb`** (protowire#509), for the spec's new write gate. It reads a PXF document and writes the PB bytes on stdout. The full tier decodes with `unmarshalFull`, as `dump-envelope` does, and the harness sorts map entries by key before writing, because protobuf-java writes a `DynamicMessage`'s map entries in insertion order. On its first run the gate found 20 pairs that differ from protowire-go, filed as #156, #157 and #158.
 
 - **`PxfMeta.implicitPresenceFields()`** (#156): the singular fields with no presence, which a writer leaves out at their zero value. The default is empty. `protoc-gen-pxf-java-meta` emits `IMPLICIT_PRESENCE_FIELDS` from trendvidia/protowire#532 on, so a companion generated by an older plugin keeps writing every zero.

check-decode/src/main/java/org/protowire/check/CheckDecode.java

@@ -206,12 +206,21 @@ private static void pxfDecode(byte[] data, String schema, String protoPath, Deco
     // fields and bounding nothing, and they are then written as PXF. Pb.java
     // is bypassed on purpose, or it would refuse the row and the encoder would
     // go untested (protowire#448).
+    /** How deep {@link #pbRender}'s stock reader reads: past any producer's limit. */
+    private static final int PB_RENDER_READ_DEPTH = 10_000;
+
     private static void pbRender(byte[] data, String schema, String protoPath) throws Exception {
         if (protoPath == null || protoPath.isEmpty()) {
             throw new RejectException("--proto is required for format=pb-render");
         }
         Descriptor desc = loadDescriptor(protoPath, schema);
-        DynamicMessage msg = DynamicMessage.parseFrom(desc, data);
+        // The stock reader stops at 100 levels by default, which would refuse
+        // a 101-level row before the producer runs, and the reject row would
+        // pass untested (HARDENING.md § Recursion, #167). Read deeper, and
+        // leave the refusal to the producer.
+        com.google.protobuf.CodedInputStream in = com.google.protobuf.CodedInputStream.newInstance(data);
+        in.setRecursionLimit(PB_RENDER_READ_DEPTH);
+        DynamicMessage msg = DynamicMessage.parseFrom(desc, in);
         try {
             Pxf.marshal(msg);
         } catch (RuntimeException e) {

pb/src/main/java/org/protowire/pb/MarshalOptions.java

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 TrendVidia, LLC.
+package org.protowire.pb;
+
+/**
+ * Per-call options for {@link Pb#marshal(Object, MarshalOptions)}, as
+ * protowire-go's {@code pb.MarshalOptions}.
+ *
+ * @param maxNestingDepth the deepest value {@link Pb#marshal} writes, counted
+ *     as a decoder counts it: the root is depth 0, and every submessage or map
+ *     entry is one descent. A value that needs more is refused with an
+ *     {@code IOException}, because a reader at the same limit would refuse the
+ *     bytes, and a reference cycle meets the limit instead of the stack
+ *     (HARDENING.md § Recursion, protowire#502, #167). Raise it only with
+ *     {@link UnmarshalOptions#maxNestingDepth()} on every reader.
+ */
+public record MarshalOptions(int maxNestingDepth) {
+
+    public MarshalOptions {
+        if (maxNestingDepth <= 0) {
+            throw new IllegalArgumentException("MaxNestingDepth must be positive, got " + maxNestingDepth);
+        }
+    }
+
+    /** The cross-port default, {@link Pb#MAX_NESTING_DEPTH}. */
+    public static MarshalOptions defaults() {
+        return new MarshalOptions(Pb.MAX_NESTING_DEPTH);
+    }
+
+    public MarshalOptions withMaxNestingDepth(int v) { return new MarshalOptions(v); }
+}

pb/src/main/java/org/protowire/pb/Pb.java

@@ -51,10 +51,26 @@ public final class Pb {
     private Pb() {}
 
     public static byte[] marshal(Object obj) throws IOException {
+        return marshal(obj, MarshalOptions.defaults());
+    }
+
+    /**
+     * As {@link #marshal(Object)} under per-call options: a value nested more
+     * deeply than {@link MarshalOptions#maxNestingDepth()} is refused (#167).
+     */
+    public static byte[] marshal(Object obj, MarshalOptions opts) throws IOException {
         if (obj == null) throw new IllegalArgumentException("pb.marshal: object is null");
+        return marshalNested(obj, 0, opts.maxNestingDepth());
+    }
+
+    /** A struct's bytes at submessage depth {@code depth}, counted as a decoder counts it. */
+    private static byte[] marshalNested(Object obj, int depth, int maxDepth) throws IOException {
+        if (depth > maxDepth) {
+            throw new IOException("nesting depth exceeds MaxNestingDepth=" + maxDepth);
+        }
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         CodedOutputStream out = CodedOutputStream.newInstance(baos);
-        marshalStruct(out, obj);
+        marshalStruct(out, obj, depth, maxDepth);
         out.flush();
         return baos.toByteArray();
     }
@@ -213,18 +229,19 @@ private static Class<?> mapKeyClass(Field f) {
 
     // -- marshal -------------------------------------------------------------
 
-    private static void marshalStruct(CodedOutputStream out, Object obj) throws IOException {
+    private static void marshalStruct(CodedOutputStream out, Object obj, int depth, int maxDepth) throws IOException {
         StructInfo info = info(obj.getClass());
         for (FieldInfo fi : info.ordered) {
             try {
-                marshalField(out, fi, fi.field.get(obj));
+                marshalField(out, fi, fi.field.get(obj), depth, maxDepth);
             } catch (IllegalAccessException e) {
                 throw new IOException("field " + fi.field.getName() + ": " + e.getMessage(), e);
             }
         }
     }
 
-    private static void marshalField(CodedOutputStream out, FieldInfo fi, Object value) throws IOException {
+    private static void marshalField(CodedOutputStream out, FieldInfo fi, Object value, int depth, int maxDepth)
+            throws IOException {
         if (value == null) return;
 
         if (fi.kind == FieldKind.LIST) {
@@ -241,7 +258,7 @@ private static void marshalField(CodedOutputStream out, FieldInfo fi, Object val
             }
             for (Object elem : list) {
                 marshalScalar(out, fi.number, fi.elementClass, elem == null ? scalarZero(fi.elementClass) : elem,
-                        fi.zigzag, true);
+                        fi.zigzag, true, depth, maxDepth);
             }
             return;
         }
@@ -255,20 +272,24 @@ private static void marshalField(CodedOutputStream out, FieldInfo fi, Object val
             // values inherit the field's zigzag flag.
             Map<?, ?> map = (Map<?, ?>) value;
             if (map.isEmpty()) return;
+            // A map entry is a descent, as a decoder counts it.
+            if (depth + 1 > maxDepth) {
+                throw new IOException("nesting depth exceeds MaxNestingDepth=" + maxDepth);
+            }
             for (Map.Entry<?, ?> e : map.entrySet()) {
                 ByteArrayOutputStream entryBuf = new ByteArrayOutputStream();
                 CodedOutputStream entryOut = CodedOutputStream.newInstance(entryBuf);
                 Object k = e.getKey() == null ? scalarZero(fi.mapKeyClass) : e.getKey();
                 Object v = e.getValue() == null ? scalarZero(fi.elementClass) : e.getValue();
-                marshalScalar(entryOut, 1, fi.mapKeyClass, k, fi.zigzag, true);
-                marshalScalar(entryOut, 2, fi.elementClass, v, fi.zigzag, true);
+                marshalScalar(entryOut, 1, fi.mapKeyClass, k, fi.zigzag, true, depth + 1, maxDepth);
+                marshalScalar(entryOut, 2, fi.elementClass, v, fi.zigzag, true, depth + 1, maxDepth);
                 entryOut.flush();
                 out.writeByteArray(fi.number, entryBuf.toByteArray());
             }
             return;
         }
 
-        marshalScalar(out, fi.number, fi.field.getType(), value, fi.zigzag, false);
+        marshalScalar(out, fi.number, fi.field.getType(), value, fi.zigzag, false, depth, maxDepth);
     }
 
     /**
@@ -320,7 +341,7 @@ private static void marshalPacked(CodedOutputStream out, int num, Class<?> type,
      * key / value ({@code element} true) is always written.
      */
     private static void marshalScalar(CodedOutputStream out, int num, Class<?> type, Object v,
-                                      boolean zigzag, boolean element) throws IOException {
+                                      boolean zigzag, boolean element, int depth, int maxDepth) throws IOException {
         if (type == boolean.class || type == Boolean.class) {
             boolean b = (Boolean) v;
             if (!b && !element) return;
@@ -387,8 +408,7 @@ private static void marshalScalar(CodedOutputStream out, int num, Class<?> type,
                 out.writeByteArray(num, new byte[0]);
                 return;
             }
-            byte[] msg = marshal(v);
-            out.writeByteArray(num, msg);
+            out.writeByteArray(num, marshalNested(v, depth + 1, maxDepth));
         }
     }
 

pb/src/test/java/org/protowire/pb/MarshalDepthTest.java

@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 TrendVidia, LLC.
+package org.protowire.pb;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Pb.marshal refuses a value nested more deeply than MaxNestingDepth,
+ * counted as a decoder counts it: every submessage or map entry is one
+ * descent (HARDENING.md § Recursion, protowire#502, #167), as protowire-go's
+ * pb.MarshalOptions does.
+ */
+final class MarshalDepthTest {
+
+    public static class Node {
+        @ProtoField(1) Node child;
+        @ProtoField(2) Map<String, Node> kids = new HashMap<>();
+        public Node() {}
+    }
+
+    private static Node chain(int n) {
+        Node root = new Node();
+        Node at = root;
+        for (int i = 0; i < n; i++) {
+            at.child = new Node();
+            at = at.child;
+        }
+        return root;
+    }
+
+    private static Node kids(int n) {
+        Node root = new Node();
+        Node at = root;
+        for (int i = 0; i < n; i++) {
+            Node next = new Node();
+            at.kids.put("k", next);
+            at = next;
+        }
+        return root;
+    }
+
+    private static int depthOf(Node n) {
+        int d = 0;
+        while (n.child != null) { n = n.child; d++; }
+        return d;
+    }
+
+    private static void assertRefused(Node n, MarshalOptions opts) {
+        IOException e = assertThrows(IOException.class, () -> Pb.marshal(n, opts));
+        assertTrue(e.getMessage().contains("nesting depth exceeds MaxNestingDepth=" + opts.maxNestingDepth()), e.getMessage());
+    }
+
+    @Test
+    void submessagesUpToTheLimitAreWritten() throws IOException {
+        assertEquals(100, depthOf(Pb.unmarshal(Pb.marshal(chain(100)), Node.class)));
+        assertRefused(chain(101), MarshalOptions.defaults());
+    }
+
+    @Test
+    void aMapEntryIsADescent() throws IOException {
+        // 50 entries, each holding a message: 100 descents.
+        Pb.unmarshal(Pb.marshal(kids(50)), Node.class);
+        Node past = kids(50);
+        Node at = past;
+        while (!at.kids.isEmpty()) at = at.kids.get("k");
+        at.child = new Node();
+        assertRefused(past, MarshalOptions.defaults());
+    }
+
+    @Test
+    void aCycleMeetsTheLimitInsteadOfTheStack() {
+        Node loop = new Node();
+        loop.child = loop;
+        assertRefused(loop, MarshalOptions.defaults());
+    }
+
+    @Test
+    void theLimitIsPerCall() throws IOException {
+        Pb.marshal(chain(150), MarshalOptions.defaults().withMaxNestingDepth(200));
+        assertRefused(chain(11), MarshalOptions.defaults().withMaxNestingDepth(10));
+        assertThrows(IllegalArgumentException.class, () -> new MarshalOptions(0));
+    }
+}

pxf/src/main/java/org/protowire/pxf/Encoder.java

@@ -59,6 +59,18 @@
         return Collections.emptySet();
     }
 
+    /**
+     * Refuses a block or list whose contents sit at {@code level}, the depth a
+     * decoder of the output counts there (HARDENING.md § Recursion): a
+     * producer must not write a document a reader at the same limit refuses
+     * (protowire#502, #167). As protowire-go's encoder words it.
+     */
+    private void checkDepth(int level) {
+        if (level > opts.maxNestingDepth()) {
+            throw new PxfException(Position.UNKNOWN, "pxf: nesting depth exceeds MaxNestingDepth=" + opts.maxNestingDepth());
+        }
+    }
+
     private void writeIndent(int level) { for (int i = 0; i < level; i++) buf.append(opts.indent()); }
 
     /**
@@ -121,6 +133,7 @@
             if (tryEncodeAny(fd, sub, level)) return;
         }
 
+        checkDepth(level + 1);
         writeIndent(level);
         buf.append(fd.getName()).append(" {\n");
         String saved = pathPrefix;
@@ -151,8 +164,8 @@
     /** Writes the literal of a message {@link #hasLiteral} accepts. */
     private void writeLiteral(Message m) {
         Descriptor md = m.getDescriptorForType();
         if (WellKnown.isTimestamp(md))      buf.append(WellKnown.formatRfc3339(WellKnown.readTimestamp(m)));
         else if (WellKnown.isDuration(md))  buf.append(WellKnown.formatGoDuration(WellKnown.readDuration(m)));
         else if (WellKnown.isWrapper(md)) {
             FieldDescriptor inner = md.findFieldByName("value");
             writeScalar(inner, m.getField(inner));
@@ -183,6 +196,7 @@
             encodeKeyedList(msg, fd, keyFd, level);
             return;
         }
+        checkDepth(level + 1);
         writeIndent(level);
         buf.append(fd.getName()).append(" = [\n");
         for (int i = 0; i < n; i++) {
@@ -193,6 +207,7 @@
                 if (hasLiteral(sub)) {
                     writeLiteral(sub);
                 } else {
+                    checkDepth(level + 2);
                     buf.append("{\n");
                     encodeMessage(sub, level + 2);
                     writeIndent(level + 1);
@@ -227,6 +242,7 @@
      */
     private void encodeKeyedList(Message msg, FieldDescriptor fd, FieldDescriptor keyFd, int level) {
         int n = msg.getRepeatedFieldCount(fd);
+        checkDepth(level + 2);
         writeIndent(level);
         buf.append(fd.getName()).append(" {\n");
         for (int i = 0; i < n; i++) {
@@ -248,6 +264,7 @@
     @SuppressWarnings("unchecked")
     private void encodeMap(Message msg, FieldDescriptor fd, int level) {
         int n = msg.getRepeatedFieldCount(fd);
+        checkDepth(level + 1);
         writeIndent(level);
         buf.append(fd.getName()).append(" = {\n");
 
@@ -272,6 +289,7 @@
                 writeLiteral((Message) v);
                 buf.append('\n');
             } else if (valFd.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
+                checkDepth(level + 2);
                 writeIndent(level + 1);
                 buf.append(keyStr).append(": {\n");
                 encodeMessage((Message) v, level + 2);
@@ -347,7 +365,10 @@
         Descriptor anyDesc = fd.getMessageType();
         String typeUrl = (String) anyMsg.getField(anyDesc.findFieldByName("type_url"));
         ByteString valueBytes = (ByteString) anyMsg.getField(anyDesc.findFieldByName("value"));
-        if (typeUrl.isEmpty() || level >= DecodeLimits.defaults().maxNestingDepth()) return false;
+        // At the limit the block has no room for a body, and stopping here
+        // bounds a chain of Anys. A body too deep fails inside the try, and
+        // the Any stays as its own fields.
+        if (typeUrl.isEmpty() || level >= opts.maxNestingDepth()) return false;
 
         Descriptor inner = opts.typeResolver().findMessageByUrl(typeUrl);
         if (inner == null) return false;

pxf/src/main/java/org/protowire/pxf/MarshalOptions.java

@@ -4,23 +4,46 @@
 
 import com.google.protobuf.Message;
 
-/** Configurable PXF encoding options. */
+/**
+ * Configurable PXF encoding options.
+ *
+ * @param maxNestingDepth the deepest document {@link #marshal} writes,
+ *     counted as a decoder counts it: the root is depth 0, and every
+ *     {@code {} or {@code [} is one descent. A message that needs more is an
+ *     error, because a reader at the same limit would refuse the document
+ *     (HARDENING.md § Recursion, protowire#502). An Any past the limit is
+ *     not: it stays as {@code type_url} and {@code value}. Raise it only with
+ *     the same limit on every reader.
+ */
 public record MarshalOptions(
         String indent,
         boolean emitDefaults,
         String typeUrl,
         TypeResolver typeResolver,
-        Result nullFields) {
+        Result nullFields,
+        int maxNestingDepth) {
+
+    public MarshalOptions {
+        if (maxNestingDepth <= 0) {
+            throw new IllegalArgumentException("MaxNestingDepth must be positive, got " + maxNestingDepth);
+        }
+    }
+
+    /** As the six-argument form at the default {@code MaxNestingDepth}. */
+    public MarshalOptions(String indent, boolean emitDefaults, String typeUrl, TypeResolver typeResolver, Result nullFields) {
+        this(indent, emitDefaults, typeUrl, typeResolver, nullFields, Limits.MAX_NESTING_DEPTH);
+    }
 
     public static MarshalOptions defaults() {
         return new MarshalOptions("  ", false, "", null, null);
     }
 
-    public MarshalOptions withIndent(String s)         { return new MarshalOptions(s, emitDefaults, typeUrl, typeResolver, nullFields); }
-    public MarshalOptions withEmitDefaults(boolean v)  { return new MarshalOptions(indent, v, typeUrl, typeResolver, nullFields); }
-    public MarshalOptions withTypeUrl(String s)        { return new MarshalOptions(indent, emitDefaults, s, typeResolver, nullFields); }
-    public MarshalOptions withTypeResolver(TypeResolver r) { return new MarshalOptions(indent, emitDefaults, typeUrl, r, nullFields); }
-    public MarshalOptions withNullFields(Result r)     { return new MarshalOptions(indent, emitDefaults, typeUrl, typeResolver, r); }
+    public MarshalOptions withIndent(String s)         { return new MarshalOptions(s, emitDefaults, typeUrl, typeResolver, nullFields, maxNestingDepth); }
+    public MarshalOptions withEmitDefaults(boolean v)  { return new MarshalOptions(indent, v, typeUrl, typeResolver, nullFields, maxNestingDepth); }
+    public MarshalOptions withTypeUrl(String s)        { return new MarshalOptions(indent, emitDefaults, s, typeResolver, nullFields, maxNestingDepth); }
+    public MarshalOptions withTypeResolver(TypeResolver r) { return new MarshalOptions(indent, emitDefaults, typeUrl, r, nullFields, maxNestingDepth); }
+    public MarshalOptions withNullFields(Result r)     { return new MarshalOptions(indent, emitDefaults, typeUrl, typeResolver, r, maxNestingDepth); }
+    public MarshalOptions withMaxNestingDepth(int v)   { return new MarshalOptions(indent, emitDefaults, typeUrl, typeResolver, nullFields, v); }
 
     public byte[] marshal(Message m) { return new Encoder(this).encode(m); }
 }