Run the spec's whole per-port corpus through both tiers, from one copy CI checks

.github/workflows/ci.yml

@@ -110,13 +110,12 @@ jobs:
         run: "$PROTOWIRE_DIR/scripts/cross_write_check.sh"
 
       # pb-go-golden.txt is a byte copy of the spec's struct-binding table
-      # (protowire#518). Until that merges the spec's main has no table, and
-      # this says so rather than fails; drop the guard once it has.
+      # (protowire#518).
       - name: Struct-binding table matches the spec's
-        run: |
-          table="$PROTOWIRE_DIR/testdata/values/values.tsv"
-          if [[ ! -f "$table" ]]; then
-            echo "::warning::protowire main has no testdata/values/values.tsv yet (protowire#518)"
-            exit 0
-          fi
-          cmp "$table" pb/src/test/resources/golden/pb-go-golden.txt
+        run: cmp "$PROTOWIRE_DIR/testdata/values/values.tsv" pb/src/test/resources/golden/pb-go-golden.txt
+
+      # testdata/ is the spec corpus both tiers run (#211). A fixture the spec
+      # adds or changes fails here until it is copied, and the copy fails a
+      # tier's test until that tier meets it or lists it as a known gap.
+      - name: Spec corpus copy matches the spec's
+        run: scripts/check-spec-corpus.sh "$PROTOWIRE_DIR"

CHANGELOG.md

@@ -20,6 +20,8 @@ format changes.
 
 ### Added
 
+- **Both tiers run the spec's whole per-port corpus, from one copy CI keeps equal to the spec** (#211). `testdata/` holds protowire's `testdata/{annotations,any,bigfloat,blocks,dataset,duplicates,duration,keyed,map-keys,nulls,repeated,strings}/` and `test.proto` byte for byte, schemas included. The full tier compiles the schemas as test protos. `lite-fixtures` generates them lite with their companions, adding the `java_package` that `protoc-gen-pxf-java-meta` requires at build time. `LiteCorpusTest` runs every fixture through `LiteWireWriter` and `LiteWireReader`. Before, the lite tier ran hand-typed copies of two directories and none of the rest, and the full tier ran no `duration` or `repeated` fixture. `scripts/check-spec-corpus.sh`, run by the conformance job, fails when a copy differs from spec main or when the spec has a corpus directory with no copy. A fixture a tier does not meet is listed against its issue, and a listed fixture that starts passing fails its test. The first run found the lite gaps #213 (Any `@type` blocks), #214 (a scalar on a repeated field), #215 (three error texts) and more of #191 (`IllegalArgumentException`), and both tiers' #189 cases from protowire#569.
+
 - **The lite tier reads and writes fields of `google.protobuf.Any`, `Empty`, `FieldMask`, `Struct`, `Value`, `ListValue` and `NullValue`** (trendvidia/protowire#539). A lite schema with such a field did not compile, because its generated meta named `com.google.protobuf.AnyPxfMeta`, which nothing generates. `:pxf-runtime` now ships these companions in `org.protowire.pxf.wkt`, as `protoc-gen-pxf-java-meta`'s output for the four google files, and the plugin from trendvidia/protowire#550 on points there. They are written as ordinary blocks, the same text protowire-go writes. An Any is written as `type_url` and `value`, since the lite tier has no type resolver. `pxf-runtime/scripts/regen-wkt-companions.sh` regenerates the companions.
 
 - **Producers refuse a message nested past `MaxNestingDepth`** (#167; HARDENING.md § Recursion, protowire#502). `Pxf.marshal` counts each `{` and `[` it writes, as a decoder of the text counts them. `Pb.marshal` counts each submessage and map entry. Past the limit, `Pxf.marshal` throws `PxfException` and `Pb.marshal` throws `IOException`, both with the message `nesting depth exceeds MaxNestingDepth=100`. Before, they wrote a document that a reader at the same limit refuses, and a reference cycle in a `Pb` struct overflowed the stack. The limit is per call: `MarshalOptions.withMaxNestingDepth` in `pxf`, and the new `org.protowire.pb.MarshalOptions` with `Pb.marshal(Object, MarshalOptions)`. An Any whose body would go past the limit is still written, as `type_url` and `value`. `check-decode --format pb-render` now reads its input with the stock reader's limit raised, so the corpus's 101-level rows reach the producer.

lite-fixtures/build.gradle.kts

@@ -7,12 +7,13 @@ plugins {
     id("com.google.protobuf") version "0.10.0"
 }
 
-// The lite tier's copies of protowire's cross-port fixtures, generated lite
-// with their protoc-gen-pxf-java-meta companions, plus the --pb/--sbe modes
-// both lite dumpers expose to scripts/cross_envelope_check.sh:
+// The spec corpus schemas (../testdata, protowire's testdata byte for byte),
+// generated lite with their protoc-gen-pxf-java-meta companions, for the lite
+// corpus test and for the --pb/--sbe modes both lite dumpers expose to
+// scripts/cross_envelope_check.sh:
 //
 //   testdata/annotations/settings.proto  (pxf.required) = 1314, (pxf.default) = 1315
-//   testdata/sbe-bench.proto             (sbe.*) 1319-1323, bench.v1.Order
+//   testdata/sbe-bench.proto             (sbe.*) 1319-1323, bench.v1.Order (src/main/proto)
 //
 // A codegen port cannot bind a message from a descriptor set at runtime, so
 // the dumpers name a type generated here and ignore the FDS argument; what
@@ -23,6 +24,9 @@ dependencies {
     api("com.google.protobuf:protobuf-javalite:4.36.2")
     api(project(":pxf-android"))
     api(project(":sbe-runtime"))
+    // pxf.BigFloat and pxf.Secret, which the corpus schemas import, generated
+    // lite with their companions.
+    api(project(":pxf-android-codegen-it"))
     compileOnly(project(":proto-annotations"))
     // compile-only, like sbe-android-codegen-it: the well-known protos on the
     // proto include path come from this jar; it is not on the runtime classpath.
@@ -71,6 +75,38 @@ val buildPxfJavaMetaPlugin by tasks.registering(Exec::class) {
     }
 }
 
+// protoc-gen-pxf-java-meta requires a java_package, and the spec's schemas
+// carry none, so the build adds one (the proto package, as protoc's default
+// would be) and java_multiple_files. Nothing else in a file changes.
+// settings_annotated.proto needs a v1.2 compiler: src/main/proto spells it for
+// stock protoc.
+val corpusProto = layout.buildDirectory.dir("corpus-proto")
+val prepareCorpusProto by tasks.registering(Sync::class) {
+    from(rootDir.resolve("testdata")) {
+        include("**/*.proto")
+        exclude("annotations/settings_annotated.proto")
+    }
+    into(corpusProto)
+    filter { line: String ->
+        val m = Regex("""^package\s+([\w.]+)\s*;""").find(line)
+        if (m == null) line
+        else "$line\noption java_package = \"${m.groupValues[1]}\";\noption java_multiple_files = true;"
+    }
+}
+
+sourceSets {
+    main {
+        proto {
+            srcDir(prepareCorpusProto)
+        }
+    }
+}
+
+tasks.test {
+    useJUnitPlatform()
+    systemProperty("protowire.testdata", rootDir.resolve("testdata").absolutePath)
+}
+
 protobuf {
     protoc {
         artifact = "com.google.protobuf:protoc:4.36.2"