diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57e8098..6398374 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,13 +110,12 @@ jobs: run: "$PROTOWIRE_DIR/scripts/cross_write_check.sh" # pb-go-golden.txt is a byte copy of the spec's struct-binding table - # (protowire#518). Until that merges the spec's main has no table, and - # this says so rather than fails; drop the guard once it has. + # (protowire#518). - name: Struct-binding table matches the spec's - run: | - table="$PROTOWIRE_DIR/testdata/values/values.tsv" - if [[ ! -f "$table" ]]; then - echo "::warning::protowire main has no testdata/values/values.tsv yet (protowire#518)" - exit 0 - fi - cmp "$table" pb/src/test/resources/golden/pb-go-golden.txt + run: cmp "$PROTOWIRE_DIR/testdata/values/values.tsv" pb/src/test/resources/golden/pb-go-golden.txt + + # testdata/ is the spec corpus both tiers run (#211). A fixture the spec + # adds or changes fails here until it is copied, and the copy fails a + # tier's test until that tier meets it or lists it as a known gap. + - name: Spec corpus copy matches the spec's + run: scripts/check-spec-corpus.sh "$PROTOWIRE_DIR" diff --git a/CHANGELOG.md b/CHANGELOG.md index e419630..95a693f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ format changes. ### Added +- **Both tiers run the spec's whole per-port corpus, from one copy CI keeps equal to the spec** (#211). `testdata/` holds protowire's `testdata/{annotations,any,bigfloat,blocks,dataset,duplicates,duration,keyed,map-keys,nulls,repeated,strings}/` and `test.proto` byte for byte, schemas included. The full tier compiles the schemas as test protos. `lite-fixtures` generates them lite with their companions, adding the `java_package` that `protoc-gen-pxf-java-meta` requires at build time. `LiteCorpusTest` runs every fixture through `LiteWireWriter` and `LiteWireReader`. Before, the lite tier ran hand-typed copies of two directories and none of the rest, and the full tier ran no `duration` or `repeated` fixture. `scripts/check-spec-corpus.sh`, run by the conformance job, fails when a copy differs from spec main or when the spec has a corpus directory with no copy. A fixture a tier does not meet is listed against its issue, and a listed fixture that starts passing fails its test. The first run found the lite gaps #213 (Any `@type` blocks), #214 (a scalar on a repeated field), #215 (three error texts) and more of #191 (`IllegalArgumentException`), and both tiers' #189 cases from protowire#569. + - **The lite tier reads and writes fields of `google.protobuf.Any`, `Empty`, `FieldMask`, `Struct`, `Value`, `ListValue` and `NullValue`** (trendvidia/protowire#539). A lite schema with such a field did not compile, because its generated meta named `com.google.protobuf.AnyPxfMeta`, which nothing generates. `:pxf-runtime` now ships these companions in `org.protowire.pxf.wkt`, as `protoc-gen-pxf-java-meta`'s output for the four google files, and the plugin from trendvidia/protowire#550 on points there. They are written as ordinary blocks, the same text protowire-go writes. An Any is written as `type_url` and `value`, since the lite tier has no type resolver. `pxf-runtime/scripts/regen-wkt-companions.sh` regenerates the companions. - **Producers refuse a message nested past `MaxNestingDepth`** (#167; HARDENING.md § Recursion, protowire#502). `Pxf.marshal` counts each `{` and `[` it writes, as a decoder of the text counts them. `Pb.marshal` counts each submessage and map entry. Past the limit, `Pxf.marshal` throws `PxfException` and `Pb.marshal` throws `IOException`, both with the message `nesting depth exceeds MaxNestingDepth=100`. Before, they wrote a document that a reader at the same limit refuses, and a reference cycle in a `Pb` struct overflowed the stack. The limit is per call: `MarshalOptions.withMaxNestingDepth` in `pxf`, and the new `org.protowire.pb.MarshalOptions` with `Pb.marshal(Object, MarshalOptions)`. An Any whose body would go past the limit is still written, as `type_url` and `value`. `check-decode --format pb-render` now reads its input with the stock reader's limit raised, so the corpus's 101-level rows reach the producer. diff --git a/lite-fixtures/build.gradle.kts b/lite-fixtures/build.gradle.kts index d9bb51d..1399ecf 100644 --- a/lite-fixtures/build.gradle.kts +++ b/lite-fixtures/build.gradle.kts @@ -7,12 +7,13 @@ plugins { id("com.google.protobuf") version "0.10.0" } -// The lite tier's copies of protowire's cross-port fixtures, generated lite -// with their protoc-gen-pxf-java-meta companions, plus the --pb/--sbe modes -// both lite dumpers expose to scripts/cross_envelope_check.sh: +// The spec corpus schemas (../testdata, protowire's testdata byte for byte), +// generated lite with their protoc-gen-pxf-java-meta companions, for the lite +// corpus test and for the --pb/--sbe modes both lite dumpers expose to +// scripts/cross_envelope_check.sh: // // testdata/annotations/settings.proto (pxf.required) = 1314, (pxf.default) = 1315 -// testdata/sbe-bench.proto (sbe.*) 1319-1323, bench.v1.Order +// testdata/sbe-bench.proto (sbe.*) 1319-1323, bench.v1.Order (src/main/proto) // // A codegen port cannot bind a message from a descriptor set at runtime, so // the dumpers name a type generated here and ignore the FDS argument; what @@ -23,6 +24,9 @@ dependencies { api("com.google.protobuf:protobuf-javalite:4.36.2") api(project(":pxf-android")) api(project(":sbe-runtime")) + // pxf.BigFloat and pxf.Secret, which the corpus schemas import, generated + // lite with their companions. + api(project(":pxf-android-codegen-it")) compileOnly(project(":proto-annotations")) // compile-only, like sbe-android-codegen-it: the well-known protos on the // proto include path come from this jar; it is not on the runtime classpath. @@ -71,6 +75,38 @@ val buildPxfJavaMetaPlugin by tasks.registering(Exec::class) { } } +// protoc-gen-pxf-java-meta requires a java_package, and the spec's schemas +// carry none, so the build adds one (the proto package, as protoc's default +// would be) and java_multiple_files. Nothing else in a file changes. +// settings_annotated.proto needs a v1.2 compiler: src/main/proto spells it for +// stock protoc. +val corpusProto = layout.buildDirectory.dir("corpus-proto") +val prepareCorpusProto by tasks.registering(Sync::class) { + from(rootDir.resolve("testdata")) { + include("**/*.proto") + exclude("annotations/settings_annotated.proto") + } + into(corpusProto) + filter { line: String -> + val m = Regex("""^package\s+([\w.]+)\s*;""").find(line) + if (m == null) line + else "$line\noption java_package = \"${m.groupValues[1]}\";\noption java_multiple_files = true;" + } +} + +sourceSets { + main { + proto { + srcDir(prepareCorpusProto) + } + } +} + +tasks.test { + useJUnitPlatform() + systemProperty("protowire.testdata", rootDir.resolve("testdata").absolutePath) +} + protobuf { protoc { artifact = "com.google.protobuf:protoc:4.36.2" diff --git a/lite-fixtures/src/main/proto/keyed.proto b/lite-fixtures/src/main/proto/keyed.proto deleted file mode 100644 index 0e40284..0000000 --- a/lite-fixtures/src/main/proto/keyed.proto +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 TrendVidia, LLC. - -// The lite tier's copy of protowire's testdata/keyed/keyed.proto, with -// java_package added because protoc-gen-pxf-java-meta requires one. The fixtures -// themselves are read from pxf/src/test/resources/keyed/. - -// Schema for the keyed-repeated-field conformance fixtures under -// testdata/keyed/ (issue #116; IETF draft -01 "Keyed Repeated Fields"). -// -// Node.children is the reference keyed field (identifier-safe keys and -// self-recursion); Deployment.regions exercises keys that are not -// identifier-safe and therefore require the quoted entry-name form; -// Doc.items is a plain (un-keyed) repeated message field used by the -// quoted-name-outside-keyed-context schema-error fixture; Catalog.choices -// and Catalog.tags have a key field that a block could undo, a oneof member -// and an optional field. - -syntax = "proto3"; -package keyed.v1; - -option java_package = "keyed.v1"; -option java_multiple_files = true; - -import "pxf/annotations.proto"; - -message Node { - string id = 1; - string type = 2; - int32 weight = 3; - repeated Node children = 4 [(pxf.key) = "id"]; -} - -message Region { - string name = 1; - int32 replicas = 2; -} - -message Deployment { - string service = 1; - repeated Region regions = 2 [(pxf.key) = "name"]; -} - -message Plain { - string id = 1; - string label = 2; -} - -message Doc { - repeated Plain items = 1; -} - -message Choice { - oneof pick { - string name = 1; - string other = 2; - } -} - -message Tag { - optional string name = 1; -} - -message Catalog { - repeated Choice choices = 1 [(pxf.key) = "name"]; - repeated Tag tags = 2 [(pxf.key) = "name"]; -} diff --git a/lite-fixtures/src/main/proto/nulls.proto b/lite-fixtures/src/main/proto/nulls.proto deleted file mode 100644 index a70048b..0000000 --- a/lite-fixtures/src/main/proto/nulls.proto +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 TrendVidia, LLC. - -// Schema for the null-binding fixtures under testdata/nulls/ (issue #339; -// IETF draft -01 § Booleans, Null, and Identifier Values). The proto2 -// case is in nulls_proto2.proto. - -syntax = "proto3"; - -package nulls.v1; - -// Java options for this port's codegen; the rest of the file is protowire's -// testdata/nulls/nulls.proto verbatim. -option java_package = "nulls.v1"; -option java_multiple_files = true; - -import "google/protobuf/wrappers.proto"; -import "pxf/annotations.proto"; - -message Inner { - int32 x = 1; -} - -message Holder { - int32 n = 1; - optional int32 o = 2; - repeated int32 l = 3; - oneof k { - string ks = 4; - Inner km = 5; - } - Inner msg = 6; - google.protobuf.Int32Value w = 7; - map m = 8; - map mm = 9; -} - -// A null optional scalar counts as present for (pxf.required). -message Required { - optional int32 r = 1 [(pxf.required) = true]; -} - -// A null optional scalar is not given its (pxf.default). -message Defaulted { - optional int32 d = 1 [(pxf.default) = "5"]; -} diff --git a/lite-fixtures/src/main/proto/nulls_editions.proto b/lite-fixtures/src/main/proto/nulls_editions.proto deleted file mode 100644 index 39104de..0000000 --- a/lite-fixtures/src/main/proto/nulls_editions.proto +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 TrendVidia, LLC. - -// Editions schema for the null-binding fixtures under testdata/nulls/ -// (issue #380; IETF draft -01 § Booleans, Null, and Identifier Values). -// In an editions file a scalar's presence is its field_presence -// feature: EXPLICIT, the 2023 default, may hold a null; IMPLICIT may not. - -edition = "2023"; - -package nulls.ed.v1; - -// Java options for this port's codegen; the rest of the file is protowire's -// testdata/nulls/nulls_editions.proto verbatim. -option java_package = "nulls.ed.v1"; -option java_multiple_files = true; - -message Ed { - int32 explicit_scalar = 1; - int32 implicit_scalar = 2 [features.field_presence = IMPLICIT]; -} diff --git a/lite-fixtures/src/main/proto/nulls_proto2.proto b/lite-fixtures/src/main/proto/nulls_proto2.proto deleted file mode 100644 index f085d64..0000000 --- a/lite-fixtures/src/main/proto/nulls_proto2.proto +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 TrendVidia, LLC. - -// The proto2 schema for the null-binding fixtures under testdata/nulls/ -// (issue #339): a proto2 singular scalar has explicit presence, and a -// proto2 required field rejects null (#379). - -syntax = "proto2"; - -package nulls.v1; - -// Java options for this port's codegen; the rest of the file is protowire's -// testdata/nulls/nulls_proto2.proto verbatim. -option java_package = "nulls.v1"; -option java_multiple_files = true; - -message Legacy { - optional int32 p = 1; -} - -message Strict { - required int32 r = 1; -} diff --git a/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteCorpusTest.java b/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteCorpusTest.java new file mode 100644 index 0000000..40d8269 --- /dev/null +++ b/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteCorpusTest.java @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 TrendVidia, LLC. +package org.protowire.lite.fixtures; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.function.Executable; +import org.protowire.pxf.Format; +import org.protowire.pxf.Parser; +import org.protowire.pxf.PxfException; +import org.protowire.pxf.PxfMeta; +import org.protowire.pxf.SimplePxfRegistry; +import org.protowire.pxf.android.LiteWireReader; +import org.protowire.pxf.android.LiteWireWriter; + +/** + * Every fixture of the spec corpus under {@code testdata/} (protowire's + * {@code testdata//}, byte for byte) through the lite tier: text to wire + * through {@link LiteWireWriter}, wire to text through {@link LiteWireReader}, + * with the companions protoc-gen-pxf-java-meta generated from the same + * schemas. The full tier runs the same files (pxf's {@code *CorpusTest}); + * the lite tier has to give the same verdicts and the same bytes. + * + *

A fixture the lite tier does not meet yet is in {@link #KNOWN_GAPS} + * against its open issue. A listed fixture that meets its verdict fails the + * test, so the list only shrinks. + */ +class LiteCorpusTest { + + private static final Path ROOT = Path.of(System.getProperty("protowire.testdata", "testdata-unset")); + + /** + * {@code

/} to the open issue that fixes it. A key ending in + * {@code #text} is the error-text check of that fixture, not its verdict. + */ + private static final Map KNOWN_GAPS = Map.ofEntries( + // null on a (pxf.required) field is accepted + Map.entry("nulls/err-required-optional.pxf", "#189"), + Map.entry("nulls/err-required-wrapper.pxf", "#189"), + // rejected, but with IllegalArgumentException rather than PxfException + Map.entry("nulls/err-list-element.pxf", "#191"), + Map.entry("nulls/err-oneof-sibling.pxf", "#191"), + Map.entry("annotations/missing-required.pxf", "#191"), + // an Any block's @type entry is refused + Map.entry("any/assignment.pxf", "#213"), + Map.entry("any/block.pxf", "#213"), + Map.entry("any/list-element.pxf", "#213"), + Map.entry("any/map-value.pxf", "#213"), + Map.entry("any/url-verbatim.pxf", "#213"), + // a scalar or an assigned block on a repeated field is accepted + Map.entry("repeated/err-scalar-entries.pxf", "#214"), + Map.entry("repeated/err-assigned-block.pxf", "#214"), + // the error text is not Go's + Map.entry("map-keys/invalid#text", "#215"), + Map.entry("duplicates/err-oneof-member-twice.pxf#text", "#215"), + Map.entry("duplicates/err-map-key-twice-int-spellings.pxf#text", "#215")); + + /** Every corpus directory, and the test here that drives it. */ + private static final Set DRIVEN = Set.of( + "annotations", "any", "bigfloat", "blocks", "dataset", "duplicates", "duration", + "keyed", "map-keys", "nulls", "repeated", "strings"); + + /** The directories whose verdict is the file name: {@code err-*}, or under {@code invalid/}. */ + private static final List BY_NAME = List.of( + "any", "blocks", "duplicates", "duration", "keyed", "map-keys", "nulls", "repeated", "strings"); + + // -- harness ------------------------------------------------------------- + + private static Path dir(String dir) { + Path p = ROOT.resolve(dir); + assertTrue(Files.isDirectory(p), "missing vendored corpus " + p + " (run through Gradle)"); + return p; + } + + private static byte[] read(Path p) { + try { + return Files.readAllBytes(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static byte[] fixture(String dir, String name) { + return read(dir(dir).resolve(name)); + } + + private static String text(String dir, String name) { + return new String(fixture(dir, name), StandardCharsets.UTF_8); + } + + /** The documents of a directory, {@code invalid/} included, without the {@code .expected.pxf} halves of pairs. */ + private static List documents(String dir) { + Path root = dir(dir); + try (Stream s = Files.walk(root)) { + List out = s.filter(Files::isRegularFile) + .map(p -> root.relativize(p).toString()) + .filter(n -> n.endsWith(".pxf") && !n.endsWith(".expected.pxf")) + .sorted() + .collect(Collectors.toList()); + assertTrue(!out.isEmpty(), "empty corpus " + dir); + return out; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static boolean rejects(String name) { + String file = name.substring(name.lastIndexOf('/') + 1); + return file.startsWith("err-") || name.startsWith("invalid/"); + } + + /** The companion of a generated message, by its proto full name. */ + static PxfMeta meta(String fullName) { + int dot = fullName.lastIndexOf('.'); + String cls = fullName.substring(0, dot) + "." + fullName.substring(dot + 1) + "PxfMeta"; + try { + return (PxfMeta) Class.forName(cls).getField("INSTANCE").get(null); + } catch (ReflectiveOperationException e) { + throw new AssertionError("no companion " + cls + " for " + fullName, e); + } + } + + /** The type a document names in its {@code @type} directive. */ + private static String typeOf(byte[] doc) { + for (String line : new String(doc, StandardCharsets.UTF_8).split("\n")) { + if (line.startsWith("@type ")) return line.substring("@type ".length()).trim(); + } + return null; + } + + /** The Any corpus resolves exactly one URL, as the full tier's resolver does. */ + private static final SimplePxfRegistry ANY_TYPES = new SimplePxfRegistry(); + static { + ANY_TYPES.register(meta("any.v1.Leaf")); + } + + private static byte[] decode(String dir, byte[] doc, String type) { + PxfMeta m = meta(type); + return dir.equals("any") + ? LiteWireWriter.encode(Parser.parse(doc), m, ANY_TYPES) + : LiteWireWriter.encode(Parser.parse(doc), m); + } + + private static String marshal(byte[] wire, String type) { + return Format.formatDocument(LiteWireReader.toAst(wire, meta(type))); + } + + private static void meet(String key, Executable verdict) throws Throwable { + String issue = KNOWN_GAPS.get(key); + if (issue == null) { + verdict.execute(); + return; + } + try { + verdict.execute(); + } catch (Throwable stillAGap) { + return; + } + fail(key + " meets its verdict now: drop it from KNOWN_GAPS (" + issue + ")"); + } + + // -- coverage -------------------------------------------------------------- + + @Test + void everyVendoredDirectoryHasADriver() throws IOException { + Set dirs; + try (Stream s = Files.list(ROOT)) { + dirs = s.filter(Files::isDirectory).map(p -> p.getFileName().toString()) + .collect(Collectors.toCollection(TreeSet::new)); + } + assertEquals(new TreeSet<>(DRIVEN), dirs); + } + + @Test + void knownGapsNameFixturesThatExist() { + KNOWN_GAPS.keySet().forEach(k -> assertTrue(Files.exists(ROOT.resolve(k.replace("#text", ""))), k)); + } + + // -- verdicts by file name --------------------------------------------------- + + @TestFactory + Stream everyFixtureMeetsItsVerdict() { + List out = new ArrayList<>(); + for (String dir : BY_NAME) { + for (String name : documents(dir)) { + String key = dir + "/" + name; + out.add(DynamicTest.dynamicTest(key, () -> meet(key, () -> { + byte[] doc = fixture(dir, name); + String type = typeOf(doc); + assertNotNull(type, key + " names no @type"); + if (rejects(name)) { + assertThrows(PxfException.class, () -> decode(dir, doc, type), key); + } else { + decode(dir, doc, type); + } + }))); + } + } + return out.stream(); + } + + /** Every accepted null leaves its field unset, so the message writes no bytes. */ + @Test + void acceptedNullsLeaveNoFieldSet() { + int n = 0; + for (String name : documents("nulls")) { + if (rejects(name)) continue; + byte[] doc = fixture("nulls", name); + assertEquals(0, decode("nulls", doc, typeOf(doc)).length, name); + n++; + } + assertEquals(8, n); + } + + // -- the error text the full tier pins --------------------------------------- + + @Test + void duplicateErrorsNameTheField() throws Throwable { + Map want = Map.of( + "err-scalar-twice.pxf", "field \"s\" is already set", + "err-message-twice.pxf", "field \"m\" is already set", + "err-null-then-value.pxf", "field \"w\" is already set", + "err-nested-scalar-twice.pxf", "field \"a\" is already set", + "err-oneof-member-twice.pxf", "conflicts with already-set field \"km\"", + "err-map-key-twice-int-spellings.pxf", "map field \"nums\": key 1 is already set", + "err-map-key-twice-in-literal.pxf", "map field \"labels\": key \"a\" is already set"); + for (Map.Entry w : want.entrySet()) { + String name = w.getKey(); + meet("duplicates/" + name + "#text", () -> { + PxfException e = assertThrows(PxfException.class, + () -> decode("duplicates", fixture("duplicates", name), "dup.v1.Holder"), name); + assertTrue(e.getMessage().contains(w.getValue()), name + " -> " + e.getMessage()); + }); + } + } + + @Test + void invalidBoolMapKeysNameTheKeyAndTheField() throws Throwable { + meet("map-keys/invalid#text", LiteCorpusTest::invalidBoolMapKeyTexts); + } + + private static void invalidBoolMapKeyTexts() { + for (String name : documents("map-keys")) { + if (!name.startsWith("invalid/")) continue; + String stem = name.substring("invalid/".length(), name.length() - ".pxf".length()); + String key = stem.substring(stem.lastIndexOf('-') + 1); + PxfException e = assertThrows(PxfException.class, + () -> decode("map-keys", fixture("map-keys", name), "mapkeys.v1.Flags"), name); + assertTrue(e.getMessage().contains("invalid bool map key"), name + " -> " + e.getMessage()); + assertTrue(e.getMessage().contains(key), name + " must name the key: " + e.getMessage()); + assertTrue(e.getMessage().contains("\"by_flag\""), name + " must name the field: " + e.getMessage()); + } + } + + @Test + void aRawNewlineIsRejectedByTheParser() { + PxfException e = assertThrows(PxfException.class, + () -> Parser.parse(fixture("strings", "err-raw-newline.pxf"))); + assertTrue(e.getMessage().contains("raw newline"), e.getMessage()); + } + + // -- bytes and text the corpus gives ------------------------------------------ + + @Test + void annotations() throws Throwable { + String hex = text("annotations", "ok.expected.hex").trim(); + assertEquals(hex, HexFormat.of().formatHex(decode("annotations", fixture("annotations", "ok.pxf"), + "settings.v1.Settings")), "ok.pxf"); + meet("annotations/missing-required.pxf", () -> assertThrows(PxfException.class, () -> decode("annotations", + fixture("annotations", "missing-required.pxf"), "settings.v1.Settings"), "missing-required.pxf")); + } + + private static String lastLine(String doc) { + String[] lines = doc.strip().split("\n"); + return lines[lines.length - 1]; + } + + @Test + void bigfloat() { + int enc = 0, dec = 0, err = 0; + for (String name : documents("bigfloat")) { + String stem = name.substring(0, name.length() - ".pxf".length()); + byte[] doc = fixture("bigfloat", name); + if (stem.startsWith("enc-")) { + enc++; + assertEquals(lastLine(new String(doc, StandardCharsets.UTF_8)), + marshal(fixture("bigfloat", stem + ".binpb"), "bf.v1.H").strip(), stem); + } else if (stem.startsWith("dec-")) { + dec++; + assertEquals(HexFormat.of().formatHex(fixture("bigfloat", stem + ".binpb")), + HexFormat.of().formatHex(decode("bigfloat", doc, "bf.v1.H")), stem); + } else { + err++; + assertThrows(PxfException.class, () -> decode("bigfloat", doc, "bf.v1.H"), stem); + } + } + assertEquals(List.of(13, 18, 4), List.of(enc, dec, err)); + } + + @Test + void durationsBindWithOneSign() throws Exception { + record Want(String file, long seconds, int nanos) {} + for (Want w : List.of( + new Want("negative-seconds.pxf", -30, 0), + new Want("negative-fractional.pxf", -1, -500_000_000), + new Want("negative-multi-segment.pxf", -5400, 0), + new Want("negative-zero.pxf", 0, 0))) { + com.google.protobuf.Duration d = duration.v1.Holder.parseFrom( + decode("duration", fixture("duration", w.file()), "duration.v1.Holder")).getD(); + assertEquals(w.seconds(), d.getSeconds(), w.file()); + assertEquals(w.nanos(), d.getNanos(), w.file()); + } + } + + /** What a marshaller writes for a pair's input: the expected file's body, after its {@code @type}. */ + private static String body(String doc) { + int i = doc.indexOf("\n\n"); + return doc.startsWith("@type ") && i >= 0 ? doc.substring(i + 2) : doc; + } + + @Test + void aNegativeDurationReEncodesWithOneLeadingSign() { + byte[] wire = decode("duration", fixture("duration", "roundtrip-negative.pxf"), "duration.v1.Holder"); + assertEquals(body(text("duration", "roundtrip-negative.expected.pxf")), marshal(wire, "duration.v1.Holder")); + } + + @Test + void listBindingsConcatenate() throws Exception { + test.v1.AllTypes m = test.v1.AllTypes.parseFrom( + decode("repeated", fixture("repeated", "concat-lists.pxf"), "test.v1.AllTypes")); + assertEquals(List.of("a", "b", "c"), m.getRepeatedStringList()); + assertEquals(List.of("n1", "n2"), + m.getRepeatedNestedList().stream().map(n -> n.getName()).collect(Collectors.toList())); + } + + /** The schema-less formatter is pxf-runtime's, shared by both tiers. */ + @Test + void mapKeyFormatPairs() { + for (String pair : List.of("fmt-bare-keys", "fmt-dotted-keys", "fmt-keyword-keys")) { + String expected = text("map-keys", pair + ".expected.pxf"); + assertEquals(expected, Format.formatDocument(Parser.parse(fixture("map-keys", pair + ".pxf"))), pair); + assertEquals(expected, Format.formatDocument(Parser.parse(expected.getBytes(StandardCharsets.UTF_8))), + pair + ": expected is a fixed point"); + } + } + + // -- @dataset ------------------------------------------------------------------ + + /** Rejected by the header or a row's syntax: pxf-runtime's parser rejects them. Same table as pxf's DatasetCorpusTest. */ + private static final Set DATASET_PARSE_REJECTS = Set.of( + "arity-long", "arity-short", "with-body", "with-type", "truncated-row-eof", + "truncated-row-then-rows", "junk-between-rows", "string-newline-in-cell", + "duplicate-column", "dotted-overlap", "dotted-overlap-deep", "dotted-empty-segment", "dotted-trailing-dot"); + + /** + * Decided when a row binds to its type. The lite tier binds no row (#212), + * so these parse and are not checked further. + */ + private static final Set DATASET_BOUND = Set.of( + "block-on-scalar", "null-on-scalar-cell", "dotted-oneof-two-members", "dotted-through-repeated", + "dotted-through-map", "dotted-through-scalar", "dotted-into-timestamp", "dotted-into-wrapper", + "dotted-into-duration", "dotted-into-bignum", "dotted-unknown-segment", "dotted-required-missing", + "dotted-bad-header-empty-row", "null-on-required-cell", + "empty-rows", "rows-end-at-directive", "list-cell", "block-cell", "nested-cells", "cell-framing", + "null-on-optional-cell", "dotted-column", "dotted-grouping", "dotted-null-leaf", "dotted-oneof", + "dotted-required", "dotted-with-value-cells", "dotted-into-any", "dotted-into-secret", + "dotted-bad-header-no-rows"); + + @Test + void datasetParseVerdicts() { + Set all = new TreeSet<>(); + for (String name : documents("dataset")) { + String stem = name.substring(0, name.length() - ".pxf".length()); + all.add(stem); + byte[] doc = fixture("dataset", name); + if (DATASET_PARSE_REJECTS.contains(stem)) { + assertThrows(PxfException.class, () -> Parser.parse(doc), stem); + } else { + assertTrue(DATASET_BOUND.contains(stem), stem + " has no verdict here"); + assertNotNull(Parser.parse(doc), stem); + } + } + Set known = new TreeSet<>(DATASET_PARSE_REJECTS); + known.addAll(DATASET_BOUND); + assertEquals(known, all); + } +} diff --git a/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteTablesTest.java b/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteTablesTest.java index a38b66a..f05c458 100644 --- a/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteTablesTest.java +++ b/lite-fixtures/src/test/java/org/protowire/lite/fixtures/LiteTablesTest.java @@ -6,71 +6,15 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.function.Function; -import nulls.ed.v1.EdPxfCodec; -import nulls.v1.DefaultedPxfCodec; -import nulls.v1.HolderPxfCodec; -import nulls.v1.LegacyPxfCodec; -import nulls.v1.RequiredPxfCodec; -import nulls.v1.StrictPxfCodec; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import java.util.stream.Stream; /** - * Tables only the plugin can fill, proved through generated companions: - * protowire's {@code testdata/nulls/} verdicts, whose documents are inlined by - * file name, and {@code testdata/annotations/} in the annotation form. + * {@code testdata/annotations/} in the annotation form, which only the plugin + * can turn into tables: {@code settings_annotated.proto} needs a v1.2 + * compiler, so {@code src/main/proto} spells it for stock protoc. The rest of + * the spec corpus runs from its files in {@link LiteCorpusTest}. */ class LiteTablesTest { - private static Arguments row(String file, Function codec, String doc) { - return Arguments.of(file, codec, doc); - } - - static Stream accepted() { - Function holder = HolderPxfCodec::unmarshal; - return Stream.of( - row("message.pxf", holder, "msg = null"), - row("wrapper.pxf", holder, "w = null"), - row("optional-scalar.pxf", holder, "o = null"), - row("oneof-scalar.pxf", holder, "ks = null"), - row("oneof-message.pxf", holder, "km = null"), - row("proto2-scalar.pxf", LegacyPxfCodec::unmarshal, "p = null"), - row("required-optional.pxf", RequiredPxfCodec::unmarshal, "r = null"), - row("default-optional.pxf", DefaultedPxfCodec::unmarshal, "d = null"), - row("editions-explicit.pxf", EdPxfCodec::unmarshal, "explicit_scalar = null")); - } - - static Stream rejected() { - Function holder = HolderPxfCodec::unmarshal; - return Stream.of( - row("err-scalar.pxf", holder, "n = null"), - row("err-repeated.pxf", holder, "l = null"), - row("err-list-element.pxf", holder, "l = [1, null]"), - row("err-map.pxf", holder, "m = null"), - row("err-map-value.pxf", holder, "m = {\n \"a\": null\n}"), - row("err-map-message-value.pxf", holder, "mm = {\n \"a\": null\n}"), - row("err-proto2-required.pxf", StrictPxfCodec::unmarshal, "r = null"), - row("err-nested-scalar.pxf", holder, "msg { x = null }"), - row("err-oneof-sibling.pxf", holder, "km { x = 1 }\nks = null"), - row("err-editions-implicit.pxf", EdPxfCodec::unmarshal, "implicit_scalar = null")); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("accepted") - void nullFixtureIsAccepted(String file, Function codec, String doc) { - // Every accept fixture leaves no field set. - assertEquals(0, codec.apply(doc).toByteArray().length, file); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("rejected") - void nullFixtureIsRejected(String file, Function codec, String doc) { - assertThrows(RuntimeException.class, () -> codec.apply(doc), file); - } - @Test void annotationForm_takesItsDefaults() { assertEquals(FixtureModesTest.OK_EXPECTED_HEX, FixtureModes.hex( diff --git a/pxf-android/build.gradle.kts b/pxf-android/build.gradle.kts index 9783a11..6942e58 100644 --- a/pxf-android/build.gradle.kts +++ b/pxf-android/build.gradle.kts @@ -14,13 +14,14 @@ plugins { // descriptor-typed import sneaks into a source file, the test task fails to // compile. That's the contract enforcement. -// The keyed conformance fixtures are shared with :pxf, which vendors them from -// the spec repo's testdata/keyed/, so both tiers read the same files. +// The keyed conformance fixtures are the repository's testdata/keyed/, the +// spec's testdata/keyed/ byte for byte, which both tiers read. sourceSets { test { resources { - srcDir("../pxf/src/test/resources") + srcDir(rootDir.resolve("testdata")) include("keyed/**") + exclude("**/*.proto") } } } diff --git a/pxf/build.gradle.kts b/pxf/build.gradle.kts index 7351bd0..cf09f6a 100644 --- a/pxf/build.gradle.kts +++ b/pxf/build.gradle.kts @@ -24,3 +24,26 @@ protobuf { // `src/test/proto` is the protobuf plugin's default — registering it // explicitly via sourceSets adds it twice, which Gradle 9 rejects with // a duplicate-entry error in `processTestProtoResources`. + +// The spec corpus, one copy for both tiers: ../testdata/ is protowire's +// testdata/ byte for byte (the conformance job diffs it against spec +// main). Its schemas compile here, and its fixtures are test resources at +// //. Two schemas stay out: settings_annotated.proto needs a v1.2 +// compiler, and protoc 3.25.5 reads nulls_editions.proto only behind +// --experimental_editions, so the test loads a checked-in descriptor set. +sourceSets { + test { + proto { + srcDir(rootDir.resolve("testdata")) + exclude("annotations/settings_annotated.proto", "nulls/nulls_editions.proto", "test.proto") + } + resources { + srcDir(rootDir.resolve("testdata")) + exclude("**/*.proto") + } + } +} + +tasks.test { + systemProperty("protowire.testdata", rootDir.resolve("testdata").absolutePath) +} diff --git a/pxf/src/test/java/org/protowire/pxf/AnyCorpusTest.java b/pxf/src/test/java/org/protowire/pxf/AnyCorpusTest.java index f27019d..a174a27 100644 --- a/pxf/src/test/java/org/protowire/pxf/AnyCorpusTest.java +++ b/pxf/src/test/java/org/protowire/pxf/AnyCorpusTest.java @@ -22,7 +22,7 @@ /** * Drives the spec repo's {@code testdata/any/} corpus (#100), vendored under - * {@code src/test/resources/corpus/any/} from trendvidia/protowire 234cf29, + * {@code testdata/any/} and kept equal to the spec's by the conformance job, * through the schema-less parse and a decoder bound to {@code any.v1.Holder} * with a resolver that maps a URL to the message named after its last * {@code /}. The verdict table is the corpus README's. diff --git a/pxf/src/test/java/org/protowire/pxf/BigFloatCorpusTest.java b/pxf/src/test/java/org/protowire/pxf/BigFloatCorpusTest.java index f7c7d41..f148268 100644 --- a/pxf/src/test/java/org/protowire/pxf/BigFloatCorpusTest.java +++ b/pxf/src/test/java/org/protowire/pxf/BigFloatCorpusTest.java @@ -20,14 +20,14 @@ /** * Drives the spec repo's {@code testdata/bigfloat/} corpus (#104), vendored - * under {@code src/test/resources/corpus/bigfloat/} from - * trendvidia/protowire 234cf29. Its expected values come from exact rational + * under {@code testdata/bigfloat/} and kept equal to the spec's by the + * conformance job. Its expected values come from exact rational * arithmetic in {@code generate.py}, independently of any port. */ class BigFloatCorpusTest { private static byte[] binpb(String name) throws IOException { - try (InputStream in = BigFloatCorpusTest.class.getResourceAsStream("/corpus/bigfloat/" + name + ".binpb")) { + try (InputStream in = BigFloatCorpusTest.class.getResourceAsStream("/bigfloat/" + name + ".binpb")) { assertNotNull(in, "missing " + name + ".binpb"); return in.readAllBytes(); } diff --git a/pxf/src/test/java/org/protowire/pxf/CorpusFixtures.java b/pxf/src/test/java/org/protowire/pxf/CorpusFixtures.java index 38d94c0..95e7c71 100644 --- a/pxf/src/test/java/org/protowire/pxf/CorpusFixtures.java +++ b/pxf/src/test/java/org/protowire/pxf/CorpusFixtures.java @@ -2,20 +2,23 @@ // Copyright (c) 2026 TrendVidia, LLC. package org.protowire.pxf; +import org.junit.jupiter.api.function.Executable; + import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** - * Lists the {@code .pxf} files of a corpus vendored under - * {@code src/test/resources/corpus/} from trendvidia/protowire {@code testdata/}. A + * Lists the {@code .pxf} files of a corpus vendored under the repository's + * {@code testdata/}, protowire's {@code testdata/} byte for byte. A * driver walks the directory rather than a list of names, so a fixture added * by the next re-vendor runs without a test change. The verdict is the file * name: the corpora name every reject fixture {@code err-*}. @@ -23,19 +26,50 @@ final class CorpusFixtures { private CorpusFixtures() {} + /** + * The corpus directory on disk. Not a classpath lookup: {@code /any}, + * {@code /nulls} and the rest are also the generated packages' class + * directories, and a directory lookup finds those first. + */ + static Path dir(String dir) { + String base = System.getProperty("protowire.testdata"); + assertNotNull(base, "protowire.testdata is unset; run through Gradle"); + Path root = Path.of(base, dir); + assertTrue(Files.isDirectory(root), "missing vendored corpus " + root); + return root; + } + + /** + * Fixtures the full tier does not meet yet, each against the open issue + * that fixes it, keyed {@code /}. {@link #meet} fails on a + * listed fixture that meets its verdict, so the list only shrinks. + */ + static final Map KNOWN_GAPS = Map.of( + "nulls/err-required-optional.pxf", "#189", + "nulls/err-required-wrapper.pxf", "#189", + "dataset/null-on-required-cell.pxf", "#189"); + + /** Runs a fixture's verdict, or for a known gap checks that it still fails. */ + static void meet(String dir, String name, Executable verdict) throws Throwable { + String issue = KNOWN_GAPS.get(dir + "/" + name); + if (issue == null) { + verdict.execute(); + return; + } + try { + verdict.execute(); + } catch (Throwable stillAGap) { + return; + } + fail(dir + "/" + name + " meets its verdict now: drop it from KNOWN_GAPS (" + issue + ")"); + } + record Fixture(String name, byte[] data) { boolean rejects() { return name.startsWith("err-"); } } static List load(String dir) throws IOException { - URL url = CorpusFixtures.class.getResource("/corpus/" + dir); - assertNotNull(url, "missing vendored corpus " + dir); - Path root; - try { - root = Path.of(url.toURI()); - } catch (URISyntaxException e) { - throw new IOException(e); - } + Path root = dir(dir); try (Stream files = Files.list(root)) { List out = files .filter(p -> p.getFileName().toString().endsWith(".pxf")) diff --git a/pxf/src/test/java/org/protowire/pxf/DatasetCorpusTest.java b/pxf/src/test/java/org/protowire/pxf/DatasetCorpusTest.java index bcc98fa..f87443f 100644 --- a/pxf/src/test/java/org/protowire/pxf/DatasetCorpusTest.java +++ b/pxf/src/test/java/org/protowire/pxf/DatasetCorpusTest.java @@ -4,6 +4,7 @@ import com.google.protobuf.Message; import dataset.v1.Paths.Event; +import dataset.v1.Paths.RowReq; import org.junit.jupiter.api.Test; import org.protowire.pxf.testproto.AllTypes; @@ -27,8 +28,8 @@ /** * Drives the spec repo's {@code testdata/dataset/} corpus (#99, #101, #102), - * vendored under {@code src/test/resources/corpus/dataset/} from - * trendvidia/protowire 234cf29, on every dataset path: {@code Pxf.parse} plus + * vendored under {@code testdata/dataset/} and kept equal to the spec's by + * the conformance job, on every dataset path: {@code Pxf.parse} plus * {@code BindRow}, {@code DatasetReader}, and the materializing decoder. Each * fixture's leading comment and the corpus README state its verdict; this * table is that statement, and {@link #everyFixtureHasAVerdict} fails if a @@ -47,7 +48,7 @@ class DatasetCorpusTest { "block-on-scalar", "null-on-scalar-cell", "dotted-oneof-two-members", "dotted-through-repeated", "dotted-through-map", "dotted-through-scalar", "dotted-into-timestamp", "dotted-into-wrapper", "dotted-into-duration", "dotted-into-bignum", "dotted-unknown-segment", "dotted-required-missing", - "dotted-bad-header-empty-row"); + "dotted-bad-header-empty-row", "null-on-required-cell"); private static final Set ACCEPTS = Set.of( "empty-rows", "rows-end-at-directive", "list-cell", "block-cell", "nested-cells", "cell-framing", @@ -60,8 +61,10 @@ private static String stem(CorpusFixtures.Fixture f) { } private static Supplier rowType(byte[] doc) { - return new String(doc, StandardCharsets.UTF_8).contains("@dataset dataset.v1.Event") - ? Event::newBuilder : AllTypes::newBuilder; + String text = new String(doc, StandardCharsets.UTF_8); + if (text.contains("@dataset dataset.v1.Event")) return Event::newBuilder; + if (text.contains("@dataset dataset.v1.RowReq")) return RowReq::newBuilder; + return AllTypes::newBuilder; } private static List viaParse(byte[] doc) { @@ -110,26 +113,30 @@ void everyFixtureHasAVerdict() throws IOException { known.addAll(BIND_REJECTS); known.addAll(ACCEPTS); assertEquals(known, all); - assertEquals(42, all.size()); + assertEquals(43, all.size()); } @Test - void everyPathMeetsEveryVerdict() throws IOException { + void everyPathMeetsEveryVerdict() throws Throwable { for (CorpusFixtures.Fixture f : CorpusFixtures.load("dataset")) { - String name = stem(f); - if (PARSE_REJECTS.contains(name)) { - assertThrows(PxfException.class, () -> Pxf.parse(f.data()), name + " (parse)"); - assertThrows(RuntimeException.class, () -> viaReader(f.data()), name + " (reader)"); - assertThrows(PxfException.class, - () -> Pxf.unmarshalFull(f.data(), rowType(f.data()).get()), name + " (decoder)"); - } else if (BIND_REJECTS.contains(name)) { - Pxf.parse(f.data()); - assertThrows(PxfException.class, () -> viaParse(f.data()), name + " (parse + bindRow)"); - assertThrows(PxfException.class, () -> viaReader(f.data()), name + " (reader)"); - } else { - assertEquals(viaParse(f.data()), viaReader(f.data()), name); - Pxf.unmarshalFull(f.data(), rowType(f.data()).get()); - } + CorpusFixtures.meet("dataset", f.name(), () -> meetsItsVerdict(f)); + } + } + + private static void meetsItsVerdict(CorpusFixtures.Fixture f) throws IOException { + String name = stem(f); + if (PARSE_REJECTS.contains(name)) { + assertThrows(PxfException.class, () -> Pxf.parse(f.data()), name + " (parse)"); + assertThrows(RuntimeException.class, () -> viaReader(f.data()), name + " (reader)"); + assertThrows(PxfException.class, + () -> Pxf.unmarshalFull(f.data(), rowType(f.data()).get()), name + " (decoder)"); + } else if (BIND_REJECTS.contains(name)) { + Pxf.parse(f.data()); + assertThrows(PxfException.class, () -> viaParse(f.data()), name + " (parse + bindRow)"); + assertThrows(PxfException.class, () -> viaReader(f.data()), name + " (reader)"); + } else { + assertEquals(viaParse(f.data()), viaReader(f.data()), name); + Pxf.unmarshalFull(f.data(), rowType(f.data()).get()); } } diff --git a/pxf/src/test/java/org/protowire/pxf/KeyedFixtureTest.java b/pxf/src/test/java/org/protowire/pxf/KeyedFixtureTest.java index 28e4f06..ad07969 100644 --- a/pxf/src/test/java/org/protowire/pxf/KeyedFixtureTest.java +++ b/pxf/src/test/java/org/protowire/pxf/KeyedFixtureTest.java @@ -25,10 +25,10 @@ /** * Keyed repeated fields (draft -01 §3.13, protowire#116, #50): the spec - * repo's {@code testdata/keyed/} fixtures (vendored under - * {@code src/test/resources/keyed/} from trendvidia/protowire, with - * {@code keyed.proto} compiled beside the other test protos; the README - * there states the verdicts), plus the edge cases protowire-go's + * repo's {@code testdata/keyed/} fixtures (the repository's {@code testdata/keyed/}, + * kept equal to the spec's by the conformance job, with {@code keyed.proto} + * compiled beside the other test protos; the README there states the + * verdicts), plus the edge cases protowire-go's * {@code keyed_test.go} pins. */ class KeyedFixtureTest { diff --git a/pxf/src/test/java/org/protowire/pxf/MapKeysFixtureTest.java b/pxf/src/test/java/org/protowire/pxf/MapKeysFixtureTest.java index 335312b..485db81 100644 --- a/pxf/src/test/java/org/protowire/pxf/MapKeysFixtureTest.java +++ b/pxf/src/test/java/org/protowire/pxf/MapKeysFixtureTest.java @@ -18,11 +18,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Drives the spec repo's {@code testdata/map-keys/} fixtures (vendored - * under {@code src/test/resources/map-keys/} from trendvidia/protowire - * a0dd520, 2026-09-08, the {@code fmt-dotted-keys} pair from its branch - * {@code map-keys-dotted-313} at f7d467f; the README there states the - * verdicts). Bool keys (#76): three documents MUST bind to the keys true + * Drives the spec repo's {@code testdata/map-keys/} fixtures (the + * repository's {@code testdata/map-keys/}, kept equal to the spec's by the + * conformance job; the README there states the verdicts). Bool keys (#76): three documents MUST bind to the keys true * and false; every file under {@code invalid/} MUST NOT bind, with an * error naming the key. Key spelling (#82, #83): each {@code fmt-*} input * formats to exactly its {@code .expected.pxf}, the expected file is a diff --git a/pxf/src/test/java/org/protowire/pxf/RejectRulesCorpusTest.java b/pxf/src/test/java/org/protowire/pxf/RejectRulesCorpusTest.java index c41c3ac..970e1bb 100644 --- a/pxf/src/test/java/org/protowire/pxf/RejectRulesCorpusTest.java +++ b/pxf/src/test/java/org/protowire/pxf/RejectRulesCorpusTest.java @@ -30,8 +30,8 @@ /** * Drives the spec repo's {@code testdata/{blocks,duplicates,nulls,strings}/} - * corpora, vendored under {@code src/test/resources/corpus/} from - * trendvidia/protowire 234cf29, through this port's decoder on both presence + * corpora, vendored under {@code testdata/} and kept equal to the spec's by + * the conformance job, through this port's decoder on both presence * paths and through the schema-less parse. Each corpus's README states the * verdicts. */ @@ -56,19 +56,28 @@ private static void assertVerdict(CorpusFixtures.Fixture f, Descriptor d) { } } - private static int run(String dir, Function schema) throws IOException { + private static int run(String dir, Function schema) throws Throwable { List all = CorpusFixtures.load(dir); for (CorpusFixtures.Fixture f : all) { Descriptor d = schema.apply(f); - if (d != null) assertVerdict(f, d); + if (d != null) CorpusFixtures.meet(dir, f.name(), () -> assertVerdict(f, d)); } return all.size(); } + @Test + void knownGapsNameFixturesThatExist() { + CorpusFixtures.KNOWN_GAPS.keySet().forEach(k -> { + int slash = k.indexOf('/'); + assertTrue(java.nio.file.Files.isRegularFile( + CorpusFixtures.dir(k.substring(0, slash)).resolve(k.substring(slash + 1))), k); + }); + } + // -- blocks, #106 -------------------------------------------------------- @Test - void blocks() throws IOException { + void blocks() throws Throwable { assertEquals(7, run("blocks", f -> Doc.getDescriptor())); } @@ -97,7 +106,7 @@ void blocksBindTheirValues() throws IOException { // -- duplicates, #103 and #105 ------------------------------------------- @Test - void duplicates() throws IOException { + void duplicates() throws Throwable { assertEquals(19, run("duplicates", f -> Holder.getDescriptor())); } @@ -148,7 +157,8 @@ void acceptedDuplicatesKeepEveryBinding() throws IOException { private static final Map NULL_SCHEMAS = Map.of( "err-proto2-required.pxf", NullsProto2.Strict.getDescriptor(), "proto2-scalar.pxf", NullsProto2.Legacy.getDescriptor(), - "required-optional.pxf", Nulls.Required.getDescriptor(), + "err-required-optional.pxf", Nulls.Required.getDescriptor(), + "err-required-wrapper.pxf", Nulls.RequiredWrapper.getDescriptor(), "default-optional.pxf", Nulls.Defaulted.getDescriptor()); /** {@code nulls.ed.v1.Ed}, loaded from a descriptor set: see {@link #nullsEditions}. */ @@ -157,8 +167,8 @@ private static boolean editions(CorpusFixtures.Fixture f) { } @Test - void nulls() throws IOException { - assertEquals(19, run("nulls", f -> editions(f) ? null + void nulls() throws Throwable { + assertEquals(20, run("nulls", f -> editions(f) ? null : NULL_SCHEMAS.getOrDefault(f.name(), Nulls.Holder.getDescriptor()))); } @@ -174,7 +184,7 @@ void acceptedNullsLeaveTheFieldUnsetAndPresent() throws IOException { Map field = Map.of( "message.pxf", "msg", "wrapper.pxf", "w", "optional-scalar.pxf", "o", "oneof-scalar.pxf", "ks", "oneof-message.pxf", "km", "proto2-scalar.pxf", "p", - "required-optional.pxf", "r", "default-optional.pxf", "d"); + "default-optional.pxf", "d"); Map by = byName("nulls"); field.forEach((name, fd) -> { Descriptor d = NULL_SCHEMAS.getOrDefault(name, Nulls.Holder.getDescriptor()); @@ -196,7 +206,7 @@ void acceptedNullsLeaveTheFieldUnsetAndPresent() throws IOException { @Test void nullsEditions() throws Exception { FileDescriptorProto proto; - try (InputStream in = getClass().getResourceAsStream("/corpus/nulls/nulls_editions.binpb")) { + try (InputStream in = getClass().getResourceAsStream("/editions/nulls_editions.binpb")) { assertNotNull(in, "missing nulls_editions.binpb"); proto = FileDescriptorSet.parseFrom(in).getFile(0); } @@ -212,7 +222,7 @@ void nullsEditions() throws Exception { // -- strings, #102 ------------------------------------------------------- @Test - void strings() throws IOException { + void strings() throws Throwable { assertEquals(4, run("strings", f -> Strings.Holder.getDescriptor())); } diff --git a/pxf/src/test/java/org/protowire/pxf/SpecCorpusTest.java b/pxf/src/test/java/org/protowire/pxf/SpecCorpusTest.java new file mode 100644 index 0000000..823a9b1 --- /dev/null +++ b/pxf/src/test/java/org/protowire/pxf/SpecCorpusTest.java @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 TrendVidia, LLC. +package org.protowire.pxf; + +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import duration.v1.Duration.Holder; +import org.junit.jupiter.api.Test; +import org.protowire.pxf.testproto.AllTypes; +import settings.v1.SettingsOuterClass.Settings; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The spec corpus directories under {@code testdata/} that no other test + * here drives ({@code annotations}, {@code duration}, {@code repeated}), and + * the check that every vendored directory has a driver. + */ +class SpecCorpusTest { + + /** Every directory this tier runs, and the test class that runs it. */ + private static final Set DRIVEN = Set.of( + "annotations", // here + "any", // AnyCorpusTest + "bigfloat", // BigFloatCorpusTest + "blocks", // RejectRulesCorpusTest + "dataset", // DatasetCorpusTest, RejectRulesCorpusTest + "duplicates", // RejectRulesCorpusTest + "duration", // here + "keyed", // KeyedFixtureTest + "map-keys", // MapKeysFixtureTest + "nulls", // RejectRulesCorpusTest + "repeated", // here + "strings"); // RejectRulesCorpusTest + + @Test + void everyVendoredDirectoryHasADriver() throws IOException { + Path root = CorpusFixtures.dir(""); + Set dirs; + try (Stream s = Files.list(root)) { + dirs = s.filter(Files::isDirectory).map(p -> p.getFileName().toString()) + .collect(Collectors.toCollection(TreeSet::new)); + } + assertEquals(new TreeSet<>(DRIVEN), dirs); + } + + private static void verdicts(String dir, Descriptor d) throws Throwable { + for (CorpusFixtures.Fixture f : CorpusFixtures.load(dir)) { + if (f.name().endsWith(".expected.pxf")) continue; + CorpusFixtures.meet(dir, f.name(), () -> { + if (f.rejects()) { + assertThrows(PxfException.class, () -> Pxf.unmarshal(f.data(), d), f.name()); + assertThrows(PxfException.class, + () -> Pxf.unmarshalFull(f.data(), DynamicMessage.newBuilder(d)), f.name()); + } else { + Pxf.unmarshal(f.data(), d); + Pxf.unmarshalFull(f.data(), DynamicMessage.newBuilder(d)); + } + }); + } + } + + private static byte[] fixture(String dir, String name) throws IOException { + return Files.readAllBytes(CorpusFixtures.dir(dir).resolve(name)); + } + + private static B bind(String dir, String name, B b) throws IOException { + Pxf.unmarshal(fixture(dir, name), b); + return b; + } + + // -- annotations: (pxf.required) = 1314, (pxf.default) = 1315 ------------ + + @Test + void annotations() throws Throwable { + Settings.Builder b = Settings.newBuilder(); + Pxf.unmarshalFull(fixture("annotations", "ok.pxf"), b); + String hex = new String(fixture("annotations", "ok.expected.hex"), StandardCharsets.US_ASCII).trim(); + assertArrayEquals(HexFormat.of().parseHex(hex), b.build().toByteArray(), "ok.pxf"); + + assertThrows(PxfException.class, + () -> Pxf.unmarshalFull(fixture("annotations", "missing-required.pxf"), Settings.newBuilder())); + } + + // -- duration: signed literals ------------------------------------------ + + @Test + void duration() throws Throwable { + verdicts("duration", Holder.getDescriptor()); + } + + @Test + void durationsBindWithOneSign() throws IOException { + record Want(String file, long seconds, int nanos) {} + for (Want w : List.of( + new Want("negative-seconds.pxf", -30, 0), + new Want("negative-fractional.pxf", -1, -500_000_000), + new Want("negative-multi-segment.pxf", -5400, 0), + new Want("negative-zero.pxf", 0, 0))) { + com.google.protobuf.Duration d = bind("duration", w.file(), Holder.newBuilder()).getD(); + assertEquals(w.seconds(), d.getSeconds(), w.file()); + assertEquals(w.nanos(), d.getNanos(), w.file()); + } + } + + @Test + void aNegativeDurationReEncodesWithOneLeadingSign() throws IOException { + String expected = new String(fixture("duration", "roundtrip-negative.expected.pxf"), StandardCharsets.UTF_8); + Holder h = bind("duration", "roundtrip-negative.pxf", Holder.newBuilder()).build(); + String out = new String(MarshalOptions.defaults().withTypeUrl("duration.v1.Holder").marshal(h), + StandardCharsets.UTF_8); + assertEquals(expected, out); + } + + // -- repeated: a repeated field is written with a list ------------------ + + @Test + void repeated() throws Throwable { + verdicts("repeated", AllTypes.getDescriptor()); + } + + @Test + void listBindingsConcatenate() throws IOException { + AllTypes m = bind("repeated", "concat-lists.pxf", AllTypes.newBuilder()).build(); + assertEquals(List.of("a", "b", "c"), m.getRepeatedStringList()); + assertEquals(List.of("n1", "n2"), m.getRepeatedNestedList().stream().map(n -> n.getName()).toList()); + } +} diff --git a/pxf/src/test/resources/corpus/nulls/required-optional.pxf b/pxf/src/test/resources/corpus/nulls/required-optional.pxf deleted file mode 100644 index 5cea532..0000000 --- a/pxf/src/test/resources/corpus/nulls/required-optional.pxf +++ /dev/null @@ -1,6 +0,0 @@ -@type nulls.v1.Required - -# Accept: null on an optional scalar marked (pxf.required). A null field -# is present for that check: r is unset, and present-but-null. - -r = null diff --git a/pxf/src/test/resources/corpus/nulls/nulls_editions.binpb b/pxf/src/test/resources/editions/nulls_editions.binpb similarity index 100% rename from pxf/src/test/resources/corpus/nulls/nulls_editions.binpb rename to pxf/src/test/resources/editions/nulls_editions.binpb diff --git a/scripts/check-spec-corpus.sh b/scripts/check-spec-corpus.sh new file mode 100755 index 0000000..4abf29b --- /dev/null +++ b/scripts/check-spec-corpus.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 TrendVidia, LLC. +# +# Checks testdata/ against the spec repo's testdata/: +# +# 1. every directory under testdata/ is the spec's directory byte for byte, +# and testdata/test.proto is the spec's; +# 2. every spec testdata directory is either vendored here (and so run by +# both tiers: pxf's SpecCorpusTest and lite-fixtures' LiteCorpusTest +# each fail on a directory they have no driver for) or listed below +# with the reason it is not. +# +# Usage: scripts/check-spec-corpus.sh +# The conformance job runs it against protowire main. + +set -euo pipefail + +spec="${1:?usage: $0 }/testdata" +here="$(cd "$(dirname "$0")/.." && pwd)/testdata" +[[ -d "$spec" ]] || { echo "no testdata/ in $1" >&2; exit 2; } + +# Spec directories with no copy here, and what covers them instead. +declare -A NOT_VENDORED=( + [adversarial]="the spec's cross_security_check.sh, run by this repo's conformance job" + [text]="the spec's cross_text_check.sh, run by this repo's conformance job" + [write]="the spec's cross_write_check.sh, run by this repo's conformance job" + [envelope]="the spec's cross_envelope_check.sh" + [values]="pb's PbGoldenTest, through a byte copy the conformance job compares" + [docs]="cmd/pxf tooling; neither Java tier has it" + [fmt]="cmd/pxf .proto formatting; neither Java tier has it" + [lint]="cmd/pxf lint; neither Java tier has it" + [openapi]="cmd/pxf openapi; neither Java tier has it" + [proto-imports]="the cmd/pxf .proto compiler; neither Java tier has it" + [schema-extensions]="the v1.2 .proto compiler; neither Java tier has it" + [stock-image]="the cmd/pxf .proto compiler; neither Java tier has it" +) + +fail=0 +for d in "$here"/*/; do + name="$(basename "$d")" + if [[ ! -d "$spec/$name" ]]; then + echo "testdata/$name: the spec has no testdata/$name" >&2 + fail=1 + elif ! diff -r "$spec/$name" "$here/$name"; then + echo "testdata/$name differs from the spec's: copy it again" >&2 + fail=1 + fi +done +if ! cmp "$spec/test.proto" "$here/test.proto"; then + echo "testdata/test.proto differs from the spec's" >&2 + fail=1 +fi + +for d in "$spec"/*/; do + name="$(basename "$d")" + [[ -d "$here/$name" || -n "${NOT_VENDORED[$name]:-}" ]] && continue + echo "spec testdata/$name is neither vendored in testdata/ nor listed in $0" >&2 + fail=1 +done + +if [[ "$fail" == 0 ]]; then + echo "testdata/: $(ls -d "$here"/*/ | wc -l | tr -d ' ') directories match the spec's" +fi +exit "$fail" diff --git a/testdata/annotations/README.md b/testdata/annotations/README.md new file mode 100644 index 0000000..17928cb --- /dev/null +++ b/testdata/annotations/README.md @@ -0,0 +1,54 @@ +# Annotation extension-number fixtures + +Cross-port conformance fixtures for the **PXF annotation extension +numbers** — `(pxf.required) = 1314` and `(pxf.default) = 1315` on +`google.protobuf.FieldOptions`, allocated in +[`proto/pxf/annotations.proto`](../../proto/pxf/annotations.proto) inside +the registered block [`STABILITY.md`](../../STABILITY.md) promise 3 +describes (issue #244). + +Unlike the other fixture directories, these are not consumed by each +port's own test suite. They are driven by +[`scripts/cross_envelope_check.sh`](../../scripts/cross_envelope_check.sh), +which hands every descriptor-driven port the **same compiled descriptor +set** and document through its `dump-envelope --pb` mode and compares the +bytes with the golden here. That is the difference from a port's unit +tests: those compile the port's *own* vendored `annotations.proto`, so a +port whose reader and copy both still say `50001` passes them and fails +here. + +All fixtures bind against [`settings.proto`](settings.proto) (package +`settings.v1`, message `Settings`), compiled with its imports into +[`settings.binpb`](settings.binpb) by + +``` +protoc -I testdata/annotations -I proto --include_imports \ + --descriptor_set_out=testdata/annotations/settings.binpb settings.proto +``` + +`cmd/pxf/fixture_numbers_test.go` pins that the compiled set carries +`1314` and `1315` and no retired number. + +| File | Asserts | +|---|---| +| `ok.pxf` | Only `name` is written; `retries`, `region` and `verbose` take their `(pxf.default)`. The pb bytes equal `ok.expected.hex`, which carries all four fields — a port that misses `1315` emits `name` alone. | +| `missing-required.pxf` | `name` is absent and carries `(pxf.required) = true`. A port must reject the document (`dump-envelope` exits 1); a port that misses `1314` accepts it. | + +[`settings_annotated.proto`](settings_annotated.proto) is the same schema in +the annotation form: `@required` and `@default(v)` lower only to the `1327` +carrier, so it needs a v1.2 compiler, and +[`settings_annotated.binpb`](settings_annotated.binpb) is that compiler's +image. `ok.pxf` and `missing-required.pxf` get the same verdicts against it. +The gate does not run it yet. It covers `protoc-gen-pxf-java-meta`, whose +tables must not depend on the spelling: the plugin's test compares the two +schemas' tables, and protowire-java's `lite-fixtures` runs both documents +through the generated code. + +`ok.expected.hex` was produced by the reference implementation +(protowire-go) and checked by decoding it back with +`protoc --decode=settings.v1.Settings`. + +The SBE numbers `1319`–`1323` are proved the same way by +[`../sbe-bench.pxf`](../sbe-bench.pxf) against +[`../sbe-bench.binpb`](../sbe-bench.binpb) through `dump-envelope --sbe`, +with the golden in [`../sbe-bench.expected.hex`](../sbe-bench.expected.hex). diff --git a/testdata/annotations/missing-required.pxf b/testdata/annotations/missing-required.pxf new file mode 100644 index 0000000..5e89e19 --- /dev/null +++ b/testdata/annotations/missing-required.pxf @@ -0,0 +1,4 @@ +// `name` carries (pxf.required) = true and is absent: a conformant port +// rejects this document at decode time. A port that does not read field +// number 1314 accepts it and emits bytes. +retries = 5 diff --git a/testdata/annotations/ok.expected.hex b/testdata/annotations/ok.expected.hex new file mode 100644 index 0000000..34fc25b --- /dev/null +++ b/testdata/annotations/ok.expected.hex @@ -0,0 +1 @@ +0a0373766310031a0975732d656173742d312001 diff --git a/testdata/annotations/ok.pxf b/testdata/annotations/ok.pxf new file mode 100644 index 0000000..6a25a25 --- /dev/null +++ b/testdata/annotations/ok.pxf @@ -0,0 +1,4 @@ +// Only the required field is written. Every other field is absent and takes +// its (pxf.default): retries = 3, region = "us-east-1", verbose = true. The +// pb bytes in ok.expected.hex carry all four fields. +name = "svc" diff --git a/testdata/annotations/settings.binpb b/testdata/annotations/settings.binpb new file mode 100644 index 0000000..392ff89 Binary files /dev/null and b/testdata/annotations/settings.binpb differ diff --git a/lite-fixtures/src/main/proto/settings.proto b/testdata/annotations/settings.proto similarity index 84% rename from lite-fixtures/src/main/proto/settings.proto rename to testdata/annotations/settings.proto index fe3420b..767ca27 100644 --- a/lite-fixtures/src/main/proto/settings.proto +++ b/testdata/annotations/settings.proto @@ -22,11 +22,6 @@ syntax = "proto3"; package settings.v1; -// Java options for this port's codegen: protoc-gen-pxf-java-meta requires an -// explicit java_package; the rest of the file is protowire's testdata/annotations/settings.proto verbatim. -option java_package = "settings.v1"; -option java_multiple_files = true; - import "pxf/annotations.proto"; message Settings { diff --git a/testdata/annotations/settings_annotated.binpb b/testdata/annotations/settings_annotated.binpb new file mode 100644 index 0000000..d2482cc Binary files /dev/null and b/testdata/annotations/settings_annotated.binpb differ diff --git a/testdata/annotations/settings_annotated.proto b/testdata/annotations/settings_annotated.proto new file mode 100644 index 0000000..80fb035 --- /dev/null +++ b/testdata/annotations/settings_annotated.proto @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 TrendVidia, LLC. +// +// settings.proto in the annotation form. @required and @default lower only to +// the 1327 carrier (RFC-001 §8.5), so a reader of 1314/1315 alone sees no +// annotation here and enforces nothing. It accepts the same documents as +// settings.proto, and ok.pxf gives the same bytes. +// +// It needs a v1.2 compiler. settings_annotated.binpb is +// `pxf build --null-mask=false --source-info=false +// -o testdata/annotations/settings_annotated.binpb +// testdata/annotations/settings_annotated.proto`; +// cmd/pxf/fixture_numbers_test.go pins that the image carries 1327 and that it +// matches this file. +syntax = "proto3"; + +package settings.annotated.v1; + +import "protowire/schema/v1/annotations.proto"; + +option java_multiple_files = true; +option java_package = "settings.annotated.v1"; + +message Settings { + string name = 1 @required; + int32 retries = 2 @default(3); + string region = 3 @default("us-east-1"); + bool verbose = 4 @default(true); +} diff --git a/pxf/src/test/resources/corpus/any/README.md b/testdata/any/README.md similarity index 100% rename from pxf/src/test/resources/corpus/any/README.md rename to testdata/any/README.md diff --git a/pxf/src/test/proto/any/any.proto b/testdata/any/any.proto similarity index 100% rename from pxf/src/test/proto/any/any.proto rename to testdata/any/any.proto diff --git a/pxf/src/test/resources/corpus/any/assignment.pxf b/testdata/any/assignment.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/assignment.pxf rename to testdata/any/assignment.pxf diff --git a/pxf/src/test/resources/corpus/any/block.pxf b/testdata/any/block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/block.pxf rename to testdata/any/block.pxf diff --git a/pxf/src/test/resources/corpus/any/err-not-any.pxf b/testdata/any/err-not-any.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/err-not-any.pxf rename to testdata/any/err-not-any.pxf diff --git a/pxf/src/test/resources/corpus/any/err-not-first.pxf b/testdata/any/err-not-first.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/err-not-first.pxf rename to testdata/any/err-not-first.pxf diff --git a/pxf/src/test/resources/corpus/any/err-twice.pxf b/testdata/any/err-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/err-twice.pxf rename to testdata/any/err-twice.pxf diff --git a/pxf/src/test/resources/corpus/any/err-unquoted-url.pxf b/testdata/any/err-unquoted-url.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/err-unquoted-url.pxf rename to testdata/any/err-unquoted-url.pxf diff --git a/pxf/src/test/resources/corpus/any/err-unresolved.pxf b/testdata/any/err-unresolved.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/err-unresolved.pxf rename to testdata/any/err-unresolved.pxf diff --git a/pxf/src/test/resources/corpus/any/list-element.pxf b/testdata/any/list-element.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/list-element.pxf rename to testdata/any/list-element.pxf diff --git a/pxf/src/test/resources/corpus/any/map-value.pxf b/testdata/any/map-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/map-value.pxf rename to testdata/any/map-value.pxf diff --git a/pxf/src/test/resources/corpus/any/plain-fields.pxf b/testdata/any/plain-fields.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/plain-fields.pxf rename to testdata/any/plain-fields.pxf diff --git a/pxf/src/test/resources/corpus/any/url-verbatim.pxf b/testdata/any/url-verbatim.pxf similarity index 100% rename from pxf/src/test/resources/corpus/any/url-verbatim.pxf rename to testdata/any/url-verbatim.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/README.md b/testdata/bigfloat/README.md similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/README.md rename to testdata/bigfloat/README.md diff --git a/pxf/src/test/proto/bigfloat/bigfloat.proto b/testdata/bigfloat/bigfloat.proto similarity index 100% rename from pxf/src/test/proto/bigfloat/bigfloat.proto rename to testdata/bigfloat/bigfloat.proto diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-1e4932.binpb b/testdata/bigfloat/dec-1e4932.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-1e4932.binpb rename to testdata/bigfloat/dec-1e4932.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-1e4932.pxf b/testdata/bigfloat/dec-1e4932.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-1e4932.pxf rename to testdata/bigfloat/dec-1e4932.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-2pow1024-p64.binpb b/testdata/bigfloat/dec-2pow1024-p64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-2pow1024-p64.binpb rename to testdata/bigfloat/dec-2pow1024-p64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-2pow1024-p64.pxf b/testdata/bigfloat/dec-2pow1024-p64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-2pow1024-p64.pxf rename to testdata/bigfloat/dec-2pow1024-p64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-exponent-bound-prec256.binpb b/testdata/bigfloat/dec-exponent-bound-prec256.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-exponent-bound-prec256.binpb rename to testdata/bigfloat/dec-exponent-bound-prec256.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-exponent-bound-prec256.pxf b/testdata/bigfloat/dec-exponent-bound-prec256.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-exponent-bound-prec256.pxf rename to testdata/bigfloat/dec-exponent-bound-prec256.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-exponent-only-p24.binpb b/testdata/bigfloat/dec-exponent-only-p24.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-exponent-only-p24.binpb rename to testdata/bigfloat/dec-exponent-only-p24.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-exponent-only-p24.pxf b/testdata/bigfloat/dec-exponent-only-p24.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-exponent-only-p24.pxf rename to testdata/bigfloat/dec-exponent-only-p24.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-p333.binpb b/testdata/bigfloat/dec-hundred-digits-p333.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-p333.binpb rename to testdata/bigfloat/dec-hundred-digits-p333.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-p333.pxf b/testdata/bigfloat/dec-hundred-digits-p333.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-p333.pxf rename to testdata/bigfloat/dec-hundred-digits-p333.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-unsuffixed.binpb b/testdata/bigfloat/dec-hundred-digits-unsuffixed.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-unsuffixed.binpb rename to testdata/bigfloat/dec-hundred-digits-unsuffixed.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-unsuffixed.pxf b/testdata/bigfloat/dec-hundred-digits-unsuffixed.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-hundred-digits-unsuffixed.pxf rename to testdata/bigfloat/dec-hundred-digits-unsuffixed.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-integer-p64.binpb b/testdata/bigfloat/dec-integer-p64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-integer-p64.binpb rename to testdata/bigfloat/dec-integer-p64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-integer-p64.pxf b/testdata/bigfloat/dec-integer-p64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-integer-p64.pxf rename to testdata/bigfloat/dec-integer-p64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-max-precision-with-exponent.binpb b/testdata/bigfloat/dec-max-precision-with-exponent.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-max-precision-with-exponent.binpb rename to testdata/bigfloat/dec-max-precision-with-exponent.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-max-precision-with-exponent.pxf b/testdata/bigfloat/dec-max-precision-with-exponent.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-max-precision-with-exponent.pxf rename to testdata/bigfloat/dec-max-precision-with-exponent.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-minus-two-and-a-half.binpb b/testdata/bigfloat/dec-minus-two-and-a-half.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-minus-two-and-a-half.binpb rename to testdata/bigfloat/dec-minus-two-and-a-half.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-minus-two-and-a-half.pxf b/testdata/bigfloat/dec-minus-two-and-a-half.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-minus-two-and-a-half.pxf rename to testdata/bigfloat/dec-minus-two-and-a-half.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-negative-zero-p64.binpb b/testdata/bigfloat/dec-negative-zero-p64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-negative-zero-p64.binpb rename to testdata/bigfloat/dec-negative-zero-p64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-negative-zero-p64.pxf b/testdata/bigfloat/dec-negative-zero-p64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-negative-zero-p64.pxf rename to testdata/bigfloat/dec-negative-zero-p64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-negative-zero.binpb b/testdata/bigfloat/dec-negative-zero.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-negative-zero.binpb rename to testdata/bigfloat/dec-negative-zero.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-negative-zero.pxf b/testdata/bigfloat/dec-negative-zero.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-negative-zero.pxf rename to testdata/bigfloat/dec-negative-zero.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-one-at-max-precision.binpb b/testdata/bigfloat/dec-one-at-max-precision.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-one-at-max-precision.binpb rename to testdata/bigfloat/dec-one-at-max-precision.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-one-at-max-precision.pxf b/testdata/bigfloat/dec-one-at-max-precision.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-one-at-max-precision.pxf rename to testdata/bigfloat/dec-one-at-max-precision.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-rounds-up-to-power-p8.binpb b/testdata/bigfloat/dec-rounds-up-to-power-p8.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-rounds-up-to-power-p8.binpb rename to testdata/bigfloat/dec-rounds-up-to-power-p8.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-rounds-up-to-power-p8.pxf b/testdata/bigfloat/dec-rounds-up-to-power-p8.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-rounds-up-to-power-p8.pxf rename to testdata/bigfloat/dec-rounds-up-to-power-p8.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-small-positional.binpb b/testdata/bigfloat/dec-small-positional.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-small-positional.binpb rename to testdata/bigfloat/dec-small-positional.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-small-positional.pxf b/testdata/bigfloat/dec-small-positional.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-small-positional.pxf rename to testdata/bigfloat/dec-small-positional.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-tenth-p53.binpb b/testdata/bigfloat/dec-tenth-p53.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-tenth-p53.binpb rename to testdata/bigfloat/dec-tenth-p53.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-tenth-p53.pxf b/testdata/bigfloat/dec-tenth-p53.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-tenth-p53.pxf rename to testdata/bigfloat/dec-tenth-p53.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-tenth-unsuffixed.binpb b/testdata/bigfloat/dec-tenth-unsuffixed.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-tenth-unsuffixed.binpb rename to testdata/bigfloat/dec-tenth-unsuffixed.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-tenth-unsuffixed.pxf b/testdata/bigfloat/dec-tenth-unsuffixed.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-tenth-unsuffixed.pxf rename to testdata/bigfloat/dec-tenth-unsuffixed.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-zero-p64.binpb b/testdata/bigfloat/dec-zero-p64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-zero-p64.binpb rename to testdata/bigfloat/dec-zero-p64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-zero-p64.pxf b/testdata/bigfloat/dec-zero-p64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-zero-p64.pxf rename to testdata/bigfloat/dec-zero-p64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-zero.binpb b/testdata/bigfloat/dec-zero.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-zero.binpb rename to testdata/bigfloat/dec-zero.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/dec-zero.pxf b/testdata/bigfloat/dec-zero.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/dec-zero.pxf rename to testdata/bigfloat/dec-zero.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-2pow1024-prec64.binpb b/testdata/bigfloat/enc-2pow1024-prec64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-2pow1024-prec64.binpb rename to testdata/bigfloat/enc-2pow1024-prec64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-2pow1024-prec64.pxf b/testdata/bigfloat/enc-2pow1024-prec64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-2pow1024-prec64.pxf rename to testdata/bigfloat/enc-2pow1024-prec64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-exponent-bound-prec256.binpb b/testdata/bigfloat/enc-exponent-bound-prec256.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-exponent-bound-prec256.binpb rename to testdata/bigfloat/enc-exponent-bound-prec256.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-exponent-bound-prec256.pxf b/testdata/bigfloat/enc-exponent-bound-prec256.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-exponent-bound-prec256.pxf rename to testdata/bigfloat/enc-exponent-bound-prec256.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-integer.binpb b/testdata/bigfloat/enc-integer.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-integer.binpb rename to testdata/bigfloat/enc-integer.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-integer.pxf b/testdata/bigfloat/enc-integer.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-integer.pxf rename to testdata/bigfloat/enc-integer.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-long-mantissa.binpb b/testdata/bigfloat/enc-long-mantissa.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-long-mantissa.binpb rename to testdata/bigfloat/enc-long-mantissa.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-long-mantissa.pxf b/testdata/bigfloat/enc-long-mantissa.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-long-mantissa.pxf rename to testdata/bigfloat/enc-long-mantissa.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-max-precision-with-exponent.binpb b/testdata/bigfloat/enc-max-precision-with-exponent.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-max-precision-with-exponent.binpb rename to testdata/bigfloat/enc-max-precision-with-exponent.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-max-precision-with-exponent.pxf b/testdata/bigfloat/enc-max-precision-with-exponent.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-max-precision-with-exponent.pxf rename to testdata/bigfloat/enc-max-precision-with-exponent.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-one-and-a-half.binpb b/testdata/bigfloat/enc-one-and-a-half.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-one-and-a-half.binpb rename to testdata/bigfloat/enc-one-and-a-half.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-one-and-a-half.pxf b/testdata/bigfloat/enc-one-and-a-half.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-one-and-a-half.pxf rename to testdata/bigfloat/enc-one-and-a-half.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-prec-zero.binpb b/testdata/bigfloat/enc-prec-zero.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-prec-zero.binpb rename to testdata/bigfloat/enc-prec-zero.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-prec-zero.pxf b/testdata/bigfloat/enc-prec-zero.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-prec-zero.pxf rename to testdata/bigfloat/enc-prec-zero.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-tenth-as-double.binpb b/testdata/bigfloat/enc-tenth-as-double.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-tenth-as-double.binpb rename to testdata/bigfloat/enc-tenth-as-double.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-tenth-as-double.pxf b/testdata/bigfloat/enc-tenth-as-double.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-tenth-as-double.pxf rename to testdata/bigfloat/enc-tenth-as-double.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-three-quarters-negative.binpb b/testdata/bigfloat/enc-three-quarters-negative.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-three-quarters-negative.binpb rename to testdata/bigfloat/enc-three-quarters-negative.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-three-quarters-negative.pxf b/testdata/bigfloat/enc-three-quarters-negative.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-three-quarters-negative.pxf rename to testdata/bigfloat/enc-three-quarters-negative.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-tiny-negative-exponent.binpb b/testdata/bigfloat/enc-tiny-negative-exponent.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-tiny-negative-exponent.binpb rename to testdata/bigfloat/enc-tiny-negative-exponent.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-tiny-negative-exponent.pxf b/testdata/bigfloat/enc-tiny-negative-exponent.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-tiny-negative-exponent.pxf rename to testdata/bigfloat/enc-tiny-negative-exponent.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-wide-mantissa-prec53.binpb b/testdata/bigfloat/enc-wide-mantissa-prec53.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-wide-mantissa-prec53.binpb rename to testdata/bigfloat/enc-wide-mantissa-prec53.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-wide-mantissa-prec53.pxf b/testdata/bigfloat/enc-wide-mantissa-prec53.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-wide-mantissa-prec53.pxf rename to testdata/bigfloat/enc-wide-mantissa-prec53.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64-negative.binpb b/testdata/bigfloat/enc-zero-prec64-negative.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64-negative.binpb rename to testdata/bigfloat/enc-zero-prec64-negative.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64-negative.pxf b/testdata/bigfloat/enc-zero-prec64-negative.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64-negative.pxf rename to testdata/bigfloat/enc-zero-prec64-negative.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64.binpb b/testdata/bigfloat/enc-zero-prec64.binpb similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64.binpb rename to testdata/bigfloat/enc-zero-prec64.binpb diff --git a/pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64.pxf b/testdata/bigfloat/enc-zero-prec64.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/enc-zero-prec64.pxf rename to testdata/bigfloat/enc-zero-prec64.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/err-precision-overflow.pxf b/testdata/bigfloat/err-precision-overflow.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/err-precision-overflow.pxf rename to testdata/bigfloat/err-precision-overflow.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/err-precision-past-limit.pxf b/testdata/bigfloat/err-precision-past-limit.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/err-precision-past-limit.pxf rename to testdata/bigfloat/err-precision-past-limit.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/err-precision-zero.pxf b/testdata/bigfloat/err-precision-zero.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/err-precision-zero.pxf rename to testdata/bigfloat/err-precision-zero.pxf diff --git a/pxf/src/test/resources/corpus/bigfloat/err-suffix-on-int32.pxf b/testdata/bigfloat/err-suffix-on-int32.pxf similarity index 100% rename from pxf/src/test/resources/corpus/bigfloat/err-suffix-on-int32.pxf rename to testdata/bigfloat/err-suffix-on-int32.pxf diff --git a/testdata/bigfloat/generate.py b/testdata/bigfloat/generate.py new file mode 100644 index 0000000..590c836 --- /dev/null +++ b/testdata/bigfloat/generate.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 TrendVidia, LLC. +"""Generate the pxf.BigFloat text-form corpus (issues #393, #406). + +Expected values are computed here with exact rational arithmetic, from +the rule in draft -01 § PXF Annotations, independently of any port: + + * Writing: a BigFloat (mantissa M, exponent E, prec P, negative S) is + the value (-1)^S * M * 2^E. It is written with N significant decimal + digits, N = (number of decimal digits of 2^P) + 1, rounded to + nearest with ties to even, trailing zeros removed, in the form of C's + %g with precision N: exponent form (d.ddd e+XX, at least two exponent + digits) when the decimal exponent is below -4 or at least N, + positional otherwise, followed by the precision suffix "p

". The + value is M rounded to P bits first (#410: a mantissa wider than P is + the rounded value). A zero is "0p

" or "-0p

"; only a prec-0 + zero is written without a suffix ("0", or "-0"). + * Reading: a literal with a suffix "p

" is read at exactly P bits, + one without at 256 bits, rounded to nearest with ties to even. It is + stored as mantissa = the significand as a P-bit integer (top bit + set), exponent = binary exponent - P, prec = P. A zero is stored + with an empty mantissa, exponent 0, prec P (0 without a suffix) and + negative for a sign. + +Each enc- case is a PB message (enc-.binpb) and the PXF text +an encoder MUST write for it (enc-.pxf). Each dec- case is a +PXF document (dec-.pxf) and the PB bytes a decoder MUST produce +(dec-.binpb). Each err-.pxf is a document a decoder MUST +reject. Run from anywhere; it writes next to itself, and is +idempotent. +""" +from fractions import Fraction +from pathlib import Path + +HERE = Path(__file__).resolve().parent + +MAX_NUMERIC_LITERAL_DIGITS = 4096 + + +def digits_for(prec: int) -> int: + """N, the significant digits a prec-bit value is written with.""" + return len(str(2 ** prec)) + 1 + + +# The largest precision a suffix may state: its N is within the limit. +MAX_PREC = 1 +while digits_for(MAX_PREC + 1) <= MAX_NUMERIC_LITERAL_DIGITS: + MAX_PREC += 1 + + +def varint(n: int) -> bytes: + n &= 0xFFFFFFFFFFFFFFFF + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def bigfloat_pb(mant: int, exp: int, prec: int, neg: bool) -> bytes: + """H { bf = BigFloat{...} } with proto3 zero fields omitted, in + field-number order (deterministic marshaling).""" + sub = b"" + if mant: + mb = mant.to_bytes((mant.bit_length() + 7) // 8, "big") + sub += b"\x0a" + varint(len(mb)) + mb + if exp: + sub += b"\x10" + varint(exp) + if prec: + sub += b"\x18" + varint(prec) + if neg: + sub += b"\x20\x01" + return b"\x0a" + varint(len(sub)) + sub + + +def round_half_even(x: Fraction) -> int: + q, r = divmod(x.numerator, x.denominator) + twice = 2 * r + if twice > x.denominator or (twice == x.denominator and q % 2 == 1): + q += 1 + return q + + +def round_to_prec(v: Fraction, p: int) -> Fraction: + """v rounded to p significant bits, half to even.""" + e2 = 0 # 2^(p-1) <= v / 2^e2 < 2^p + while v / Fraction(2) ** e2 >= 2 ** p: + e2 += 1 + while v / Fraction(2) ** e2 < 2 ** (p - 1): + e2 -= 1 + m = round_half_even(v / Fraction(2) ** e2) + return Fraction(m) * Fraction(2) ** e2 + + +def write_literal(mant: int, exp: int, prec: int, neg: bool) -> str: + sign = "-" if neg else "" + if mant == 0 or prec == 0: + return f"{sign}0" if prec == 0 else f"{sign}0p{prec}" + # A mantissa wider than prec bits: the value is the rounded one. + v = round_to_prec(Fraction(mant) * (Fraction(2) ** exp), prec) + n = digits_for(prec) + k = 0 # decimal exponent: 10^k <= v < 10^(k+1) + while Fraction(10) ** (k + 1) <= v: + k += 1 + while Fraction(10) ** k > v: + k -= 1 + q = round_half_even(v / Fraction(10) ** (k - n + 1)) + if q == 10 ** n: + q //= 10 + k += 1 + digits = str(q).rstrip("0") or "0" + if k < -4 or k >= n: + body = digits[0] + ("." + digits[1:] if len(digits) > 1 else "") + return f"{sign}{body}e{'-' if k < 0 else '+'}{abs(k):02d}p{prec}" + if k >= 0: + ip, fp = digits[: k + 1].ljust(k + 1, "0"), digits[k + 1 :] + return sign + ip + ("." + fp if fp else "") + f"p{prec}" + return sign + "0." + "0" * (-k - 1) + digits + f"p{prec}" + + +def read_literal(lit: str): + neg = lit.startswith("-") + s = lit.lstrip("+-") + s, _, suffix = s.partition("p") + mant_part, _, exp_part = s.lower().partition("e") + ip, _, fp = mant_part.partition(".") + v = Fraction(int(ip + fp or "0")) / (Fraction(10) ** len(fp)) + if exp_part: + v *= Fraction(10) ** int(exp_part) + p = int(suffix) if suffix else 256 + if v == 0: + return 0, 0, (p if suffix else 0), neg + e2 = 0 # 2^(p-1) <= v / 2^e2 < 2^p + while v / Fraction(2) ** e2 >= 2 ** p: + e2 += 1 + while v / Fraction(2) ** e2 < 2 ** (p - 1): + e2 -= 1 + m = round_half_even(v / Fraction(2) ** e2) + if m == 2 ** p: + m //= 2 + e2 += 1 + return m, e2, p, neg + + +ENCODE = { + # name: (mantissa, exponent, prec, negative) + "2pow1024-prec64": (1, 1024, 64, False), + "three-quarters-negative": (3, -2, 64, True), + "tenth-as-double": (0x1999999999999A, -56, 53, False), + "one-and-a-half": (3, -1, 53, False), + "long-mantissa": (2 ** 100 - 1, -50, 100, False), + "tiny-negative-exponent": (1, -16385, 64, False), + "prec-zero": (0, 0, 0, False), + "integer": (1000, 0, 64, False), + # #410: a zero keeps its precision and sign; a wide mantissa is the + # value rounded to prec; the exponent bound at the 256-bit default; + # a literal of MaxNumericLiteralDigits digits plus exponent and suffix. + "zero-prec64": (0, 0, 64, False), + "zero-prec64-negative": (0, 0, 64, True), + "wide-mantissa-prec53": (2 ** 100 - 1, -50, 53, False), + "exponent-bound-prec256": (1, -16385, 256, False), + "max-precision-with-exponent": (2 ** MAX_PREC - 1, 16384 - MAX_PREC, MAX_PREC, False), +} + +DECODE = { + # name: literal + "tenth-unsuffixed": "0.1", + "tenth-p53": "0.10000000000000001p53", + "minus-two-and-a-half": "-2.5", + "1e4932": "1e4932", + "zero": "0", + "hundred-digits-unsuffixed": "1" + "2" * 99 + "e-50", + "hundred-digits-p333": "1" + "2" * 99 + "e-50p333", + "small-positional": "0.000125", + "2pow1024-p64": "1.79769313486231590773e+308p64", + "integer-p64": "1000p64", + "exponent-only-p24": "1e-5p24", + # 0.999999 at 8 bits rounds up to 2^8, past the significand: 1 at 8 bits. + "rounds-up-to-power-p8": "0.999999p8", + "negative-zero": "-0", + # The largest precision whose N, 4096, is within MaxNumericLiteralDigits. + "one-at-max-precision": f"1p{MAX_PREC}", + # #410: a zero with a precision and sign; the producer's own literals + # at the exponent bound and at the digit limit read back. + "zero-p64": "0p64", + "negative-zero-p64": "-0p64", + "exponent-bound-prec256": write_literal(1, -16385, 256, False), + "max-precision-with-exponent": write_literal(2 ** MAX_PREC - 1, 16384 - MAX_PREC, MAX_PREC, False), +} + +# Documents a decoder MUST reject: a precision suffix on a field that is +# not a pxf.BigFloat (the error names the field, count), a zero +# precision, and a precision whose N exceeds MaxNumericLiteralDigits, +# one past the limit and one that overflows every fixed-width integer. +REJECT = { + "suffix-on-int32": "count = 5p64", + "precision-zero": "bf = 1.5p0", + "precision-past-limit": f"bf = 1p{MAX_PREC + 1}", + "precision-overflow": "bf = 1p99999999999999999999999", +} + + +def main() -> None: + for name, (m, e, p, s) in ENCODE.items(): + (HERE / f"enc-{name}.binpb").write_bytes(bigfloat_pb(m, e, p, s)) + (HERE / f"enc-{name}.pxf").write_text(f"bf = {write_literal(m, e, p, s)}\n") + for name, lit in DECODE.items(): + (HERE / f"dec-{name}.pxf").write_text(f"@type bf.v1.H\n\nbf = {lit}\n") + (HERE / f"dec-{name}.binpb").write_bytes(bigfloat_pb(*read_literal(lit))) + for name, body in REJECT.items(): + (HERE / f"err-{name}.pxf").write_text(f"@type bf.v1.H\n\n{body}\n") + print(f"wrote {len(ENCODE)} encode, {len(DECODE)} decode and {len(REJECT)} reject cases") + + +if __name__ == "__main__": + main() diff --git a/pxf/src/test/resources/corpus/blocks/README.md b/testdata/blocks/README.md similarity index 100% rename from pxf/src/test/resources/corpus/blocks/README.md rename to testdata/blocks/README.md diff --git a/pxf/src/test/proto/blocks/blocks.proto b/testdata/blocks/blocks.proto similarity index 100% rename from pxf/src/test/proto/blocks/blocks.proto rename to testdata/blocks/blocks.proto diff --git a/pxf/src/test/resources/corpus/blocks/comma-between-elements.pxf b/testdata/blocks/comma-between-elements.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/comma-between-elements.pxf rename to testdata/blocks/comma-between-elements.pxf diff --git a/pxf/src/test/resources/corpus/blocks/err-comma-in-block.pxf b/testdata/blocks/err-comma-in-block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/err-comma-in-block.pxf rename to testdata/blocks/err-comma-in-block.pxf diff --git a/pxf/src/test/resources/corpus/blocks/err-comma-in-list-element.pxf b/testdata/blocks/err-comma-in-list-element.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/err-comma-in-list-element.pxf rename to testdata/blocks/err-comma-in-list-element.pxf diff --git a/pxf/src/test/resources/corpus/blocks/err-comma-in-map.pxf b/testdata/blocks/err-comma-in-map.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/err-comma-in-map.pxf rename to testdata/blocks/err-comma-in-map.pxf diff --git a/pxf/src/test/resources/corpus/blocks/err-semicolon-in-block.pxf b/testdata/blocks/err-semicolon-in-block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/err-semicolon-in-block.pxf rename to testdata/blocks/err-semicolon-in-block.pxf diff --git a/pxf/src/test/resources/corpus/blocks/newline-in-block.pxf b/testdata/blocks/newline-in-block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/newline-in-block.pxf rename to testdata/blocks/newline-in-block.pxf diff --git a/pxf/src/test/resources/corpus/blocks/space-in-block.pxf b/testdata/blocks/space-in-block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/blocks/space-in-block.pxf rename to testdata/blocks/space-in-block.pxf diff --git a/pxf/src/test/resources/corpus/dataset/README.md b/testdata/dataset/README.md similarity index 95% rename from pxf/src/test/resources/corpus/dataset/README.md rename to testdata/dataset/README.md index 7ce152d..76b9caa 100644 --- a/pxf/src/test/resources/corpus/dataset/README.md +++ b/testdata/dataset/README.md @@ -52,7 +52,10 @@ fixtures are in [`../strings/`](../strings/). applies the singular-field null rules, so `null` in a plain `int32` column is an error. `null-on-optional-cell` (accept) is the other side: `null` in an `optional int32` column leaves the field unset and -present-but-null. Its general fixtures are in [`../nulls/`](../nulls/). +present-but-null. `null-on-required-cell` (reject) is the exception: in a +column whose field has `(pxf.required)`, `null` is an error, as an empty +cell is, since a required field holds a value. Its general fixtures are +in [`../nulls/`](../nulls/). The `dotted-*` fixtures and `duplicate-column` pin column paths (issue [#342](https://github.com/trendvidia/protowire/issues/342), v1.13). A diff --git a/pxf/src/test/resources/corpus/dataset/arity-long.pxf b/testdata/dataset/arity-long.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/arity-long.pxf rename to testdata/dataset/arity-long.pxf diff --git a/pxf/src/test/resources/corpus/dataset/arity-short.pxf b/testdata/dataset/arity-short.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/arity-short.pxf rename to testdata/dataset/arity-short.pxf diff --git a/pxf/src/test/resources/corpus/dataset/block-cell.pxf b/testdata/dataset/block-cell.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/block-cell.pxf rename to testdata/dataset/block-cell.pxf diff --git a/pxf/src/test/resources/corpus/dataset/block-on-scalar.pxf b/testdata/dataset/block-on-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/block-on-scalar.pxf rename to testdata/dataset/block-on-scalar.pxf diff --git a/pxf/src/test/resources/corpus/dataset/cell-framing.pxf b/testdata/dataset/cell-framing.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/cell-framing.pxf rename to testdata/dataset/cell-framing.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-bad-header-empty-row.pxf b/testdata/dataset/dotted-bad-header-empty-row.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-bad-header-empty-row.pxf rename to testdata/dataset/dotted-bad-header-empty-row.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-bad-header-no-rows.pxf b/testdata/dataset/dotted-bad-header-no-rows.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-bad-header-no-rows.pxf rename to testdata/dataset/dotted-bad-header-no-rows.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-column.pxf b/testdata/dataset/dotted-column.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-column.pxf rename to testdata/dataset/dotted-column.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-empty-segment.pxf b/testdata/dataset/dotted-empty-segment.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-empty-segment.pxf rename to testdata/dataset/dotted-empty-segment.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-grouping.pxf b/testdata/dataset/dotted-grouping.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-grouping.pxf rename to testdata/dataset/dotted-grouping.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-any.pxf b/testdata/dataset/dotted-into-any.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-any.pxf rename to testdata/dataset/dotted-into-any.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-bignum.pxf b/testdata/dataset/dotted-into-bignum.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-bignum.pxf rename to testdata/dataset/dotted-into-bignum.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-duration.pxf b/testdata/dataset/dotted-into-duration.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-duration.pxf rename to testdata/dataset/dotted-into-duration.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-secret.pxf b/testdata/dataset/dotted-into-secret.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-secret.pxf rename to testdata/dataset/dotted-into-secret.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-timestamp.pxf b/testdata/dataset/dotted-into-timestamp.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-timestamp.pxf rename to testdata/dataset/dotted-into-timestamp.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-into-wrapper.pxf b/testdata/dataset/dotted-into-wrapper.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-into-wrapper.pxf rename to testdata/dataset/dotted-into-wrapper.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-null-leaf.pxf b/testdata/dataset/dotted-null-leaf.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-null-leaf.pxf rename to testdata/dataset/dotted-null-leaf.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-oneof-two-members.pxf b/testdata/dataset/dotted-oneof-two-members.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-oneof-two-members.pxf rename to testdata/dataset/dotted-oneof-two-members.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-oneof.pxf b/testdata/dataset/dotted-oneof.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-oneof.pxf rename to testdata/dataset/dotted-oneof.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-overlap-deep.pxf b/testdata/dataset/dotted-overlap-deep.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-overlap-deep.pxf rename to testdata/dataset/dotted-overlap-deep.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-overlap.pxf b/testdata/dataset/dotted-overlap.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-overlap.pxf rename to testdata/dataset/dotted-overlap.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-required-missing.pxf b/testdata/dataset/dotted-required-missing.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-required-missing.pxf rename to testdata/dataset/dotted-required-missing.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-required.pxf b/testdata/dataset/dotted-required.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-required.pxf rename to testdata/dataset/dotted-required.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-through-map.pxf b/testdata/dataset/dotted-through-map.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-through-map.pxf rename to testdata/dataset/dotted-through-map.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-through-repeated.pxf b/testdata/dataset/dotted-through-repeated.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-through-repeated.pxf rename to testdata/dataset/dotted-through-repeated.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-through-scalar.pxf b/testdata/dataset/dotted-through-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-through-scalar.pxf rename to testdata/dataset/dotted-through-scalar.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-trailing-dot.pxf b/testdata/dataset/dotted-trailing-dot.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-trailing-dot.pxf rename to testdata/dataset/dotted-trailing-dot.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-unknown-segment.pxf b/testdata/dataset/dotted-unknown-segment.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-unknown-segment.pxf rename to testdata/dataset/dotted-unknown-segment.pxf diff --git a/pxf/src/test/resources/corpus/dataset/dotted-with-value-cells.pxf b/testdata/dataset/dotted-with-value-cells.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/dotted-with-value-cells.pxf rename to testdata/dataset/dotted-with-value-cells.pxf diff --git a/pxf/src/test/resources/corpus/dataset/duplicate-column.pxf b/testdata/dataset/duplicate-column.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/duplicate-column.pxf rename to testdata/dataset/duplicate-column.pxf diff --git a/pxf/src/test/resources/corpus/dataset/empty-rows.pxf b/testdata/dataset/empty-rows.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/empty-rows.pxf rename to testdata/dataset/empty-rows.pxf diff --git a/pxf/src/test/resources/corpus/dataset/junk-between-rows.pxf b/testdata/dataset/junk-between-rows.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/junk-between-rows.pxf rename to testdata/dataset/junk-between-rows.pxf diff --git a/pxf/src/test/resources/corpus/dataset/list-cell.pxf b/testdata/dataset/list-cell.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/list-cell.pxf rename to testdata/dataset/list-cell.pxf diff --git a/pxf/src/test/resources/corpus/dataset/nested-cells.pxf b/testdata/dataset/nested-cells.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/nested-cells.pxf rename to testdata/dataset/nested-cells.pxf diff --git a/pxf/src/test/resources/corpus/dataset/null-on-optional-cell.pxf b/testdata/dataset/null-on-optional-cell.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/null-on-optional-cell.pxf rename to testdata/dataset/null-on-optional-cell.pxf diff --git a/testdata/dataset/null-on-required-cell.pxf b/testdata/dataset/null-on-required-cell.pxf new file mode 100644 index 0000000..3972b2b --- /dev/null +++ b/testdata/dataset/null-on-required-cell.pxf @@ -0,0 +1,9 @@ +# §3.4.4 + § Booleans, Null, and Identifier Values. Row type +# dataset.v1.RowReq in paths.proto. MUST be rejected: o is an optional +# scalar with (pxf.required), and a null cell, like an empty one, does +# not satisfy a required field. + +@dataset dataset.v1.RowReq (id, o) + +("a", 1) +("b", null) diff --git a/pxf/src/test/resources/corpus/dataset/null-on-scalar-cell.pxf b/testdata/dataset/null-on-scalar-cell.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/null-on-scalar-cell.pxf rename to testdata/dataset/null-on-scalar-cell.pxf diff --git a/pxf/src/test/proto/dataset/paths.proto b/testdata/dataset/paths.proto similarity index 89% rename from pxf/src/test/proto/dataset/paths.proto rename to testdata/dataset/paths.proto index 7ea7bc1..e89ca25 100644 --- a/pxf/src/test/proto/dataset/paths.proto +++ b/testdata/dataset/paths.proto @@ -58,3 +58,9 @@ message Event { pxf.Secret pw = 14; optional int32 opt = 15; } + +// A required field that could otherwise hold null (null-on-required-cell). +message RowReq { + string id = 1; + optional int32 o = 2 [(pxf.required) = true]; +} diff --git a/pxf/src/test/resources/corpus/dataset/rows-end-at-directive.pxf b/testdata/dataset/rows-end-at-directive.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/rows-end-at-directive.pxf rename to testdata/dataset/rows-end-at-directive.pxf diff --git a/pxf/src/test/resources/corpus/dataset/string-newline-in-cell.pxf b/testdata/dataset/string-newline-in-cell.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/string-newline-in-cell.pxf rename to testdata/dataset/string-newline-in-cell.pxf diff --git a/pxf/src/test/resources/corpus/dataset/truncated-row-eof.pxf b/testdata/dataset/truncated-row-eof.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/truncated-row-eof.pxf rename to testdata/dataset/truncated-row-eof.pxf diff --git a/pxf/src/test/resources/corpus/dataset/truncated-row-then-rows.pxf b/testdata/dataset/truncated-row-then-rows.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/truncated-row-then-rows.pxf rename to testdata/dataset/truncated-row-then-rows.pxf diff --git a/pxf/src/test/resources/corpus/dataset/with-body.pxf b/testdata/dataset/with-body.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/with-body.pxf rename to testdata/dataset/with-body.pxf diff --git a/pxf/src/test/resources/corpus/dataset/with-type.pxf b/testdata/dataset/with-type.pxf similarity index 100% rename from pxf/src/test/resources/corpus/dataset/with-type.pxf rename to testdata/dataset/with-type.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/README.md b/testdata/duplicates/README.md similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/README.md rename to testdata/duplicates/README.md diff --git a/pxf/src/test/proto/duplicates/duplicates.proto b/testdata/duplicates/duplicates.proto similarity index 100% rename from pxf/src/test/proto/duplicates/duplicates.proto rename to testdata/duplicates/duplicates.proto diff --git a/pxf/src/test/resources/corpus/duplicates/err-map-key-twice-across-bindings.pxf b/testdata/duplicates/err-map-key-twice-across-bindings.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-map-key-twice-across-bindings.pxf rename to testdata/duplicates/err-map-key-twice-across-bindings.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-map-key-twice-in-literal.pxf b/testdata/duplicates/err-map-key-twice-in-literal.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-map-key-twice-in-literal.pxf rename to testdata/duplicates/err-map-key-twice-in-literal.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-map-key-twice-int-spellings.pxf b/testdata/duplicates/err-map-key-twice-int-spellings.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-map-key-twice-int-spellings.pxf rename to testdata/duplicates/err-map-key-twice-int-spellings.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-map-key-twice-message-values.pxf b/testdata/duplicates/err-map-key-twice-message-values.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-map-key-twice-message-values.pxf rename to testdata/duplicates/err-map-key-twice-message-values.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-map-key-twice-same-value.pxf b/testdata/duplicates/err-map-key-twice-same-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-map-key-twice-same-value.pxf rename to testdata/duplicates/err-map-key-twice-same-value.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-message-assign-then-block.pxf b/testdata/duplicates/err-message-assign-then-block.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-message-assign-then-block.pxf rename to testdata/duplicates/err-message-assign-then-block.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-message-twice.pxf b/testdata/duplicates/err-message-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-message-twice.pxf rename to testdata/duplicates/err-message-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-nested-scalar-twice.pxf b/testdata/duplicates/err-nested-scalar-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-nested-scalar-twice.pxf rename to testdata/duplicates/err-nested-scalar-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-null-then-value.pxf b/testdata/duplicates/err-null-then-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-null-then-value.pxf rename to testdata/duplicates/err-null-then-value.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-oneof-member-twice.pxf b/testdata/duplicates/err-oneof-member-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-oneof-member-twice.pxf rename to testdata/duplicates/err-oneof-member-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-oneof-scalar-twice.pxf b/testdata/duplicates/err-oneof-scalar-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-oneof-scalar-twice.pxf rename to testdata/duplicates/err-oneof-scalar-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-optional-twice.pxf b/testdata/duplicates/err-optional-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-optional-twice.pxf rename to testdata/duplicates/err-optional-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-scalar-same-value.pxf b/testdata/duplicates/err-scalar-same-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-scalar-same-value.pxf rename to testdata/duplicates/err-scalar-same-value.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-scalar-twice.pxf b/testdata/duplicates/err-scalar-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-scalar-twice.pxf rename to testdata/duplicates/err-scalar-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/err-wrapper-twice.pxf b/testdata/duplicates/err-wrapper-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/err-wrapper-twice.pxf rename to testdata/duplicates/err-wrapper-twice.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/map-key-once-per-body.pxf b/testdata/duplicates/map-key-once-per-body.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/map-key-once-per-body.pxf rename to testdata/duplicates/map-key-once-per-body.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/map-twice-distinct-keys.pxf b/testdata/duplicates/map-twice-distinct-keys.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/map-twice-distinct-keys.pxf rename to testdata/duplicates/map-twice-distinct-keys.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/once-per-body.pxf b/testdata/duplicates/once-per-body.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/once-per-body.pxf rename to testdata/duplicates/once-per-body.pxf diff --git a/pxf/src/test/resources/corpus/duplicates/repeated-twice.pxf b/testdata/duplicates/repeated-twice.pxf similarity index 100% rename from pxf/src/test/resources/corpus/duplicates/repeated-twice.pxf rename to testdata/duplicates/repeated-twice.pxf diff --git a/testdata/duration/README.md b/testdata/duration/README.md new file mode 100644 index 0000000..766fae6 --- /dev/null +++ b/testdata/duration/README.md @@ -0,0 +1,57 @@ +# Duration literal fixtures + +Cross-port conformance fixtures for **signed duration literals** +([issue #234](https://github.com/trendvidia/protowire/issues/234); +IETF draft `-01` §3.3 `duration = ["-"] 1*duration-segment` and +§"Timestamps and Durations"). + +`google.protobuf.Duration` is signed, and every port in the family already +emitted a negative Duration with a leading `-` (mirroring +`time.Duration.String()`) and parsed it back — while draft `-01`'s grammar +admitted no sign, and its disambiguation prose said `5seconds` was an +identifier, which no port did. These fixtures pin what the ports do and +the draft now says. As with [`testdata/keyed/`](../keyed/), the +conformance harness wiring is per port; each file's leading comment states +the expected behavior, and `cmd/pxf/duration_test.go` runs them through +the reference implementation. + +All fixtures bind against [`duration.proto`](duration.proto) (package +`duration.v1`, one field `Holder.d`); each file's `@type` directive names +the message. + +## Accept fixtures + +| File | Asserts | +|---|---| +| `negative-seconds.pxf` | `-30s` binds to seconds = -30. | +| `negative-fractional.pxf` | `-1.5s` binds to seconds = -1, nanos = -500000000 — same sign, per `google.protobuf.Duration`. | +| `negative-multi-segment.pxf` | `-1h30m` binds to seconds = -5400: one sign for the whole literal. | +| `negative-zero.pxf` | `-0s` is zero. | + +## fmt canonicalization pair + +| Pair | Asserts | +|---|---| +| `roundtrip-negative` | A negative Duration re-encodes as one leading `-` before the first segment; the expected file is a fmt fixed point. Comment-free apart from `@type`, as in `keyed/`, so the byte-level expectation pins the literal, not comment placement. | + +## Reject fixtures + +| File | Expected error | +|---|---| +| `err-sign-per-segment.pxf` | `1h-30m` — a sign inside the literal; the duration token ends at `1h`. | +| `err-digit-led-identifier.pxf` | `5seconds` — DIGIT-led token past a unit into letters is an invalid duration, not an identifier. | +| `err-unit-then-alpha.pxf` | `5sx` — the same rule at the shortest distance. | + +## Not pinned here + +A leading `+` (`+30s`) is deliberately absent. It is not in the grammar +and no encoder writes it, but ports differ on lenience (the reference +port rejects it at the lexer; several others skip a `+` inside their +duration validators), and issue #234 leaves whether to admit it as an +open decision. A fixture either way would force that decision by +breaking half the family; add one when it is taken. + +## Adding fixtures + +Extend `duration.proto` additively only (new fields / messages); renaming +or renumbering `Holder.d` invalidates every existing fixture at once. diff --git a/testdata/duration/duration.proto b/testdata/duration/duration.proto new file mode 100644 index 0000000..e3663ea --- /dev/null +++ b/testdata/duration/duration.proto @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 TrendVidia, LLC. + +// Schema for the duration-literal conformance fixtures under +// testdata/duration/ (issue #234; IETF draft -01 §3.3 "duration" and +// §"Timestamps and Durations"). +// +// Holder.d is the one field every fixture binds; a fixture that decodes +// asserts the (seconds, nanos) it must bind to in its leading comment. + +syntax = "proto3"; + +package duration.v1; + +import "google/protobuf/duration.proto"; + +message Holder { + google.protobuf.Duration d = 1; +} diff --git a/testdata/duration/err-digit-led-identifier.pxf b/testdata/duration/err-digit-led-identifier.pxf new file mode 100644 index 0000000..083c551 --- /dev/null +++ b/testdata/duration/err-digit-led-identifier.pxf @@ -0,0 +1,7 @@ +@type duration.v1.Holder + +# A DIGIT-led token that runs past a time-unit into further letters is an +# invalid duration, never an identifier: "5seconds" MUST be rejected as a +# malformed duration. + +d = 5seconds diff --git a/testdata/duration/err-sign-per-segment.pxf b/testdata/duration/err-sign-per-segment.pxf new file mode 100644 index 0000000..ae33221 --- /dev/null +++ b/testdata/duration/err-sign-per-segment.pxf @@ -0,0 +1,6 @@ +@type duration.v1.Holder + +# A sign inside the literal is not a duration: "1h-30m" MUST be rejected +# (the token ends at "1h"; "-30m" cannot follow a value). + +d = 1h-30m diff --git a/testdata/duration/err-unit-then-alpha.pxf b/testdata/duration/err-unit-then-alpha.pxf new file mode 100644 index 0000000..3f74204 --- /dev/null +++ b/testdata/duration/err-unit-then-alpha.pxf @@ -0,0 +1,6 @@ +@type duration.v1.Holder + +# Same rule at the shortest distance: "5sx" is not "5s" followed by an +# identifier; it is one malformed duration token and MUST be rejected. + +d = 5sx diff --git a/testdata/duration/negative-fractional.pxf b/testdata/duration/negative-fractional.pxf new file mode 100644 index 0000000..8b7e2ce --- /dev/null +++ b/testdata/duration/negative-fractional.pxf @@ -0,0 +1,7 @@ +@type duration.v1.Holder + +# A fractional segment under a leading "-" yields seconds and nanos of the +# same sign, as google.protobuf.Duration requires: seconds = -1, +# nanos = -500000000. + +d = -1.5s diff --git a/testdata/duration/negative-multi-segment.pxf b/testdata/duration/negative-multi-segment.pxf new file mode 100644 index 0000000..a05d802 --- /dev/null +++ b/testdata/duration/negative-multi-segment.pxf @@ -0,0 +1,6 @@ +@type duration.v1.Holder + +# The sign applies to the whole literal, not to its first segment: +# -1h30m is minus ninety minutes, seconds = -5400. + +d = -1h30m diff --git a/testdata/duration/negative-seconds.pxf b/testdata/duration/negative-seconds.pxf new file mode 100644 index 0000000..6120f76 --- /dev/null +++ b/testdata/duration/negative-seconds.pxf @@ -0,0 +1,6 @@ +@type duration.v1.Holder + +# A negative Duration is written with one leading "-" before the first +# segment. Binds to seconds = -30, nanos = 0. + +d = -30s diff --git a/testdata/duration/negative-zero.pxf b/testdata/duration/negative-zero.pxf new file mode 100644 index 0000000..0b09a44 --- /dev/null +++ b/testdata/duration/negative-zero.pxf @@ -0,0 +1,5 @@ +@type duration.v1.Holder + +# "-0s" denotes zero: seconds = 0, nanos = 0. + +d = -0s diff --git a/testdata/duration/roundtrip-negative.expected.pxf b/testdata/duration/roundtrip-negative.expected.pxf new file mode 100644 index 0000000..7552d10 --- /dev/null +++ b/testdata/duration/roundtrip-negative.expected.pxf @@ -0,0 +1,3 @@ +@type duration.v1.Holder + +d = -1.5s diff --git a/testdata/duration/roundtrip-negative.pxf b/testdata/duration/roundtrip-negative.pxf new file mode 100644 index 0000000..7552d10 --- /dev/null +++ b/testdata/duration/roundtrip-negative.pxf @@ -0,0 +1,3 @@ +@type duration.v1.Holder + +d = -1.5s diff --git a/pxf/src/test/resources/keyed/README.md b/testdata/keyed/README.md similarity index 100% rename from pxf/src/test/resources/keyed/README.md rename to testdata/keyed/README.md diff --git a/pxf/src/test/resources/keyed/anonymous-duplicate-ok.pxf b/testdata/keyed/anonymous-duplicate-ok.pxf similarity index 100% rename from pxf/src/test/resources/keyed/anonymous-duplicate-ok.pxf rename to testdata/keyed/anonymous-duplicate-ok.pxf diff --git a/pxf/src/test/resources/keyed/anonymous-equivalence.pxf b/testdata/keyed/anonymous-equivalence.pxf similarity index 100% rename from pxf/src/test/resources/keyed/anonymous-equivalence.pxf rename to testdata/keyed/anonymous-equivalence.pxf diff --git a/pxf/src/test/resources/keyed/err-duplicate-key-spelling.pxf b/testdata/keyed/err-duplicate-key-spelling.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-duplicate-key-spelling.pxf rename to testdata/keyed/err-duplicate-key-spelling.pxf diff --git a/pxf/src/test/resources/keyed/err-duplicate-key.pxf b/testdata/keyed/err-duplicate-key.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-duplicate-key.pxf rename to testdata/keyed/err-duplicate-key.pxf diff --git a/pxf/src/test/resources/keyed/err-empty-key-anonymous.pxf b/testdata/keyed/err-empty-key-anonymous.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-empty-key-anonymous.pxf rename to testdata/keyed/err-empty-key-anonymous.pxf diff --git a/pxf/src/test/resources/keyed/err-empty-key.pxf b/testdata/keyed/err-empty-key.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-empty-key.pxf rename to testdata/keyed/err-empty-key.pxf diff --git a/pxf/src/test/resources/keyed/err-key-conflict.pxf b/testdata/keyed/err-key-conflict.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-key-conflict.pxf rename to testdata/keyed/err-key-conflict.pxf diff --git a/pxf/src/test/resources/keyed/err-key-null.pxf b/testdata/keyed/err-key-null.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-key-null.pxf rename to testdata/keyed/err-key-null.pxf diff --git a/pxf/src/test/resources/keyed/err-key-oneof-sibling-null.pxf b/testdata/keyed/err-key-oneof-sibling-null.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-key-oneof-sibling-null.pxf rename to testdata/keyed/err-key-oneof-sibling-null.pxf diff --git a/pxf/src/test/resources/keyed/err-key-oneof-sibling.pxf b/testdata/keyed/err-key-oneof-sibling.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-key-oneof-sibling.pxf rename to testdata/keyed/err-key-oneof-sibling.pxf diff --git a/pxf/src/test/resources/keyed/err-quoted-name-unkeyed.pxf b/testdata/keyed/err-quoted-name-unkeyed.pxf similarity index 100% rename from pxf/src/test/resources/keyed/err-quoted-name-unkeyed.pxf rename to testdata/keyed/err-quoted-name-unkeyed.pxf diff --git a/pxf/src/test/resources/keyed/fmt-anonymous-to-keyed.expected.pxf b/testdata/keyed/fmt-anonymous-to-keyed.expected.pxf similarity index 100% rename from pxf/src/test/resources/keyed/fmt-anonymous-to-keyed.expected.pxf rename to testdata/keyed/fmt-anonymous-to-keyed.expected.pxf diff --git a/pxf/src/test/resources/keyed/fmt-anonymous-to-keyed.pxf b/testdata/keyed/fmt-anonymous-to-keyed.pxf similarity index 100% rename from pxf/src/test/resources/keyed/fmt-anonymous-to-keyed.pxf rename to testdata/keyed/fmt-anonymous-to-keyed.pxf diff --git a/pxf/src/test/resources/keyed/fmt-unquote.expected.pxf b/testdata/keyed/fmt-unquote.expected.pxf similarity index 100% rename from pxf/src/test/resources/keyed/fmt-unquote.expected.pxf rename to testdata/keyed/fmt-unquote.expected.pxf diff --git a/pxf/src/test/resources/keyed/fmt-unquote.pxf b/testdata/keyed/fmt-unquote.pxf similarity index 100% rename from pxf/src/test/resources/keyed/fmt-unquote.pxf rename to testdata/keyed/fmt-unquote.pxf diff --git a/pxf/src/test/resources/keyed/key-null-anonymous-ok.pxf b/testdata/keyed/key-null-anonymous-ok.pxf similarity index 100% rename from pxf/src/test/resources/keyed/key-null-anonymous-ok.pxf rename to testdata/keyed/key-null-anonymous-ok.pxf diff --git a/pxf/src/test/resources/keyed/key-oneof-ok.pxf b/testdata/keyed/key-oneof-ok.pxf similarity index 100% rename from pxf/src/test/resources/keyed/key-oneof-ok.pxf rename to testdata/keyed/key-oneof-ok.pxf diff --git a/pxf/src/test/proto/keyed/keyed.proto b/testdata/keyed/keyed.proto similarity index 100% rename from pxf/src/test/proto/keyed/keyed.proto rename to testdata/keyed/keyed.proto diff --git a/pxf/src/test/resources/keyed/redundant-key-ok.pxf b/testdata/keyed/redundant-key-ok.pxf similarity index 100% rename from pxf/src/test/resources/keyed/redundant-key-ok.pxf rename to testdata/keyed/redundant-key-ok.pxf diff --git a/pxf/src/test/resources/keyed/roundtrip-keyed.pxf b/testdata/keyed/roundtrip-keyed.pxf similarity index 100% rename from pxf/src/test/resources/keyed/roundtrip-keyed.pxf rename to testdata/keyed/roundtrip-keyed.pxf diff --git a/pxf/src/test/resources/keyed/roundtrip-quoted.pxf b/testdata/keyed/roundtrip-quoted.pxf similarity index 100% rename from pxf/src/test/resources/keyed/roundtrip-quoted.pxf rename to testdata/keyed/roundtrip-quoted.pxf diff --git a/pxf/src/test/resources/map-keys/README.md b/testdata/map-keys/README.md similarity index 81% rename from pxf/src/test/resources/map-keys/README.md rename to testdata/map-keys/README.md index 206d34f..e45a58a 100644 --- a/pxf/src/test/resources/map-keys/README.md +++ b/testdata/map-keys/README.md @@ -60,6 +60,7 @@ keyed entry names. Decided as option 2 on #306. |---|---| | [`fmt-keyword-keys`](fmt-keyword-keys.pxf) | On a string-keyed map: `"true"`, `"false"`, `"null"` and `"123"` stay quoted; the quoted identifier-safe `"plain"` canonicalizes to bare; `bare` stays bare. The input also MUST bind, to six string keys. | | [`fmt-bare-keys`](fmt-bare-keys.pxf) | On a bool-keyed map: a bare `true` and a bare `0` stay bare — a formatter does not add quotes the author did not write. A fixed point; the input also MUST bind, to the keys true and false. | +| [`fmt-dotted-keys`](fmt-dotted-keys.pxf) | On a string-keyed map: the *identifier* production admits `.`, so the quoted `"a.b"` canonicalizes to bare and a bare `c.d` stays bare — the same identifier-safe test keyed entry names already use (`user.name { }` is a legal unquoted entry name). `".e"` and `"1.5"` fail *ident-start* and stay quoted, so nothing float-shaped or leading-dot becomes bare. The marshaller writes `a.b:` bare under the same test. Both documents MUST bind, to four string keys. Issue #313, decided as (a): the ports' marshallers and formatters stopped at `[A-Za-z0-9_]` and wrote `"a.b":` where the text says bare. | Both pairs are comment-free apart from `@type`, as in [`testdata/keyed/`](../keyed/), so the byte-level expectation pins the @@ -69,7 +70,10 @@ key was quoted on the way through `fmt` (`404:` → `"404":`); it stays bare now, as the marshaller has always written it. The formatter-side wiring is per port (the reference's is -protowire-go#123); the marshaller side needs no change. +protowire-go#123); the marshaller side needs no change for the first two +pairs. The dotted pair moves both the formatter and the marshaller in +every port, sequenced after that port's #306 change so the two spelling +moves do not interleave (the reference's is protowire-go#125). ## Diagnostic diff --git a/pxf/src/test/resources/map-keys/bool-keys-integer.pxf b/testdata/map-keys/bool-keys-integer.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/bool-keys-integer.pxf rename to testdata/map-keys/bool-keys-integer.pxf diff --git a/pxf/src/test/resources/map-keys/bool-keys-keyword.pxf b/testdata/map-keys/bool-keys-keyword.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/bool-keys-keyword.pxf rename to testdata/map-keys/bool-keys-keyword.pxf diff --git a/pxf/src/test/proto/mapkeys/bool-keys.proto b/testdata/map-keys/bool-keys.proto similarity index 100% rename from pxf/src/test/proto/mapkeys/bool-keys.proto rename to testdata/map-keys/bool-keys.proto diff --git a/pxf/src/test/resources/map-keys/bool-keys.pxf b/testdata/map-keys/bool-keys.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/bool-keys.pxf rename to testdata/map-keys/bool-keys.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-bare-keys.expected.pxf b/testdata/map-keys/fmt-bare-keys.expected.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-bare-keys.expected.pxf rename to testdata/map-keys/fmt-bare-keys.expected.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-bare-keys.pxf b/testdata/map-keys/fmt-bare-keys.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-bare-keys.pxf rename to testdata/map-keys/fmt-bare-keys.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-dotted-keys.expected.pxf b/testdata/map-keys/fmt-dotted-keys.expected.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-dotted-keys.expected.pxf rename to testdata/map-keys/fmt-dotted-keys.expected.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-dotted-keys.pxf b/testdata/map-keys/fmt-dotted-keys.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-dotted-keys.pxf rename to testdata/map-keys/fmt-dotted-keys.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-keyword-keys.expected.pxf b/testdata/map-keys/fmt-keyword-keys.expected.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-keyword-keys.expected.pxf rename to testdata/map-keys/fmt-keyword-keys.expected.pxf diff --git a/pxf/src/test/resources/map-keys/fmt-keyword-keys.pxf b/testdata/map-keys/fmt-keyword-keys.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/fmt-keyword-keys.pxf rename to testdata/map-keys/fmt-keyword-keys.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-lower-f.pxf b/testdata/map-keys/invalid/identifier-lower-f.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-lower-f.pxf rename to testdata/map-keys/invalid/identifier-lower-f.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-lower-t.pxf b/testdata/map-keys/invalid/identifier-lower-t.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-lower-t.pxf rename to testdata/map-keys/invalid/identifier-lower-t.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-title-False.pxf b/testdata/map-keys/invalid/identifier-title-False.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-title-False.pxf rename to testdata/map-keys/invalid/identifier-title-False.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-title-True.pxf b/testdata/map-keys/invalid/identifier-title-True.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-title-True.pxf rename to testdata/map-keys/invalid/identifier-title-True.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-upper-F.pxf b/testdata/map-keys/invalid/identifier-upper-F.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-upper-F.pxf rename to testdata/map-keys/invalid/identifier-upper-F.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-upper-FALSE.pxf b/testdata/map-keys/invalid/identifier-upper-FALSE.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-upper-FALSE.pxf rename to testdata/map-keys/invalid/identifier-upper-FALSE.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-upper-T.pxf b/testdata/map-keys/invalid/identifier-upper-T.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-upper-T.pxf rename to testdata/map-keys/invalid/identifier-upper-T.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-upper-TRUE.pxf b/testdata/map-keys/invalid/identifier-upper-TRUE.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-upper-TRUE.pxf rename to testdata/map-keys/invalid/identifier-upper-TRUE.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/identifier-word-yes.pxf b/testdata/map-keys/invalid/identifier-word-yes.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/identifier-word-yes.pxf rename to testdata/map-keys/invalid/identifier-word-yes.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/string-lower-t.pxf b/testdata/map-keys/invalid/string-lower-t.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/string-lower-t.pxf rename to testdata/map-keys/invalid/string-lower-t.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/string-one-1.pxf b/testdata/map-keys/invalid/string-one-1.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/string-one-1.pxf rename to testdata/map-keys/invalid/string-one-1.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/string-upper-TRUE.pxf b/testdata/map-keys/invalid/string-upper-TRUE.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/string-upper-TRUE.pxf rename to testdata/map-keys/invalid/string-upper-TRUE.pxf diff --git a/pxf/src/test/resources/map-keys/invalid/string-zero-0.pxf b/testdata/map-keys/invalid/string-zero-0.pxf similarity index 100% rename from pxf/src/test/resources/map-keys/invalid/string-zero-0.pxf rename to testdata/map-keys/invalid/string-zero-0.pxf diff --git a/pxf/src/test/resources/corpus/nulls/README.md b/testdata/nulls/README.md similarity index 86% rename from pxf/src/test/resources/corpus/nulls/README.md rename to testdata/nulls/README.md index e892717..e40dbbc 100644 --- a/pxf/src/test/resources/corpus/nulls/README.md +++ b/testdata/nulls/README.md @@ -6,9 +6,11 @@ draft `-01` § Booleans, Null, and Identifier Values). A `null`: - on a singular field with explicit presence (a message field, wrappers included, a proto3 `optional` scalar, a oneof member, a proto2 - `optional` scalar) MUST leave it unset, and it is present-but-null. It - counts as present for `(pxf.required)`, and `(pxf.default)` is not - applied; + `optional` scalar) MUST leave it unset, and it is present-but-null, so + `(pxf.default)` is not applied; +- on a field with `(pxf.required)` or `@required` MUST be rejected, + whatever its presence: a required field holds a value, so neither + absent nor null satisfies it; - on a proto3 scalar declared without `optional` MUST be rejected: the field has no unset state, so `null` would read back as its zero value; - on a proto2 `required` field MUST be rejected: leaving it unset makes @@ -19,7 +21,9 @@ draft `-01` § Booleans, Null, and Identifier Values). A `null`: also writes a sibling MUST be rejected. The owner settled this on #339 (2026-10-27, option 3), and the proto2 -`required` and map-value cases on #379 and #380 (2026-10-28). +`required` and map-value cases on #379 and #380 (2026-10-28). On +2026-11-04 the owner reversed #339's rule that `null` satisfies +`(pxf.required)`: it is now rejected there. Every fixture binds against [`nulls.proto`](nulls.proto) (package `nulls.v1`, message `Holder` unless the table names another), or @@ -50,7 +54,8 @@ and | `oneof-scalar.pxf` | accept | `ks = null` on a `string` oneof member leaves the oneof `k` unset. | | `oneof-message.pxf` | accept | `km = null` leaves the oneof `k` unset. | | `proto2-scalar.pxf` | accept | `p = null` on a proto2 `optional int32` (`Legacy`) leaves `p` unset. | -| `required-optional.pxf` | accept | `r = null` on an `optional int32` with `(pxf.required)` (`Required`): present, and unset. | +| `err-required-optional.pxf` | reject | `r = null` on an `optional int32` with `(pxf.required)` (`Required`): a required field holds a value. | +| `err-required-wrapper.pxf` | reject | `w = null` on an `Int32Value` with `(pxf.required)` (`RequiredWrapper`). | | `default-optional.pxf` | accept | `d = null` on an `optional int32` with `(pxf.default) = "5"` (`Defaulted`): unset, not 5. | | `editions-explicit.pxf` | accept | In an editions file, `null` on a scalar with EXPLICIT presence, the 2023 default (`nulls.ed.v1.Ed`), leaves it unset ([#380](https://github.com/trendvidia/protowire/issues/380)). | | `err-editions-implicit.pxf` | reject | `null` on an editions scalar with `features.field_presence = IMPLICIT`. | diff --git a/pxf/src/test/resources/corpus/nulls/default-optional.pxf b/testdata/nulls/default-optional.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/default-optional.pxf rename to testdata/nulls/default-optional.pxf diff --git a/pxf/src/test/resources/corpus/nulls/editions-explicit.pxf b/testdata/nulls/editions-explicit.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/editions-explicit.pxf rename to testdata/nulls/editions-explicit.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-editions-implicit.pxf b/testdata/nulls/err-editions-implicit.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-editions-implicit.pxf rename to testdata/nulls/err-editions-implicit.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-list-element.pxf b/testdata/nulls/err-list-element.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-list-element.pxf rename to testdata/nulls/err-list-element.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-map-message-value.pxf b/testdata/nulls/err-map-message-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-map-message-value.pxf rename to testdata/nulls/err-map-message-value.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-map-value.pxf b/testdata/nulls/err-map-value.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-map-value.pxf rename to testdata/nulls/err-map-value.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-map.pxf b/testdata/nulls/err-map.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-map.pxf rename to testdata/nulls/err-map.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-nested-scalar.pxf b/testdata/nulls/err-nested-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-nested-scalar.pxf rename to testdata/nulls/err-nested-scalar.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-oneof-sibling.pxf b/testdata/nulls/err-oneof-sibling.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-oneof-sibling.pxf rename to testdata/nulls/err-oneof-sibling.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-proto2-required.pxf b/testdata/nulls/err-proto2-required.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-proto2-required.pxf rename to testdata/nulls/err-proto2-required.pxf diff --git a/pxf/src/test/resources/corpus/nulls/err-repeated.pxf b/testdata/nulls/err-repeated.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-repeated.pxf rename to testdata/nulls/err-repeated.pxf diff --git a/testdata/nulls/err-required-optional.pxf b/testdata/nulls/err-required-optional.pxf new file mode 100644 index 0000000..2a54986 --- /dev/null +++ b/testdata/nulls/err-required-optional.pxf @@ -0,0 +1,7 @@ +@type nulls.v1.Required + +# Reject: null on an optional scalar marked (pxf.required). A required +# field holds a value: neither absent nor null satisfies it, whatever the +# field's presence. + +r = null diff --git a/testdata/nulls/err-required-wrapper.pxf b/testdata/nulls/err-required-wrapper.pxf new file mode 100644 index 0000000..76f126e --- /dev/null +++ b/testdata/nulls/err-required-wrapper.pxf @@ -0,0 +1,6 @@ +@type nulls.v1.RequiredWrapper + +# Reject: null on a wrapper field marked (pxf.required). A message field +# may hold null, but not one that is required: it holds a value. + +w = null diff --git a/pxf/src/test/resources/corpus/nulls/err-scalar.pxf b/testdata/nulls/err-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/err-scalar.pxf rename to testdata/nulls/err-scalar.pxf diff --git a/pxf/src/test/resources/corpus/nulls/message.pxf b/testdata/nulls/message.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/message.pxf rename to testdata/nulls/message.pxf diff --git a/pxf/src/test/proto/nulls/nulls.proto b/testdata/nulls/nulls.proto similarity index 80% rename from pxf/src/test/proto/nulls/nulls.proto rename to testdata/nulls/nulls.proto index f7ee6c9..dd62a1f 100644 --- a/pxf/src/test/proto/nulls/nulls.proto +++ b/testdata/nulls/nulls.proto @@ -30,11 +30,16 @@ message Holder { map mm = 9; } -// A null optional scalar counts as present for (pxf.required). +// A required field holds a value: null on it is rejected, even where +// the field's presence would let it hold null. message Required { optional int32 r = 1 [(pxf.required) = true]; } +message RequiredWrapper { + google.protobuf.Int32Value w = 1 [(pxf.required) = true]; +} + // A null optional scalar is not given its (pxf.default). message Defaulted { optional int32 d = 1 [(pxf.default) = "5"]; diff --git a/pxf/src/test/resources/corpus/nulls/nulls_editions.proto b/testdata/nulls/nulls_editions.proto similarity index 100% rename from pxf/src/test/resources/corpus/nulls/nulls_editions.proto rename to testdata/nulls/nulls_editions.proto diff --git a/pxf/src/test/proto/nulls/nulls_proto2.proto b/testdata/nulls/nulls_proto2.proto similarity index 100% rename from pxf/src/test/proto/nulls/nulls_proto2.proto rename to testdata/nulls/nulls_proto2.proto diff --git a/pxf/src/test/resources/corpus/nulls/oneof-message.pxf b/testdata/nulls/oneof-message.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/oneof-message.pxf rename to testdata/nulls/oneof-message.pxf diff --git a/pxf/src/test/resources/corpus/nulls/oneof-scalar.pxf b/testdata/nulls/oneof-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/oneof-scalar.pxf rename to testdata/nulls/oneof-scalar.pxf diff --git a/pxf/src/test/resources/corpus/nulls/optional-scalar.pxf b/testdata/nulls/optional-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/optional-scalar.pxf rename to testdata/nulls/optional-scalar.pxf diff --git a/pxf/src/test/resources/corpus/nulls/proto2-scalar.pxf b/testdata/nulls/proto2-scalar.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/proto2-scalar.pxf rename to testdata/nulls/proto2-scalar.pxf diff --git a/pxf/src/test/resources/corpus/nulls/wrapper.pxf b/testdata/nulls/wrapper.pxf similarity index 100% rename from pxf/src/test/resources/corpus/nulls/wrapper.pxf rename to testdata/nulls/wrapper.pxf diff --git a/testdata/repeated/README.md b/testdata/repeated/README.md new file mode 100644 index 0000000..2c24a17 --- /dev/null +++ b/testdata/repeated/README.md @@ -0,0 +1,26 @@ +# Repeated-field spellings + +Cross-port fixtures for how a repeated field is written (draft `-01` +§ Entries and Keys; issue +[#337](https://github.com/trendvidia/protowire/issues/337)). A repeated +field is written with a list value. It may be bound more than once, and +the lists concatenate. A scalar or a block bound to a repeated field is +an error. The keyed surface form of a keyed repeated field is the one +exception; [`../keyed/`](../keyed/) covers it. + +All fixtures bind against `test.v1.AllTypes` in +[`../test.proto`](../test.proto). `cmd/pxf`'s `repeated_test.go` drives +them through `pxf validate` against the reference implementation. + +| File | Expected | +|---|---| +| `concat-lists.pxf` | accept; `repeated_string` is `a, b, c` and `repeated_nested` is `n1, n2` | +| `err-scalar-entries.pxf` | reject | +| `err-block-entries.pxf` | reject | +| `err-assigned-block.pxf` | reject | + +Measured before the text changed (issue #337): protowire-go v1.8.1 and +protowire-dart both concatenate repeated list bindings. Go rejects all +three error fixtures' spellings. Dart rejects the scalar and `= { }` +forms but accepts repeated blocks, and is tracked to reject them in +[protowire-dart#26](https://github.com/trendvidia/protowire-dart/issues/26). diff --git a/testdata/repeated/concat-lists.pxf b/testdata/repeated/concat-lists.pxf new file mode 100644 index 0000000..bb2725a --- /dev/null +++ b/testdata/repeated/concat-lists.pxf @@ -0,0 +1,10 @@ +# Entries and Keys: a repeated field MAY be bound more than once with a list +# value. The elements concatenate in document order. MUST be accepted: +# repeated_string = ["a", "b", "c"], repeated_nested names = ["n1", "n2"]. + +@type test.v1.AllTypes + +repeated_string = ["a"] +repeated_nested = [{ name = "n1" }] +repeated_string = ["b", "c"] +repeated_nested = [{ name = "n2" value = 2 }] diff --git a/testdata/repeated/err-assigned-block.pxf b/testdata/repeated/err-assigned-block.pxf new file mode 100644 index 0000000..4c255ee --- /dev/null +++ b/testdata/repeated/err-assigned-block.pxf @@ -0,0 +1,7 @@ +# Entries and Keys: a repeated message field assigned a single block. MUST be +# rejected, like the block-tail form it abbreviates — the element goes in a +# list: repeated_nested = [{ name = "n1" }]. + +@type test.v1.AllTypes + +repeated_nested = { name = "n1" } diff --git a/testdata/repeated/err-block-entries.pxf b/testdata/repeated/err-block-entries.pxf new file mode 100644 index 0000000..75f369d --- /dev/null +++ b/testdata/repeated/err-block-entries.pxf @@ -0,0 +1,9 @@ +# Entries and Keys: a repeated message field written as repeated blocks. +# MUST be rejected — outside the keyed surface form of a keyed repeated +# field, a block on a repeated field is an error. The list form is +# repeated_nested = [{ name = "n1" }, { name = "n2" }]. + +@type test.v1.AllTypes + +repeated_nested { name = "n1" } +repeated_nested { name = "n2" } diff --git a/testdata/repeated/err-scalar-entries.pxf b/testdata/repeated/err-scalar-entries.pxf new file mode 100644 index 0000000..c32abfc --- /dev/null +++ b/testdata/repeated/err-scalar-entries.pxf @@ -0,0 +1,7 @@ +# Entries and Keys: a repeated field bound to a scalar. MUST be rejected — +# a repeated field is written with a list value: repeated_string = ["a", "b"]. + +@type test.v1.AllTypes + +repeated_string = "a" +repeated_string = "b" diff --git a/pxf/src/test/resources/corpus/strings/README.md b/testdata/strings/README.md similarity index 100% rename from pxf/src/test/resources/corpus/strings/README.md rename to testdata/strings/README.md diff --git a/pxf/src/test/resources/corpus/strings/err-raw-newline.pxf b/testdata/strings/err-raw-newline.pxf similarity index 100% rename from pxf/src/test/resources/corpus/strings/err-raw-newline.pxf rename to testdata/strings/err-raw-newline.pxf diff --git a/pxf/src/test/resources/corpus/strings/escaped-newline.pxf b/testdata/strings/escaped-newline.pxf similarity index 100% rename from pxf/src/test/resources/corpus/strings/escaped-newline.pxf rename to testdata/strings/escaped-newline.pxf diff --git a/pxf/src/test/resources/corpus/strings/newline-after-string.pxf b/testdata/strings/newline-after-string.pxf similarity index 100% rename from pxf/src/test/resources/corpus/strings/newline-after-string.pxf rename to testdata/strings/newline-after-string.pxf diff --git a/pxf/src/test/proto/strings/strings.proto b/testdata/strings/strings.proto similarity index 100% rename from pxf/src/test/proto/strings/strings.proto rename to testdata/strings/strings.proto diff --git a/pxf/src/test/resources/corpus/strings/triple-quoted.pxf b/testdata/strings/triple-quoted.pxf similarity index 100% rename from pxf/src/test/resources/corpus/strings/triple-quoted.pxf rename to testdata/strings/triple-quoted.pxf diff --git a/testdata/test.proto b/testdata/test.proto new file mode 100644 index 0000000..4904a81 --- /dev/null +++ b/testdata/test.proto @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 TrendVidia, LLC. + +syntax = "proto3"; +package test.v1; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +message AllTypes { + string string_field = 1; + int32 int32_field = 2; + int64 int64_field = 3; + uint32 uint32_field = 4; + uint64 uint64_field = 5; + float float_field = 6; + double double_field = 7; + bool bool_field = 8; + bytes bytes_field = 9; + Status enum_field = 10; + Nested nested_field = 11; + repeated string repeated_string = 12; + repeated Nested repeated_nested = 13; + map string_map = 14; + map nested_map = 15; + map int_map = 16; + google.protobuf.Timestamp ts_field = 17; + google.protobuf.Duration dur_field = 18; + oneof choice { + string text_choice = 19; + int32 number_choice = 20; + } + google.protobuf.StringValue nullable_string = 21; + google.protobuf.Int32Value nullable_int = 22; + google.protobuf.BoolValue nullable_bool = 23; +} + +enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_ACTIVE = 1; + STATUS_INACTIVE = 2; +} + +message Nested { + string name = 1; + int32 value = 2; +}