diff --git a/build.gradle b/build.gradle index ca97d748f..f065399dc 100644 --- a/build.gradle +++ b/build.gradle @@ -214,6 +214,7 @@ dependencies { implementation libs.asm.util // ASM utilities implementation libs.icu4j // Unicode support implementation libs.jsoup // HTML5 parsing for HTML::Content::Extractor + implementation libs.jing // RELAX NG validation for XML::LibXML implementation libs.snakeyaml.engine // YAML processing implementation libs.tomlj // TOML processing implementation libs.commons.csv // CSV processing @@ -226,6 +227,8 @@ dependencies { implementation libs.sqlite.jdbc // SQLite JDBC driver implementation libs.bcprov // Bouncy Castle crypto (SHA-3, Keccak, etc.) implementation libs.bcpkix // Bouncy Castle PEM/PKCS parsing + implementation libs.snappy.java // Official Sereal Java codec compression + implementation libs.zstd.jni // Official Sereal Java codec compression implementation 'org.jruby.joni:joni:2.2.7' // Stack-safe recursive regex backend implementation 'io.netty:netty-codec-http:4.1.115.Final' // Netty HTTP codec for PSGI server diff --git a/dev/design/jcpan-compiler-tooling-followup.md b/dev/design/jcpan-compiler-tooling-followup.md new file mode 100644 index 000000000..9ff51562b --- /dev/null +++ b/dev/design/jcpan-compiler-tooling-followup.md @@ -0,0 +1,76 @@ +# jcpan compiler and tooling follow-up + +## Goal + +Remove shared PerlOnJava compiler/runtime/tooling blockers encountered while testing Data::Checks, POE::Component::MessageQueue, Net::Server::POP3::Skeleton, Mail::BIMI, Imager::Album, Test::Mimic::Recorder, Fuse::Filesys::Virtual, and their dependencies. Distribution preferences are deliberately avoided. + +## Implementation + +### Compiler and runtime semantics + +- Tied `@ISA` values are fetched through the tie interface during method resolution. +- Constant subroutines preserve their scalar-reference stash proxy without losing the callable CODE slot. +- `use open` defaults are read from lexical `%^H` call-site snapshots instead of leaking through a process-global `${^OPEN}` value. + +### Bundled module tooling + +- `DynaLoader::dl_load_flags` is available to XS-style loaders. +- HTTP::Tiny installs each convenience verb once and returns its standard status-599 response for transport failures. +- Time::HiRes provides the requested `ualarm` export. +- XML::LibXML::RelaxNG uses Jing for schema compilation and validation. +- Sereal::Encoder and Sereal::Decoder use the official Booking.com Java codec with Snappy and Zstandard support. +- Cache::FastMmap has a JVM-backed compatibility implementation and preserves the distribution's Perl serialization, expiry, and callback layer. Its map is process-local; the share file is created for API compatibility. +- Crypt::OpenSSL::X509 and Crypt::OpenSSL::Verify use the existing Bouncy Castle/JCA stack rather than adding another crypto implementation. + +The imported Sereal Java sources are based on upstream commit `9ad81cf3023ccc456c2accd83bea2c2803a82e16`. + +## System-Perl exclusions + +The following are not treated as PerlOnJava regressions because their current distributions or tests fail under the available system Perl or require unavailable platform facilities: + +- Data::Checks: its fresh system-Perl dependency set lacks `builtin.pm`. +- Net::Server::POP3::Skeleton: the release metadata points `VERSION_FROM` at a nonexistent `lib/Tk/Carp.pm`. +- Imager::Album: its system-Perl dependency closure lacks Imager and relies on the legacy Gtk/display stack. +- Test::Mimic::Recorder: its own test fails on system Perl with a hard-coded reference-history assumption. +- Fuse::Filesys::Virtual: Fuse 0.16 refuses to configure on Darwin without OSXFUSE. +- POE::Component::MessageQueue: its remaining bind test cannot run in this sandbox; a minimal system-Perl bind fails with the same platform restriction. + +## Progress Tracking + +### Current Status: implementation complete; PR ready for review + +### Completed Phases + +- [x] Phase 1: classify upstream/system failures (2026-08-15) + - Reproduced each exclusion with system Perl or the relevant platform configuration step. +- [x] Phase 2: compiler/runtime fixes (2026-08-15) + - Fixed tied inheritance, constant stash proxies, and lexical open-layer handling. +- [x] Phase 3: reusable Java module bridges (2026-08-15) + - Added Sereal, cache, OpenSSL/X509, and RelaxNG support using existing or upstream Java libraries. +- [x] Phase 4: regression coverage (2026-08-15) + - Added focused unit tests and validated new Perl semantics with system Perl before PerlOnJava. +- [x] Phase 5: final verification (2026-08-15) + - Full `make` passed. + - Mail::BIMI passed all 31 test programs and 82 assertions; network- and author-only tests skipped as expected. + - Preserved the platform-default I/O layer when no lexical `open` pragma is + active, including Windows `:crlf`, without consulting the leaked + process-global `${^OPEN}` value. +- [x] Phase 6: pull request and CI (2026-08-15) + - Opened PR #962 from `fix/jcpan-compiler-tooling-followup`. + - GitHub Actions passed on Ubuntu and Windows; the Windows run specifically + confirmed the platform-default newline fix in `pipe_jperl_shebang.t`. + +### Next Steps + +1. Review PR #962. +2. Merge after approval. + +### Open Questions + +- A future Cache::FastMmap implementation could provide true cross-process mmap sharing; current requested tests only require same-process behavior. + +## References + +- [Sereal](https://github.com/Sereal/Sereal) +- [Jing and Trang](https://github.com/relaxng/jing-trang) +- Skills: `debug-perlonjava`, `port-cpan-module`, `port-native-module` diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b06e06cfc..9c1999f26 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,10 +6,13 @@ commons-csv = "1.14.1" commonmark = "0.29.0" icu4j = "78.3" jsoup = "1.23.1" +jing = "20241231" junit-jupiter = "6.1.3" snakeyaml-engine = "3.1.1" +snappy-java = "1.1.10.8" sqlite-jdbc = "3.53.2.1" tomlj = "1.1.1" +zstd-jni = "1.5.7-8" [libraries] asm = { module = "org.ow2.asm:asm", version.ref = "asm" } @@ -25,12 +28,15 @@ commonmark-tables = { module = "org.commonmark:commonmark-ext-gfm-tables", versi commonmark-task-list = { module = "org.commonmark:commonmark-ext-task-list-items", version.ref = "commonmark" } icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +jing = { module = "org.relaxng:jing", version.ref = "jing" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit-jupiter" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit-jupiter" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" } snakeyaml-engine = { module = "org.snakeyaml:snakeyaml-engine", version.ref = "snakeyaml-engine" } +snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy-java" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } tomlj = { module = "org.tomlj:tomlj", version.ref = "tomlj" } +zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstd-jni" } [plugins] cyclonedx = "org.cyclonedx.bom:2.3.0" diff --git a/src/main/java/com/booking/sereal/ByteArray.java b/src/main/java/com/booking/sereal/ByteArray.java new file mode 100644 index 000000000..cb033e30e --- /dev/null +++ b/src/main/java/com/booking/sereal/ByteArray.java @@ -0,0 +1,32 @@ +package com.booking.sereal; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class ByteArray { + public byte[] array; + public int start; + public int length; + + public ByteArray(byte[] array) { + this(array, 0, array.length); + } + + public ByteArray(byte[] array, int length) { + this(array, 0, length); + } + + public ByteArray(byte[] array, int start, int length) { + this.array = array; + this.start = start; + this.length = length; + } + + public ByteArray(ByteBuffer buffer) { + this(buffer.array(), buffer.limit()); + } + + public void ensure(int size) { + if (size > array.length) array = Arrays.copyOf(array, size * 3 / 2); + } +} diff --git a/src/main/java/com/booking/sereal/DeSereal.java b/src/main/java/com/booking/sereal/DeSereal.java new file mode 100644 index 000000000..ffe254fcc --- /dev/null +++ b/src/main/java/com/booking/sereal/DeSereal.java @@ -0,0 +1,26 @@ +package com.booking.sereal; + +import java.io.File; +import java.io.IOException; + +public class DeSereal { + /** + * @param args command arguments + * @throws IOException Indicated file cannot accessed. + * @throws SerealException Sereal data cannot be processed. + */ + public static void main(String[] args) throws IOException, SerealException { + if (args.length == 0) { + throw new UnsupportedOperationException("Usage: DeSereal test_data"); + } + + DecoderOptions decoder_options = + new DecoderOptions().perlReferences(true).perlAliases(true).preferLatin1(true); + + Decoder dec = new Decoder(decoder_options); + final File target = new File(args[0]).getCanonicalFile(); // to absorb ".." in paths + // dec.log.setLevel( Level.FINE ); + Object data = Utils.decodeFile(dec, target); + System.out.println(Utils.dump(data)); + } +} diff --git a/src/main/java/com/booking/sereal/Decoder.java b/src/main/java/com/booking/sereal/Decoder.java new file mode 100644 index 000000000..edca41ed6 --- /dev/null +++ b/src/main/java/com/booking/sereal/Decoder.java @@ -0,0 +1,914 @@ +package com.booking.sereal; + +import com.booking.sereal.impl.RefpMap; +import com.github.luben.zstd.Zstd; + +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import org.xerial.snappy.Snappy; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +/** + * Sereal decoder with Perl-like interface. + *

+ * This class can be used to decoder Perl-like data-structures: (boxed) primitive types, strings, arrays + * and maps. + */ +public class Decoder implements SerealHeader { + + private static final DecoderOptions DEFAULT_OPTIONS = new DecoderOptions(); + private static final Charset charset_utf8 = Charset.forName("UTF-8"); + private static final Charset charset_latin1 = Charset.forName("ISO-8859-1"); + private final boolean perlRefs; + private final boolean perlAlias; + private final boolean preserveUndef; + private final boolean refuseSnappy; + private final boolean refuseZlib; + private final boolean refuseZstd; + private final boolean preferLatin1; + private final boolean forceJavaStringForByteArrayValues; + private final boolean refuseObjects; + private final boolean stripObjects; + private final TypeMapper typeMapper; + private final boolean useObjectArray; + private final int decodeBufferSize; + private final int maxSize; + + private final int maxRecursionDepth; + private final int maxNumMapEntries; + private final int maxNumArrayEntries; + private final int maxStringLength; + + private byte[] data; + private int position, end; + private ByteArray originalData; + // where we track items for REFP purposes + private RefpMap tracked = new RefpMap(); + private int protocolVersion = -1; + private int encoding = -1; + private int baseOffset = Integer.MAX_VALUE; + private long userHeaderPosition = -1; + private long userHeaderSize = -1; + private Inflater inflater; + + private int recursionDepth = 0; + + /** Create a new Decoder with default options. */ + public Decoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create a new Decoder with the specified options. + * + * @param options {@link DecoderOptions} to apply. + */ + public Decoder(DecoderOptions options) { + perlRefs = options.perlReferences(); + perlAlias = options.perlAliases(); + preserveUndef = options.preserveUndef(); + refuseSnappy = options.refuseSnappy(); + refuseZlib = options.refuseZlib(); + refuseZstd = options.refuseZstd(); + preferLatin1 = options.preferLatin1(); + forceJavaStringForByteArrayValues = options.forceJavaStringForByteArrayValues(); + refuseObjects = options.refuseObjects(); + stripObjects = options.stripObjects(); + typeMapper = options.typeMapper(); + useObjectArray = typeMapper.useObjectArray(); + decodeBufferSize = options.bufferSize(); + maxSize = options.maxBufferSize(); + + maxRecursionDepth = options.maxRecursionDepth(); + maxNumMapEntries = options.maxNumMapEntries(); + maxNumArrayEntries = options.maxNumArrayEntries(); + maxStringLength = options.maxStringLength(); + } + + private void checkHeader() throws SerealException { + + if ((end - position) < 4) { + throw new SerealException("Invalid Sereal header: too few bytes"); + } + + int magic = + ((int) (data[position] & 0xff) << 24) + + ((int) (data[position + 1] & 0xff) << 16) + + ((int) (data[position + 2] & 0xff) << 8) + + ((int) (data[position + 3] & 0xff) << 0); + position += 4; + if (magic != MAGIC && magic != MAGIC_V3) { + throw new SerealException( + String.format("Invalid Sereal header (%08x): doesn't match magic", magic)); + } + } + + private void checkHeaderSuffix() { + long suffix_size = read_varint(); + long basePosition = position; + + userHeaderSize = 0; + if (suffix_size > 0) { + byte bitfield = data[position++]; + + if ((bitfield & 0x01) == 0x01) { + userHeaderPosition = position; + userHeaderSize = suffix_size - 1; + } + } + + // skip everything in the optional suffix part + position = (int) (basePosition + suffix_size); + } + + private void checkNoEOD() throws SerealException { + + if ((end - position) <= 0) { + throw new SerealException("Unexpected end of data at byte " + position); + } + } + + private void checkProtoAndFlags() throws SerealException { + + if ((end - position) < 1) { + throw new SerealException("Invalid Sereal header: no protocol/version byte"); + } + + int protoAndFlags = data[position++]; + protocolVersion = protoAndFlags & 15; // 4 bits for version + + if (protocolVersion < 0 || protocolVersion > 4) { + throw new SerealException( + String.format("Invalid Sereal header: unsupported protocol version %d", protocolVersion)); + } + + encoding = (protoAndFlags & ~15) >> 4; + if ((encoding == 1 || encoding == 2) && refuseSnappy) { + throw new SerealException("Unsupported encoding: Snappy"); + } else if (encoding == 3 && refuseZlib) { + throw new SerealException("Unsupported encoding: Zlib"); + } else if (encoding == 4 && refuseZstd) { + throw new SerealException("Unsupported encoding: Zstd"); + } else if (encoding == 4 && protocolVersion < 4) { + throw new SerealException( + "Unsupported encoding zstd for protocol version " + protocolVersion); + } else if (encoding < 0 || encoding > 4) { + throw new SerealException("Unsupported encoding: unknown"); + } + } + + /** + * Indicates if the Sereal document has an header. + * + * @return {@code true} if the Sereal document has an header, {@code false} otherwise + * + * @throws SerealException if header cannot be parsed. + */ + public boolean hasHeader() throws SerealException { + parseHeader(); + + return userHeaderSize > 0; + } + + /** + * Size of the Sereal header. + * + * @return Size of the Sereal header, 0 if there is no header. + * + * @throws SerealException if header cannot be parsed. + */ + public long headerSize() throws SerealException { + parseHeader(); + + return userHeaderSize > 0 ? userHeaderSize : 0; + } + + /** + * Decode the Sereal document header and returns the decoded value. + * + * @return returns the header decoded value. + * + * @throws SerealException if header cannot be parsed. + */ + public Object decodeHeader() throws SerealException { + parseHeader(); + + if (userHeaderSize <= 0) throw new SerealException("Sereal user header not present"); + byte[] originalData = data; + int originalPosition = position, originalSize = end; + try { + data = originalData; + end = (int) (userHeaderPosition + userHeaderSize); + position = (int) userHeaderPosition; + + return readSingleValue(); + } finally { + data = originalData; + end = originalSize; + position = originalPosition; + resetTracked(); + } + } + + private void parseHeader() throws SerealException { + if (userHeaderSize >= 0) return; + + checkHeader(); + checkProtoAndFlags(); + checkHeaderSuffix(); + } + + /** + * Decode the Sereal document body and returns the decoded value. + * + * @return the decoded value. + * + * @throws SerealException if data cannot be parsed. + */ + public Object decode() throws SerealException { + + recursionDepth = 0; + + if (data == null) { + throw new SerealException("No data set"); + } + + parseHeader(); + + if (encoding != 0) { + if (encoding == 1 || encoding == 2) uncompressSnappy(); + else if (encoding == 3) uncompressZlib(); + else if (encoding == 4) uncompressZstd(); + if (protocolVersion == 1) baseOffset = 0; + else + // because offsets start at 1 + baseOffset = -1; + } else { + if (protocolVersion == 1) baseOffset = 0; + else + // because offsets start at 1 + baseOffset = position - 1; + } + Object out; + try { + out = readSingleValue(); + } catch (StackOverflowError error) { + throw new SerealException("StackOverflowError: Reached recursion limit during deserialization"); + } + + return out; + } + + private void uncompressSnappy() throws SerealException { + int len = originalData.length - (position - originalData.start); + int pos = protocolVersion == 1 ? position : originalData.start; + + if (encoding == 2) { + len = (int) read_varint(); + } + + byte[] uncompressed; + try { + if (!Snappy.isValidCompressedBuffer(originalData.array, position, len)) { + throw new SerealException("Invalid snappy data"); + } + int uncompressedLength = Snappy.uncompressedLength(originalData.array, position, len); + if (uncompressedLength > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + uncompressed = new byte[pos + uncompressedLength]; + Snappy.uncompress(originalData.array, position, len, uncompressed, pos); + } catch (IOException e) { + throw new SerealException(e); + } + this.data = uncompressed; + this.position = pos; + this.end = uncompressed.length; + } + + private void uncompressZlib() throws SerealException { + if (inflater == null) { + inflater = new Inflater(); + } + inflater.reset(); + + long uncompressedLength = read_varint(); + if (uncompressedLength > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + + long compressedLength = read_varint(); + inflater.setInput(originalData.array, position, (int) compressedLength); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + byte[] buffer = new byte[this.decodeBufferSize]; + while (!inflater.finished()) { + if (outputStream.size() > this.maxSize) { + throw new SerealException("The uncompressed size is larger than the allowed maximum size"); + } else if (outputStream.size() > uncompressedLength) { + throw new SerealException("The uncompressed size is larger than the expected size"); + } + int count = inflater.inflate(buffer); + if (count == 0 ) { + break; + } + outputStream.write(buffer, 0, count); + } + this.data = outputStream.toByteArray(); + } catch (DataFormatException | IOException e) { + throw new SerealException(e); + } + this.position = 0; + this.end = this.data.length; + } + + private void uncompressZstd() throws SerealException { + int len = (int) read_varint(); + + byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len); + long decompressedSize = Zstd.decompressedSize(compressedData); + + if (decompressedSize > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + if (decompressedSize > Integer.MAX_VALUE) { + throw new SerealException("Decompressed size exceeds integer MAX_VALUE: " + decompressedSize); + } + byte[] uncompressed = new byte[(int) decompressedSize]; + long status = Zstd.decompress(uncompressed, compressedData); + if (Zstd.isError(status)) { + throw new SerealException(Zstd.getErrorName(status)); + } + this.data = uncompressed; + this.position = 0; + this.end = uncompressed.length; + } + + /** + * Decode a Sereal ARRAY tag to a native Java arrya + * + * @param length number of items in the array + * @param track we might need to track since array elements could refer to us + */ + private Object[] readNativeArray(int length, int track) throws SerealException { + if (maxNumArrayEntries != 0 && length > maxNumArrayEntries) { + throw new SerealException("Got input array with " + length + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + Object[] out = new Object[length]; + if (track != 0) { // track ourself + track_stuff(track, out); + } + + for (int i = 0; i < length; i++) { + out[i] = readSingleValue(); + } + + return out; + } + + /** + * Decode a Sereal ARRAY tag to a native Java List + * + * @param length number of items in the list + * @param track we might need to track since array elements could refer to us + */ + private List readList(int length, int track) throws SerealException { + + if (maxNumArrayEntries != 0 && length > maxNumArrayEntries) { + throw new SerealException("Got input array with " + length + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + List out = typeMapper.makeArray(length); + if (track != 0) { // track ourself + track_stuff(track, out); + } + + for (int i = 0; i < length; i++) { + out.add(readSingleValue()); + } + + return out; + } + + /** + * Reads a byte array, but was called read_binary in C, so for grepping purposes I kept the name + * + *

For some reason we call them Latin1Strings. + */ + private byte[] read_binary() { + int length = (int) read_varint(); + byte[] out = Arrays.copyOfRange(data, position, position + length); + + position += length; + + return out; + } + + private Map readMap(int num_keys, int track) throws SerealException { + + if (maxNumMapEntries != 0 && num_keys > maxNumMapEntries) { + throw new SerealException("Got input hash with " + num_keys + " entries, but the configured maximum is just " + maxNumMapEntries); + } + + Map hash = typeMapper.makeMap((int) num_keys); + if (track != 0) { // track ourself + track_stuff(track, hash); + } + + for (int i = 0; i < num_keys; i++) { + String key = readString(); + Object val = readSingleValue(); + hash.put(key, val); + } + + return hash; + } + + private Object get_tracked_item() { + long offset = read_varint(); + return tracked.get(offset); + } + + // top bit set (0x80) means next byte is 7 bits more more varint + private long read_varint() { + + long uv = 0; + int lshift = 0; + + byte b = data[position++]; + while ((position < end) && (b < 0)) { + uv |= ((long) b & 127) << lshift; // add 7 bits + lshift += 7; + b = data[position++]; + } + uv |= (long) b << lshift; // add final (or first if there is only 1) + + return uv; + } + + private Object readSingleValue() throws SerealException { + + checkNoEOD(); + + byte tag = data[position++]; + + int track = 0; + if ((tag & SRL_HDR_TRACK_FLAG) != 0) { + tag = (byte) (tag & ~SRL_HDR_TRACK_FLAG); + track = position - 1 - baseOffset; + } + + Object out; + + if (tag <= SRL_HDR_POS_HIGH) { + out = (long) tag; + } else if (tag <= SRL_HDR_NEG_HIGH) { + out = (long) (tag - 32); + } else if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + byte[] short_binary = read_short_binary(tag); + if (forceJavaStringForByteArrayValues) { + out = new String(short_binary); + } else { + out = preferLatin1 ? new Latin1String(short_binary) : short_binary; + } + } else if ((tag & SRL_HDR_HASHREF) == SRL_HDR_HASHREF) { + depthIncrement(); + + Map hash = readMap(tag & 0xf, track); + if (perlRefs) { + out = new PerlReference(hash); + } else { + out = hash; + } + + depthDecrement(); + } else if ((tag & SRL_HDR_ARRAYREF) == SRL_HDR_ARRAYREF) { + depthIncrement(); + + Object arr; + if (useObjectArray) { + arr = readNativeArray(tag & 0xf, track); + } else { + arr = readList(tag & 0xf, track); + } + if (perlRefs) { + out = new PerlReference(arr); + } else { + out = arr; + } + + depthDecrement(); + } else { + switch (tag) { + case SRL_HDR_VARINT: + long l = read_varint(); + if (l >= 0) { + out = l; + } else { + // long int greater than Long.MAX_VALUE wrapped around to negative: return a BigInteger + byte[] buffer = new byte[8]; + for (int i = 7; i >= 0; --i) { + buffer[i] = (byte) (l & 0xff); + l >>= 8; + } + out = new BigInteger(1, buffer); + } + break; + case SRL_HDR_ZIGZAG: + long zz = read_zigzag(); + out = zz; + break; + case SRL_HDR_FLOAT: + int floatBits = + ((int) (data[position + 3] & 0xff) << 24) + + ((int) (data[position + 2] & 0xff) << 16) + + ((int) (data[position + 1] & 0xff) << 8) + + ((int) (data[position] & 0xff) << 0); + position += 4; + float f = Float.intBitsToFloat(floatBits); + out = f; + break; + case SRL_HDR_DOUBLE: + long doubleBits = + ((long) (data[position + 7] & 0xff) << 56) + + ((long) (data[position + 6] & 0xff) << 48) + + ((long) (data[position + 5] & 0xff) << 40) + + ((long) (data[position + 4] & 0xff) << 32) + + ((long) (data[position + 3] & 0xff) << 24) + + ((long) (data[position + 2] & 0xff) << 16) + + ((long) (data[position + 1] & 0xff) << 8) + + ((long) (data[position] & 0xff) << 0); + position += 8; + double d = Double.longBitsToDouble(doubleBits); + out = d; + break; + case SRL_HDR_TRUE: + out = true; + break; + case SRL_HDR_FALSE: + out = false; + break; + case SRL_HDR_UNDEF: + if (preserveUndef) out = new PerlUndef(); + else out = null; + break; + case SRL_HDR_CANONICAL_UNDEF: + if (preserveUndef) out = PerlUndef.CANONICAL; + else out = null; + break; + case SRL_HDR_BINARY: + byte[] bytes = read_binary(); + if (forceJavaStringForByteArrayValues) { + out = new String(bytes); + } else { + out = preferLatin1 ? new Latin1String(bytes) : bytes; + } + break; + case SRL_HDR_STR_UTF8: + String utf8 = read_UTF8(); + out = utf8; + break; + case SRL_HDR_REFN: + if (perlRefs) { + PerlReference refn = new PerlReference(null); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, refn); + } + refn.setValue(readSingleValue()); + out = refn; + } else { + depthIncrement(); + out = readSingleValue(); + depthDecrement(); + } + break; + case SRL_HDR_REFP: + long offset_prev = read_varint(); + Object prv_value = tracked.get(offset_prev); + if (prv_value == RefpMap.NOT_FOUND) { + throw new SerealException("REFP to offset " + offset_prev + ", which is not tracked"); + } + Object prev = perlRefs ? new PerlReference(prv_value) : prv_value; + out = prev; + break; + case SRL_HDR_OBJECT: + if (refuseObjects) + throw new SerealException( + String.format( + "Encountered object in input, but the 'refuseObject' option is in effect at offset %d of input", + position)); + Object obj = readObject(); + out = obj; + break; + case SRL_HDR_OBJECTV: + if (refuseObjects) + throw new SerealException( + String.format( + "Encountered object in input, but the 'refuseObject' option is in effect at offset %d of input", + position)); + String className = readStringCopy(); + out = readObject(className); + break; + case SRL_HDR_COPY: + Object copy = read_copy(); + out = copy; + break; + case SRL_HDR_ALIAS: + Object value = get_tracked_item(); + + if (perlAlias) { + out = new PerlAlias(value); + } else { + out = value; + } + break; + case SRL_HDR_WEAKEN: + // so the next thing HAS to be a ref (afaict) which means we can track it + if (perlRefs) { + PerlReference placeHolder = new PerlReference(null); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, placeHolder); + } + placeHolder.setValue(((PerlReference) readSingleValue()).getValue()); + WeakReference wref = new WeakReference(placeHolder); + out = wref; + } else { + Object ref = readSingleValue(); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, ref); + } + WeakReference wref = new WeakReference(ref); + out = wref; + } + break; + case SRL_HDR_HASH: + depthIncrement(); + Object hash = readMap((int) read_varint(), track); + out = hash; + depthDecrement(); + break; + case SRL_HDR_ARRAY: + depthIncrement(); + if (useObjectArray) { + out = readNativeArray((int) read_varint(), track); + } else { + out = readList((int) read_varint(), track); + } + depthDecrement(); + break; + case SRL_HDR_REGEXP: + Pattern pattern = read_regex(); + out = pattern; + break; + case SRL_HDR_PAD: + return readSingleValue(); + default: + throw new SerealException("Tag not supported: " + tag); + } + } + + if (track != 0) { // we double-track arrays ATM (but they just overwrite) + track_stuff(track, out); + } + + return out; + } + + /** + * Read a short binary ISO-8859-1 (latin1) string, the lower bits of the tag hold the length + * + * @param tag the Sereal SHORT_BINARY_* tag + */ + private byte[] read_short_binary(byte tag) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + byte[] buf = Arrays.copyOfRange(data, position, position + length); + position += length; + return buf; + } + + /** + * From the spec: Sometimes it is convenient to be able to reuse a previously emitted sequence in + * the packet to reduce duplication. For instance a data structure with many hashes with the same + * keys. The COPY tag is used for this. Its argument is a varint which is the offset of a + * previously emitted tag, and decoders are to behave as though the tag it references was inserted + * into the packet stream as a replacement for the COPY tag. + * + *

Note, that in this case the track flag is not set. It is assumed the decoder can jump back + * to reread the tag from its location alone. + * + *

Copy tags are forbidden from referring to another COPY tag, and are also forbidden from + * referring to anything containing a COPY tag, with the exception that a COPY tag used as a value + * may refer to an tag that uses a COPY tag for a classname or hash key. + */ + private Object read_copy() throws SerealException { + + int originalPosition = (int) read_varint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + Object copy = readSingleValue(); + position = currentPosition; // go back to where we were + + return copy; + } + + private String readStringCopy() throws SerealException { + int originalPosition = (int) read_varint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + String copy = readString(); + position = currentPosition; // go back to where we were + + return copy; + } + + private String read_UTF8() throws SerealException { + int length = (int) read_varint(); + int originalPosition = position; + + position += length; + + if (maxStringLength != 0 && length > maxStringLength) { + throw new SerealException("Got input string with " + length + " characters, but the configured maximum is just " + maxStringLength); + } + + return new String(data, originalPosition, length, charset_utf8); + } + + private long read_zigzag() { + + long n = read_varint(); + + return (n >>> 1) ^ (-(n & 1)); // note the unsigned right shift + } + + private Pattern read_regex() throws SerealException { + + int flags = 0; + Object str = readSingleValue(); + String regex; + if (str instanceof CharSequence) { + regex = ((CharSequence) str).toString(); + } else if (str instanceof byte[]) { + regex = (new Latin1String((byte[]) str)).toString(); + } else { + throw new SerealException("Regex has to be built from a char or byte sequence"); + } + + // now read modifiers + byte tag = data[position++]; + if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + while (length-- > 0) { + byte value = data[position++]; + switch (value) { + case 'm': + flags = flags | Pattern.MULTILINE; + break; + case 's': + flags = flags | Pattern.DOTALL; + break; + case 'i': + flags = flags | Pattern.CASE_INSENSITIVE; + break; + case 'x': + flags = flags | Pattern.COMMENTS; + break; + case 'p': + // ignored + break; + default: + throw new SerealException("Unknown regex modifier: " + value); + } + } + } else { + throw new SerealException( + "Expecting SRL_HDR_SHORT_BINARY for modifiers of regexp, got: " + tag); + } + + return Pattern.compile(regex, flags); + } + + private Object readObject() throws SerealException { + Object className = readString(); + + return readObject(className.toString()); + } + + private Object readObject(String className) throws SerealException { + Object structure = readSingleValue(); + if (stripObjects) return structure; + Object object = typeMapper.makeObject(className, structure); + return object; + } + + private String readString() throws SerealException { + checkNoEOD(); + + byte tag = data[position++]; + + if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + String string = new String(data, position, length, charset_latin1); + + position += length; + + return string; + } else if (tag == SRL_HDR_BINARY) { + int length = (int) read_varint(); + String string = new String(data, position, length, charset_latin1); + + position += length; + + return string; + } else if (tag == SRL_HDR_STR_UTF8) { + return read_UTF8(); + } else if (tag == SRL_HDR_COPY) { + return readStringCopy(); + } else { + throw new SerealException("Tag " + tag + " is not a string tag"); + } + } + + /** + * Set the Sereal data to be decoded. + *

+ * The caller must not modify the data while it is owned by the decoder. + * + * @param blob Sereal data to decode. + */ + public void setData(ByteArray blob) { + reset(); + originalData = blob; + data = originalData.array; + end = originalData.start + originalData.length; + position = originalData.start; + } + + /** + * Set the Sereal data to be decoded. + *

+ * The caller must not modify the data while it is owned by the decoder. + * + * @param blob Sereal data to decode. + */ + public void setData(byte[] blob) { + reset(); + originalData = new ByteArray(blob); + data = blob; + end = blob.length; + position = 0; + } + + private void track_stuff(int pos, Object ref) { + tracked.put(pos, ref); + } + + private void reset() { + originalData = null; + data = null; + protocolVersion = encoding = -1; + baseOffset = Integer.MAX_VALUE; + userHeaderPosition = userHeaderSize = -1; + resetTracked(); + } + + private void resetTracked() { + tracked.clear(); + } + + private void depthIncrement() throws SerealException { + if (recursionDepth++ > maxRecursionDepth) { + throw new SerealException("Reached recursion limit (" + maxRecursionDepth + ") during deserialization"); + } + } + + private void depthDecrement() { + recursionDepth--; + } + + /** + * Close the decoder to recycle the resources. + *

+ * Returns the native memory used by inflater explicitly before the garbage collector. + */ + public void close() { + if (inflater != null) { + inflater.end(); + inflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/DecoderOptions.java b/src/main/java/com/booking/sereal/DecoderOptions.java new file mode 100644 index 000000000..ef924e05a --- /dev/null +++ b/src/main/java/com/booking/sereal/DecoderOptions.java @@ -0,0 +1,271 @@ +package com.booking.sereal; + +public class DecoderOptions { + private boolean perlRefs = false; + private boolean perlAlias = false; + private boolean preserveUndef = false; + private boolean preferLatin1 = false; + private boolean refuseSnappy = false; + private boolean refuseObjects = false; + private boolean refuseZlib = false; + private boolean refuseZstd = false; + private boolean stripObjects = false; + private boolean forceJavaStringForByteArrayValues = false; + + private int maxRecursionDepth = 10_000; + private int maxNumMapEntries = 0; + private int maxNumArrayEntries = 0; + private int maxStringLength = 0; + + // Size to use on the buffer used to read the data. Defaults to 1KB + private int decodeBufferSize = 1024; + // Maximum size allowed for the data. Defaults to 100MB + private int maxSize = 100 * 1024 * 1024; + + private TypeMapper typeMapper = new DefaultTypeMapper(); + + public boolean perlReferences() { + return perlRefs; + } + + public boolean perlAliases() { + return perlAlias; + } + + public boolean preserveUndef() { + return preserveUndef; + } + + public boolean preferLatin1() { + return preferLatin1; + } + + /** + * If set, the decoder will refuse Snappy-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Snappy is refused, {@code False} otherwise + */ + public boolean refuseSnappy() { + return refuseSnappy; + } + + /** + * If set, the decoder will refuse deserializing any objects in the input stream and + * instead throw an exception. + * + * @return {@code True} if Objects are refused, {@code False} otherwise + */ + public boolean refuseObjects() { + return refuseObjects; + } + + /** + * If set, the decoder will refuse Zlib-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Zlib is refused, {@code False} otherwise + */ + public boolean refuseZlib() { + return refuseZlib; + } + + /** + * If set, the decoder will refuse Zstd-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Zstd is refused, {@code False} otherwise + */ + public boolean refuseZstd() { + return refuseZstd; + } + + public boolean stripObjects() { + return stripObjects; + } + + public boolean forceJavaStringForByteArrayValues() { + return forceJavaStringForByteArrayValues; + } + + public TypeMapper typeMapper() { + return typeMapper; + } + + public int bufferSize() { + return decodeBufferSize; + } + + public int maxBufferSize() { + return maxSize; + } + + /** + * {@link Decoder} is recursive. If you pass it a Sereal document that is deeply + * nested, it will eventually exhaust the Java stack. Therefore, there is a limit on + * the depth of recursion that is accepted. It defaults to 10000 nested calls. You + * may choose to override this value with the {@link DecoderOptions#maxRecursionDepth(int)} option. + * Beware that setting it too high can cause hard crashes. + * + * Do note that the setting is somewhat approximate. Setting it to 10000 may break at + * somewhere between 9997 and 10003 nested structures depending on their types. + * + * It's also important to note that this value doesn't mirror the Perl implementation. Perl and Java + * Sereal wrappers are implemented in a different way. That means that the recursion depth marked + * as max in Perl may be different than the one needed in Java. The same applies for the Encoder + * and Decoder options. + * + * The Java stack can be controlled with different flags (for example -Xss) to change it's size. + * + * @return maximum recursion depth + */ + public int maxRecursionDepth() { + return maxRecursionDepth; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any hash/dictionary (or hash-based object) with more than + * that number of entries. This is to be able to respond quickly to any future + * hash-collision attacks on Perl's hash function. Chances are, you don't want + * or need this. For a gentle introduction to the topic from the cryptographic + * point of view, see Collision attack. + * + * This value can be override with {@link DecoderOptions#maxNumMapEntries(int)} option + * + * @return maximum number of map entries + */ + public int maxNumMapEntries() { + return maxNumMapEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any array with more than that number of entries. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link DecoderOptions#maxNumArrayEntries(int)} option + * + * @return maximum number of array entries + */ + public int maxNumArrayEntries() { + return maxNumArrayEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any string with more than that number of characters. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link DecoderOptions#maxStringLength(int)} option + * + * @return maximum supported string length + */ + public int maxStringLength() { + return maxStringLength; + } + + public DecoderOptions perlReferences(boolean perlReferences) { + this.perlRefs = perlReferences; + + return this; + } + + public DecoderOptions perlAliases(boolean perlAliases) { + this.perlAlias = perlAliases; + + return this; + } + + public DecoderOptions forceJavaStringForByteArrayValues(boolean forceJavaStringForByteArrayValues) { + this.forceJavaStringForByteArrayValues = forceJavaStringForByteArrayValues; + + return this; + } + + public DecoderOptions preserveUndef(boolean preserveUndef) { + this.preserveUndef = preserveUndef; + + return this; + } + + public DecoderOptions preferLatin1(boolean preferLatin1) { + this.preferLatin1 = preferLatin1; + + return this; + } + + public DecoderOptions refuseSnappy(boolean refuseSnappy) { + this.refuseSnappy = refuseSnappy; + + return this; + } + + public DecoderOptions refuseObjects(boolean refuseObjects) { + this.refuseObjects = refuseObjects; + + return this; + } + + public DecoderOptions stripObjects(boolean stripObjects) { + this.stripObjects = stripObjects; + + return this; + } + + public DecoderOptions refuseZlib(boolean refuseZlib) { + this.refuseZlib = refuseZlib; + + return this; + } + + public DecoderOptions refuseZstd(boolean refuseZstd) { + this.refuseZstd = refuseZstd; + + return this; + } + + public DecoderOptions typeMapper(TypeMapper typeMapper) { + this.typeMapper = typeMapper; + + return this; + } + + public DecoderOptions bufferSize(int bufferSize) { + this.decodeBufferSize = bufferSize; + + return this; + } + + public DecoderOptions maxBufferSize(int maxSize) { + this.maxSize = maxSize; + + return this; + } + + public DecoderOptions maxRecursionDepth(int maxRecursionDepth) { + this.maxRecursionDepth = maxRecursionDepth; + + return this; + } + + public DecoderOptions maxNumMapEntries(int maxNumMapEntries) { + this.maxNumMapEntries = maxNumMapEntries; + + return this; + } + + public DecoderOptions maxNumArrayEntries(int maxNumArrayEntries) { + this.maxNumArrayEntries = maxNumArrayEntries; + + return this; + } + + public DecoderOptions maxStringLength(int maxStringLength) { + this.maxStringLength = maxStringLength; + + return this; + } +} diff --git a/src/main/java/com/booking/sereal/DefaultTypeMapper.java b/src/main/java/com/booking/sereal/DefaultTypeMapper.java new file mode 100644 index 000000000..48de5a8dc --- /dev/null +++ b/src/main/java/com/booking/sereal/DefaultTypeMapper.java @@ -0,0 +1,28 @@ +package com.booking.sereal; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DefaultTypeMapper implements TypeMapper { + @Override + public boolean useObjectArray() { + return false; + } + + @Override + public List makeArray(int size) { + return new ArrayList(size); + } + + @Override + public Map makeMap(int size) { + return new HashMap(size); + } + + @Override + public Object makeObject(String className, Object data) { + return new PerlObject(className, data); + } +} diff --git a/src/main/java/com/booking/sereal/Encoder.java b/src/main/java/com/booking/sereal/Encoder.java new file mode 100644 index 000000000..390b74749 --- /dev/null +++ b/src/main/java/com/booking/sereal/Encoder.java @@ -0,0 +1,1014 @@ +package com.booking.sereal; + +import com.booking.sereal.impl.BytearrayCopyMap; +import com.booking.sereal.impl.IdentityMap; +import com.booking.sereal.impl.StringCopyMap; +import com.github.luben.zstd.Zstd; +import java.math.BigInteger; +import org.xerial.snappy.Snappy; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.lang.reflect.Array; +import java.nio.charset.Charset; +import java.util.*; +import java.util.regex.Pattern; +import java.util.zip.Deflater; + +import static com.booking.sereal.EncoderOptions.CompressionType; + +/** + * Sereal encoder with Perl-like interface. + *

+ * This class can be used to encode Perl-like data-structures: (boxed) primitive types, strings, arrays + * and maps. + */ +public class Encoder { + + private static final EncoderOptions DEFAULT_OPTIONS = new EncoderOptions(); + private static final byte[] EMPTY_ARRAY = new byte[0]; + private static final BigInteger LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger UNSIGNED_LONG_MAX_VALUE = new BigInteger(1, new byte[] { + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, + }); + + private final boolean perlRefs; + private final boolean perlAlias; + private final int protocolVersion, encoding; + private final CompressionType compressionType; + private final long compressionThreshold; + private Deflater deflater; + private final int zstdCompressionLevel; + + private final int maxRecursionDepth; + private final int maxNumMapEntries; + private final int maxNumArrayEntries; + private final int maxStringLength; + + private final byte[] HEADER = + new byte[] { + (byte) (SerealHeader.MAGIC >> 24), + (byte) (SerealHeader.MAGIC >> 16), + (byte) (SerealHeader.MAGIC >> 8), + (byte) (SerealHeader.MAGIC >> 0), + }; + private final byte[] HEADER_V3 = + new byte[] { + (byte) (SerealHeader.MAGIC_V3 >> 24), + (byte) (SerealHeader.MAGIC_V3 >> 16), + (byte) (SerealHeader.MAGIC_V3 >> 8), + (byte) (SerealHeader.MAGIC_V3 >> 0), + }; + // so we don't need to allocate this every time we encode a varint + private byte[] varint_buf = new byte[12]; + // track things we've encoded so we can emit refs and copies + private IdentityMap tracked = new IdentityMap(); + private IdentityMap aliases, maybeAliases; + private BytearrayCopyMap trackedBytearrayCopy = new BytearrayCopyMap(); + private StringCopyMap trackedStringCopy = new StringCopyMap(); + private StringCopyMap trackedClassnames = new StringCopyMap(); + // where we store the various encoded things + private byte[] bytes = new byte[1024]; + private byte[] compressedBytes = EMPTY_ARRAY; + private long size = 0; // size of everything encoded so far + private long compressedSize; + private int headerSize, headerOffset; + private Charset charset_utf8 = Charset.forName("UTF-8"); + + private int recursionDepth = 0; + + /** Create a new Encoder with default options. */ + public Encoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create a new Encoder with the specified options. + * + * @param options {@link EncoderOptions} to apply. + */ + public Encoder(EncoderOptions options) { + perlRefs = options.perlReferences(); + perlAlias = options.perlAliases(); + protocolVersion = options.protocolVersion(); + compressionType = options.compressionType(); + compressionThreshold = options.compressionThreshold(); + zstdCompressionLevel = options.zstdCompressionLevel(); + + maxRecursionDepth = options.maxRecursionDepth(); + maxNumMapEntries = options.maxNumMapEntries(); + maxNumArrayEntries = options.maxNumArrayEntries(); + maxStringLength = options.maxStringLength(); + + switch (protocolVersion) { + case 4: + encoding = + compressionType.equals(CompressionType.ZSTD) + ? 4 + : compressionType.equals(CompressionType.ZLIB) + ? 3 + : compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 3: + encoding = + compressionType.equals(CompressionType.ZLIB) + ? 3 + : compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 2: + encoding = compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 1: + encoding = compressionType.equals(CompressionType.SNAPPY) ? 1 : 0; + break; + default: + encoding = 0; + break; + } + if (encoding == 3) deflater = new Deflater(options.zlibCompressionLevel()); + else deflater = null; + + if (perlAlias) { + aliases = new IdentityMap(); + maybeAliases = new IdentityMap(); + } + } + + private static void prepareHeader( + byte[] originBytes, byte[] compressedBytes, int headerSize, int sizeLength) { + System.arraycopy(originBytes, 0, compressedBytes, 0, headerSize); + // varint-encoded 0, filling all space + for (int i = headerSize; i < headerSize + sizeLength - 1; i++) compressedBytes[i] = (byte) 128; + compressedBytes[headerSize + sizeLength - 1] = 0; + } + + private static void finishHeader( + byte[] compressedBytes, long compressedSize, int headerSize, int sizeLength) { + int after = encodeVarint(compressedSize, compressedBytes, headerSize); + if (after != headerSize + sizeLength) compressedBytes[after - 1] |= (byte) 0x80; + } + + private static int varintLength(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + n >>>= 7; + length++; + } + + return length + 1; + } + + private static int encodeVarint(long n, byte[] buffer, int pos) { + while (n > 127) { + buffer[pos++] = (byte) ((n & 127) | 128); + n >>= 7; + } + buffer[pos++] = (byte) n; + + return pos; + } + + // write header and version/encoding + private void init(Object header, boolean hasHeader) throws SerealException { + if (protocolVersion >= 3) appendBytesUnsafe(HEADER_V3); + else appendBytesUnsafe(HEADER); + appendByteUnsafe((byte) ((encoding << 4) | protocolVersion)); + + if (hasHeader) { + encodeUserHeader(header); + } else { + appendByteUnsafe((byte) 0x00); + } + + headerSize = (int) size; + if (protocolVersion > 1) + // because offsets start at 1 + headerOffset = headerSize - 1; + else headerOffset = 0; + } + + private void encodeUserHeader(Object header) throws SerealException { + long originalSize = size; + + // be optimistic about encoded header size + size += 2; // one for the size, one for 8bit bitfield + encode(header); + + int suffixSize = (int) (size - originalSize - 1); + if (suffixSize < 128) { + bytes[(int) originalSize] = (byte) suffixSize; + bytes[(int) originalSize + 1] = 0x01; + } else { + // we were too optimistic + int sizeLength = varintLength(suffixSize); + + // make space + ensureAvailable(sizeLength - 1); + System.arraycopy( + bytes, + (int) originalSize + 2, + bytes, + (int) originalSize + sizeLength + 1, + suffixSize - 1); + size += sizeLength - 1; + + // now write size and 8bit bitfield + encodeVarint(suffixSize, bytes, (int) originalSize); + bytes[(int) originalSize + sizeLength] = 0x01; + } + + resetTracked(); + } + + /** + * After a call to {@code write()}, returns a reference of the encoded data. + *

+ * The reference is only valid until the next call to {@code write()}. + * + * @return reference to the encoded data. + */ + public ByteArray getDataReference() { + if (compressedSize != 0) { + return new ByteArray(compressedBytes, (int) compressedSize); + } else { + return new ByteArray(bytes, (int) size); + } + } + + /** + * After a call to {@code write()}, returns a copy of the encoded data. + * + * @return copy of the encoded data. + */ + public byte[] getData() { + if (compressedSize != 0) { + return Arrays.copyOf(compressedBytes, (int) compressedSize); + } else { + return Arrays.copyOf(bytes, (int) size); + } + } + + private void markNotCompressed() { + compressedSize = 0; + bytes[4] &= (byte) 0xf; + } + + private void compressSnappy() throws SerealException { + int maxSize = Snappy.maxCompressedLength((int) size - headerSize); + int sizeLength = encoding == 2 ? varintLength(maxSize) : 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + maxSize) > compressedBytes.length) + compressedBytes = new byte[headerSize + sizeLength + maxSize]; + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + int compressed; + try { + compressed = + Snappy.compress( + bytes, headerSize, (int) size - headerSize, compressedBytes, headerSize + sizeLength); + } catch (IOException e) { + throw new SerealException(e); + } + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + if (encoding == 2) { + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + } + + // from miniz.c + private int zlibMaxSize(int sourceLen) { + return Math.max( + 128 + (sourceLen * 110) / 100, 128 + sourceLen + ((sourceLen / (31 * 1024)) + 1) * 5); + } + + private void compressZlib() { + deflater.reset(); + + int sourceSize = (int) size - headerSize; + int maxSize = zlibMaxSize(sourceSize); + int sizeLength = varintLength(sourceSize); + int sizeLength2 = varintLength(maxSize); + int pos = 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + sizeLength2 + maxSize) > compressedBytes.length) + compressedBytes = new byte[headerSize + sizeLength + sizeLength2 + maxSize]; + + System.arraycopy(bytes, 0, compressedBytes, 0, headerSize); + pos += headerSize; + pos = encodeVarint(sourceSize, compressedBytes, pos); + + // varint-encoded 0, filling all space + int encodedSizePos = pos; + for (int max = pos + sizeLength2 - 1; pos < max; ) compressedBytes[pos++] = (byte) 128; + compressedBytes[pos++] = 0; + + deflater.setInput(bytes, headerSize, sourceSize); + deflater.finish(); + + int compressed = deflater.deflate(compressedBytes, pos, compressedBytes.length - pos); + compressedSize = headerSize + sizeLength + sizeLength2 + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + int after = encodeVarint(compressed, compressedBytes, encodedSizePos); + if (after != headerSize + sizeLength + sizeLength2) compressedBytes[after - 1] |= (byte) 0x80; + } + + private void compressZstd() throws SerealException { + long maxSize = Zstd.compressBound((int) size - headerSize); + int sizeLength = varintLength(maxSize); + + if (headerSize + sizeLength + maxSize > Integer.MAX_VALUE) + throw new SerealException( + "Compressed data size exceeds integer MAX_VALUE: " + (headerSize + maxSize)); + if (headerSize + sizeLength + maxSize > compressedBytes.length) + compressedBytes = new byte[(int) (headerSize + sizeLength + maxSize)]; + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + long compressed = + Zstd.compressUsingDict( + compressedBytes, + headerSize + sizeLength, + bytes, + headerSize, + (int) size - headerSize, + new byte[0], + zstdCompressionLevel); + if (Zstd.isError(compressed)) throw new SerealException(Zstd.getErrorName(compressed)); + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + + /** + * Write an integer as a varint + * + *

Note: sometimes the next thing while decoding is know to be a varint, sometimes there must + * be a tag that denotes the next item *is* a varint. So don't forget to write that tag. + * + * @param n positive integer + */ + private void appendVarint(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + varint_buf[length++] = (byte) ((n & 127) | 128); + n >>>= 7; + } + varint_buf[length++] = (byte) n; + + appendBytes(varint_buf, length); + } + + private void setTrackBit(long offset) { + bytes[(int) offset + headerOffset] |= (byte) 0x80; + } + + /** + * Encode a number as zigzag + * + * @param n nageative integer + */ + private void appendZigZag(long n) { + appendByte(SerealHeader.SRL_HDR_ZIGZAG); + appendVarint((n << 1) ^ (n >> 63)); // note the signed right shift + } + + /** + * Encode a short ascii string + * + * @param latin1 String to encode as US-ASCII bytes + * @throws SerealException if the string is not short enough + */ + private void appendShortBinary(byte[] latin1) throws SerealException { + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(latin1); + if (copyOffset != BytearrayCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + int length = latin1.length; + long location = size; + + if (length > 31) { + throw new SerealException("Cannot create short binary for " + latin1 + ": too long"); + } + + // length of string + appendByte((byte) (length | SerealHeader.SRL_HDR_SHORT_BINARY)); + + // save it + appendBytes(latin1); + + trackForCopy(latin1, location); + } + + /** + * Encode a long ascii string + * + * @param latin1 String to encode as US-ASCII bytes + */ + private void appendBinary(byte[] latin1) { + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(latin1); + if (copyOffset != BytearrayCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + int length = latin1.length; + long location = size; + + // length of string + appendByte(SerealHeader.SRL_HDR_BINARY); + appendBytesWithLength(latin1); + + trackForCopy(latin1, location); + } + + private void appendCopy(long location) { + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(location); + } + + /** + * Encode a regex + * + * @param p regex pattern. Only support flags "smix": DOTALL | MULTILINE | CASE_INSENSITIVE | + * COMMENTS + * @throws SerealException if the pattern is longer that a short binary string + */ + private void appendRegex(Pattern p) throws SerealException { + + byte[] flags = new byte[4]; + int flags_size = 0; + if ((p.flags() & Pattern.MULTILINE) != 0) flags[flags_size++] = 'm'; + if ((p.flags() & Pattern.DOTALL) != 0) flags[flags_size++] = 's'; + if ((p.flags() & Pattern.CASE_INSENSITIVE) != 0) flags[flags_size++] = 'i'; + if ((p.flags() & Pattern.COMMENTS) != 0) flags[flags_size++] = 'x'; + + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendStringType(new Latin1String(p.pattern())); + appendByte((byte) (flags_size | SerealHeader.SRL_HDR_SHORT_BINARY)); + appendBytes(flags, flags_size); + } + + /** + * Encodes a byte array emitting both length and data + */ + private void appendBytesWithLength(byte[] in) { + appendVarint(in.length); + appendBytes(in); + } + + private void appendBoolean(boolean b) { + appendByte(b ? SerealHeader.SRL_HDR_TRUE : SerealHeader.SRL_HDR_FALSE); + } + + /** + * Create a new Sereal document containing the specified value in the body. + *

+ * Each call to this method overwrites the current Sereal document. + * + * @param obj Object to encode + * + * @return new Sereal document + * + * @throws SerealException Object could not be encoded. + */ + public Encoder write(Object obj) throws SerealException { + return write(obj, null, false); + } + + /** + * Create a new Sereal document with the given header and body. + *

+ * Each call to this method overwrites the current Sereal document. + * + * @param obj Object to encode + * @param header Sereal header + * + * @return new Sereal document + * + * @throws SerealException Object could not be encoded. + */ + public Encoder write(Object obj, Object header) throws SerealException { + return write(obj, header, true); + } + + private Encoder write(Object obj, Object header, boolean hasHeader) throws SerealException { + if (hasHeader && protocolVersion == 1) + throw new SerealException("Can't encode user header in Sereal protocol version 1"); + + reset(); + init(header, hasHeader); + try { + encode(obj); + } catch (StackOverflowError error) { + throw new SerealException("StackOverflowError: Reached recursion limit during serialization"); + } + + if (size - headerSize > compressionThreshold) { + if (compressionType.equals(CompressionType.SNAPPY)) compressSnappy(); + else if (compressionType.equals(CompressionType.ZLIB)) compressZlib(); + else if (compressionType.equals(CompressionType.ZSTD)) compressZstd(); + } else { + // we did not do compression after all + markNotCompressed(); + } + + return this; + } + + @SuppressWarnings("unchecked") + private void encode(Object obj) throws SerealException { + // track it (for ALIAS tags) + long location = size; + + if (perlAlias) { + long aliasOffset = aliases.get(obj); + if (aliasOffset != IdentityMap.NOT_FOUND) { + appendAlias(aliasOffset); + return; + } else { + maybeAliases.put(obj, location - headerOffset); + } + } + + Class type = obj == null ? PerlUndef.class : obj.getClass(); + + // this needs to be first for obvious reasons :) + if (type == PerlUndef.class) { + if (protocolVersion == 3 && obj == PerlUndef.CANONICAL) + appendByte(SerealHeader.SRL_HDR_CANONICAL_UNDEF); + else appendByte(SerealHeader.SRL_HDR_UNDEF); + return; + } + + // this is ugly :) + if (type == Long.class || type == Integer.class || type == Byte.class) { + appendNumber(((Number) obj).longValue()); + } else if (type == Boolean.class) { + appendBoolean((Boolean) obj); + } else if (type == String.class) { + appendStringType((String) obj); + } else if (type == Latin1String.class) { + appendStringType((Latin1String) obj); + } else if (type == byte[].class) { + appendStringType((byte[]) obj); + } else if (type.isArray()) { + if (perlRefs || !tryAppendRefp(obj)) { + if (obj instanceof Object[]) { + if (!(obj instanceof String[])) { + depthIncrement(); + } + appendArray((Object[]) obj); + if (!(obj instanceof String[])) { + depthDecrement(); + } + } else { + appendArray(obj); + } + } + } else if (type == HashMap.class || obj instanceof Map) { + depthIncrement(); + if (perlRefs || !tryAppendRefp(obj)) appendMap((Map) obj); + depthDecrement(); + } else if (type == ArrayList.class || obj instanceof List) { + depthIncrement(); + if (perlRefs || !tryAppendRefp(obj)) appendArray((List) obj); + depthDecrement(); + } else if (type == Pattern.class) { + appendRegex((Pattern) obj); + } else if (type == Double.class) { + appendDouble((Double) obj); + } else if (type == Float.class) { + appendFloat((Float) obj); + } else if (type == BigInteger.class) { + appendBigInteger((BigInteger) obj); + } else if (type == PerlReference.class) { + PerlReference ref = (PerlReference) obj; + long trackedRef = getTrackedItem(ref.getValue()); + + if (trackedRef != IdentityMap.NOT_FOUND) { + appendRefp(trackedRef); + } else { + appendRef(ref); + } + } else if (type == WeakReference.class) { + Object value = ((WeakReference) obj).get(); + boolean isRef = isDefinitelyReference(value); + long currentOffset = size; + + appendByte(SerealHeader.SRL_HDR_WEAKEN); + + if (!isRef) { + appendByte(SerealHeader.SRL_HDR_PAD); + } + encode(value); + if (!isRef) { + if (!isRefTag(bytes[(int) currentOffset + 2])) { + bytes[(int) currentOffset + 1] = SerealHeader.SRL_HDR_REFN; + } + } else { + if (!isRefTag(bytes[(int) currentOffset + 1])) { + throw new SerealException("Internal error while encoding weak reference"); + } + } + } else if (type == PerlAlias.class) { + Object value = ((PerlAlias) obj).getValue(); + + if (perlAlias) { + long maybeAlias = maybeAliases.get(value); + long alias = aliases.get(value); + + if (alias != IdentityMap.NOT_FOUND) { + appendAlias(alias); + } else if (maybeAlias != IdentityMap.NOT_FOUND) { + appendAlias(maybeAlias); + aliases.put(value, maybeAlias); + } else { + encode(value); + aliases.put(value, location - headerOffset); + } + } else { + encode(value); + } + } else if (type == PerlObject.class) { + PerlObject po = (PerlObject) obj; + + appendPerlObject(po.getName(), po.getData()); + } + + if (size == location) { // didn't write anything + throw new SerealException( + "Don't know how to encode: " + type.getName() + " = " + obj.toString()); + } + } + + private void appendPerlObject(String className, Object data) throws SerealException { + long nameOffset = trackedClassnames.get(className); + if (nameOffset != StringCopyMap.NOT_FOUND) { + + appendByte(SerealHeader.SRL_HDR_OBJECTV); + appendVarint(nameOffset); + } else { + appendByte(SerealHeader.SRL_HDR_OBJECT); + trackedClassnames.put(className, size - headerOffset); + appendStringType(className); + } + + // write the data structure + encode(data); + } + + /** + * @param obj object that might have been already encoded earlier in the bytestream + * @return location of object in bytestream, or {@link IdentityMap#NOT_FOUND} + */ + private long getTrackedItem(Object obj) { + return tracked.get(obj); + } + + private long getTrackedItemCopy(byte[] bytes) { + return trackedBytearrayCopy.get(bytes); + } + + private long getTrackedItemCopy(String string) { + return trackedStringCopy.get(string); + } + + private void track(Object obj, long obj_location) { + tracked.put(obj, obj_location - headerOffset); + } + + private void trackForCopy(byte[] bytes, long location) { + trackedBytearrayCopy.put(bytes, location - headerOffset); + } + + private void trackForCopy(String string, long location) { + trackedStringCopy.put(string, location - headerOffset); + } + + private void appendDouble(Double d) { + appendByte(SerealHeader.SRL_HDR_DOUBLE); + + long bits = Double.doubleToLongBits(d); // very convienent, thanks Java guys! :) + for (int i = 0; i < 8; i++) { + varint_buf[i] = (byte) ((bits >> (i * 8)) & 0xff); + } + appendBytes(varint_buf, 8); + } + + private void appendFloat(Float f) { + appendByte(SerealHeader.SRL_HDR_FLOAT); + + int bits = Float.floatToIntBits(f); // very convienent, thanks Java guys! :) + for (int i = 0; i < 4; i++) { + varint_buf[i] = (byte) ((bits >> (i * 8)) & 0xff); + } + appendBytes(varint_buf, 4); + } + + private void appendRefp(long location) { + setTrackBit(location); + appendByte(SerealHeader.SRL_HDR_REFP); + appendVarint(location); + } + + private boolean tryAppendRefp(Object obj) { + long location = getTrackedItem(obj); + + if (location != IdentityMap.NOT_FOUND) { + appendRefp(location); + + return true; + } else { + return false; + } + } + + private void appendAlias(long location) { + setTrackBit(location); + appendByte(SerealHeader.SRL_HDR_ALIAS); + appendVarint(location); + } + + private void appendMap(Map hash) throws SerealException { + + if (maxNumMapEntries != 0 && hash.size() > maxNumMapEntries) { + throw new SerealException("Got input hash with " + hash.size() + " entries, but the configured maximum is just " + maxNumMapEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(hash, size); + } + appendByte(SerealHeader.SRL_HDR_HASH); + appendVarint(hash.size()); + + for (Map.Entry entry : hash.entrySet()) { + encode(entry.getKey().toString()); + encode(entry.getValue()); + } + } + + private void appendRef(PerlReference ref) throws SerealException { + Object refValue = ref.getValue(); + + appendByte(SerealHeader.SRL_HDR_REFN); + track(refValue, size); + encode(refValue); + } + + private void appendArray(Object obj) throws SerealException { + // checking length without casting to Object[] since they might primitives + int count = Array.getLength(obj); + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(obj, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + // write the objects (works for both Objects and primitives) + for (int index = 0; index < count; index++) { + encode(Array.get(obj, index)); + } + } + + private void appendArray(Object[] array) throws SerealException { + int count = array.length; + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(array, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + for (Object item : array) { + encode(item); + } + } + + private void appendArray(List list) throws SerealException { + int count = list.size(); + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(list, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + for (Object item : list) { + encode(item); + } + } + + private void appendStringType(byte[] bytes) throws SerealException { + if (bytes.length < SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(bytes); + } else { + appendBinary(bytes); + } + } + + private void appendStringType(Latin1String str) throws SerealException { + byte[] latin1 = str.getBytes(); + if (str.length() < SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(latin1); + } else { + appendBinary(latin1); + } + } + + private void appendStringType(String str) throws SerealException { + if (maxStringLength != 0 && str.length() > maxStringLength) { + throw new SerealException("Got input string with " + str.length() + " characters, but the configured maximum is just " + maxStringLength); + } + + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(str); + if (copyOffset != StringCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + long location = size; + + byte[] utf8 = ((String) str).getBytes(charset_utf8); + appendByte(SerealHeader.SRL_HDR_STR_UTF8); + appendVarint(utf8.length); + appendBytes(utf8); + + trackForCopy(str, location); + } + + private void appendNumber(long l) { + if (l < 0) { + if (l > -17) { + appendByte((byte) (SerealHeader.SRL_HDR_NEG_LOW | (l + 32))); + } else { + appendZigZag(l); + } + } else { + if (l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + } + + private void appendBigInteger(BigInteger bi) throws SerealException { + int compareToZero = bi.compareTo(BigInteger.ZERO); + if (compareToZero < 0 && bi.compareTo(LONG_MIN_VALUE) >= 0) { + appendNumber(bi.longValue()); + } else if (compareToZero > 0 && bi.compareTo(UNSIGNED_LONG_MAX_VALUE) <= 0) { + if (bi.compareTo(LONG_MAX_VALUE) <= 0) { + appendNumber(bi.longValue()); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(bi.longValue()); + } + } else if (compareToZero == 0) { + appendNumber(0); + } else { + throw new SerealException("BigInteger value is outside representable range"); + } + } + + private boolean isDefinitelyReference(Object value) { + if (value instanceof Map || value instanceof List) { + return true; + } else if (value instanceof PerlReference) { + return true; + } + + return false; + } + + private boolean isRefTag(byte tag) { + // the first branch is the common case, the other two branchs are unlikely + if (tag == SerealHeader.SRL_HDR_REFN || + tag == SerealHeader.SRL_HDR_REFP) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + return true; + } + + return false; + } + + /** Discard all previous tracking clear the buffers etc Call this when you reuse the encoder */ + private void reset() { + size = compressedSize = headerSize = recursionDepth = 0; + resetTracked(); + } + + private void resetTracked() { + tracked.clear(); + trackedBytearrayCopy.clear(); + trackedStringCopy.clear(); + if (perlAlias) { + aliases.clear(); + maybeAliases.clear(); + } + trackedClassnames.clear(); + } + + private void ensureAvailable(int required) { + long total = required + size; + + if (total > bytes.length) bytes = Arrays.copyOf(bytes, (int) (total * 3 / 2)); + } + + private void appendBytes(byte[] data) { + ensureAvailable(data.length); + appendBytesUnsafe(data); + } + + private void appendBytes(byte[] data, int length) { + ensureAvailable(length); + appendBytesUnsafe(data, length); + } + + private void appendBytesUnsafe(byte[] data) { + System.arraycopy(data, 0, bytes, (int) size, data.length); + size += data.length; + } + + private void appendBytesUnsafe(byte[] data, int length) { + System.arraycopy(data, 0, bytes, (int) size, length); + size += length; + } + + private void appendByte(byte data) { + ensureAvailable(1); + appendByteUnsafe(data); + } + + private void appendByteUnsafe(byte data) { + bytes[(int) size] = data; + size++; + } + + private void depthIncrement() throws SerealException { + if (recursionDepth++ > maxRecursionDepth) { + throw new SerealException("Reached recursion limit (" + maxRecursionDepth + ") during serialization"); + } + } + + private void depthDecrement() { + recursionDepth--; + } + + /** + * Close the encoder to recycle the resources, it must be called by the client at the end of the + * use, otherwise, NullPointerException will be thrown when you reuse the ZLIB encoder. + *

+ * Returns the native memory used by deflater explicitly before the garbage collector. + */ + public void close() { + if (deflater != null) { + deflater.end(); + deflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/EncoderOptions.java b/src/main/java/com/booking/sereal/EncoderOptions.java new file mode 100644 index 000000000..285d4008f --- /dev/null +++ b/src/main/java/com/booking/sereal/EncoderOptions.java @@ -0,0 +1,203 @@ +package com.booking.sereal; + +public class EncoderOptions { + private boolean perlRefs = false; + private boolean perlAlias = false; + private int protocolVersion = 4; + private CompressionType compressionType = CompressionType.NONE; + private long compressionThreshold = 1024; + private int zlibCompressionLevel = 6; + private int zstdCompressionLevel = 3; + + private int maxRecursionDepth = 10_000; + private int maxNumMapEntries = 0; + private int maxNumArrayEntries = 0; + private int maxStringLength = 0; + + public boolean perlReferences() { + return perlRefs; + } + + public boolean perlAliases() { + return perlAlias; + } + + public int protocolVersion() { + return protocolVersion; + } + + public CompressionType compressionType() { + return compressionType; + } + + public long compressionThreshold() { + return compressionThreshold; + } + + public int zlibCompressionLevel() { + return zlibCompressionLevel; + } + + public int zstdCompressionLevel() { + return zstdCompressionLevel; + } + + /** + * {@link Encoder} is recursive. If you pass it an Object that is deeply + * nested, it will eventually exhaust the C stack. Therefore, there is a limit on + * the depth of recursion that is accepted. It defaults to 10000 nested calls. You + * may choose to override this value with the {@link EncoderOptions#maxRecursionDepth(int)} option. + * Beware that setting it too high can cause hard crashes. + * + * Do note that the setting is somewhat approximate. Setting it to 10000 may break at + * somewhere between 9997 and 10003 nested structures depending on their types. + * + * It's also important to note that this value doesn't mirror the Perl implementation. Perl and Java + * Sereal wrappers are implemented in a different way. That means that the recursion depth marked + * as max in Perl may be different than the one needed in Java. The same applies for the Encoder + * and Decoder options. + * + * The Java stack can be controlled with different flags (for example -Xss) to change it's size. + * + * @return maximum recursion depth + */ + public int maxRecursionDepth() { + return maxRecursionDepth; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to deserialize any hash/dictionary (or hash-based object) with more than + * that number of entries. This is to be able to respond quickly to any future + * hash-collision attacks on Perl's hash function. Chances are, you don't want + * or need this. For a gentle introduction to the topic from the cryptographic + * point of view, see Collision attack. + * + * This value can be override with {@link EncoderOptions#maxNumMapEntries(int)} option + * + * @return maximum number of map entries + */ + public int maxNumMapEntries() { + return maxNumMapEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to serialize any array with more than that number of entries. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link EncoderOptions#maxNumArrayEntries(int)} option + * + * @return maximum number of array entries + */ + public int maxNumArrayEntries() { + return maxNumArrayEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to deserialize any string with more than that number of characters. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link EncoderOptions#maxStringLength(int)} option + * + * @return maximum supported string length + */ + public int maxStringLength() { + return maxStringLength; + } + + /** + * + * @param perlReferences {@code true} if perlReferences are supported, {@code false} otherwise. + * @return EncoderOptions + */ + public EncoderOptions perlReferences(boolean perlReferences) { + this.perlRefs = perlReferences; + + return this; + } + + public EncoderOptions perlAliases(boolean perlAliases) { + this.perlAlias = perlAliases; + + return this; + } + + public EncoderOptions protocolVersion(int protocolVersion) { + if (protocolVersion < 1 || protocolVersion > 4) { + throw new IllegalArgumentException("Unknown Sereal version " + protocolVersion); + } + this.protocolVersion = protocolVersion; + + return this; + } + + public EncoderOptions compressionType(CompressionType compressionType) { + if (protocolVersion < compressionType.minProtocolVersion) { + throw new IllegalArgumentException("Compression " + compressionType + " not supported in Sereal protocol" + protocolVersion); + } + this.compressionType = compressionType; + + return this; + } + + public EncoderOptions compressionThreshold(long compressionThreshold) { + this.compressionThreshold = compressionThreshold; + + return this; + } + + public EncoderOptions zlibCompressionLevel(int zlibCompressionLevel) { + this.zlibCompressionLevel = zlibCompressionLevel; + + return this; + } + + public EncoderOptions zstdCompressionLevel(int zstdCompressionLevel) { + this.zstdCompressionLevel = zstdCompressionLevel; + return this; + } + + + public EncoderOptions maxRecursionDepth(int maxRecursionDepth) { + this.maxRecursionDepth = maxRecursionDepth; + + return this; + } + + public EncoderOptions maxNumMapEntries(int maxNumMapEntries) { + this.maxNumMapEntries = maxNumMapEntries; + + return this; + } + + public EncoderOptions maxNumArrayEntries(int maxNumArrayEntries) { + this.maxNumArrayEntries = maxNumArrayEntries; + + return this; + } + + public EncoderOptions maxStringLength(int maxStringLength) { + this.maxStringLength = maxStringLength; + + return this; + } + + public enum CompressionType { + NONE(1, SerealHeader.SRL_ENCODING_NONE), + SNAPPY(1, -1), + ZLIB(3, SerealHeader.SRL_ENCODING_ZLIB), + ZSTD(4, SerealHeader.SRL_ENCODING_ZSTD); + + final byte minProtocolVersion; + final byte encoding; + + CompressionType(int minProtocolVersion, int encoding) { + this.minProtocolVersion = (byte) minProtocolVersion; + this.encoding = (byte) encoding; + } + } +} diff --git a/src/main/java/com/booking/sereal/Latin1String.java b/src/main/java/com/booking/sereal/Latin1String.java new file mode 100644 index 000000000..85f3e4ee0 --- /dev/null +++ b/src/main/java/com/booking/sereal/Latin1String.java @@ -0,0 +1,75 @@ +package com.booking.sereal; + +import java.nio.charset.Charset; +import java.util.Arrays; + +public class Latin1String implements CharSequence { + private static final Charset charset_latin1 = Charset.forName("ISO-8859-1"); + private final byte[] bytes; + private boolean hashCodeSet = false; + private int hashcode; + + public Latin1String(String s) { + this.bytes = s.getBytes(charset_latin1); + this.hashCodeSet = false; + } + + public Latin1String(byte[] bytes) { + this.bytes = bytes; + this.hashCodeSet = false; + } + + @Override + public String toString() { + return new String(bytes, charset_latin1); + } + + @Override + public int hashCode() { + if (!hashCodeSet) { + hashcode = Arrays.hashCode(bytes); + } + return hashcode; + } + + @Override + public char charAt(int index) { + return (char) bytes[index]; + } + + @Override + public int length() { + return bytes.length; + } + + @Override + public CharSequence subSequence(int start, int end) { + return this.toString().subSequence(start, end); + } + + public byte[] getBytes() { + return this.bytes; + } + + public String getString() { + return this.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (this.hashCode() != o.hashCode()) { + return false; + } + if (!this.getClass().equals(o.getClass())) { + return false; + } + + return Arrays.equals(this.bytes, ((Latin1String) o).bytes); + } +} diff --git a/src/main/java/com/booking/sereal/PerlAlias.java b/src/main/java/com/booking/sereal/PerlAlias.java new file mode 100644 index 000000000..f7dd45fcc --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlAlias.java @@ -0,0 +1,18 @@ +package com.booking.sereal; + +/** + * Since the Sereal format has the notion of both reference and alias we use this class to wrap + * those values. That way we can always roundtrip between Perl and Java. + */ +public class PerlAlias { + + private final Object value; + + public PerlAlias(Object value) { + this.value = value; + } + + public Object getValue() { + return value; + } +} diff --git a/src/main/java/com/booking/sereal/PerlObject.java b/src/main/java/com/booking/sereal/PerlObject.java new file mode 100644 index 000000000..deb9f0290 --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlObject.java @@ -0,0 +1,40 @@ +package com.booking.sereal; + +import java.util.Map; + +/* + * Perl object which is defined by a name and either an array, hash or ref + * (in Perl all of these are basically refs) + * + * Also, this is very ugly + */ +public class PerlObject { + + private final Object data; + private String name; + + public PerlObject(String className, Object obj) { + this.name = className; + this.data = obj; + } + + public String getName() { + return name; + } + + public boolean isHash() { + return getData() instanceof Map; + } + + public boolean isArray() { + return getData().getClass().isArray(); + } + + public boolean isReference() { + return !isHash() && !isArray(); + } + + public Object getData() { + return data; + } +} diff --git a/src/main/java/com/booking/sereal/PerlReference.java b/src/main/java/com/booking/sereal/PerlReference.java new file mode 100644 index 000000000..bc167896b --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlReference.java @@ -0,0 +1,27 @@ +package com.booking.sereal; + +/** + * So we can encode references perl style (otherwise there is no way to distinguish Strings from + * "Stringrefs" + */ +public class PerlReference { + + private Object value; + + public PerlReference(Object value) { + this.value = value; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + @Override + public String toString() { + return "Reference to: " + (value == null ? "null" : value.toString()); + } +} diff --git a/src/main/java/com/booking/sereal/PerlUndef.java b/src/main/java/com/booking/sereal/PerlUndef.java new file mode 100644 index 000000000..836c5a6b6 --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlUndef.java @@ -0,0 +1,11 @@ +package com.booking.sereal; + +/** So we can preserve the identity of different undefs */ +public final class PerlUndef { + public static final PerlUndef CANONICAL = new PerlUndef(); + + @Override + public String toString() { + return "Undef"; + } +} diff --git a/src/main/java/com/booking/sereal/SerealException.java b/src/main/java/com/booking/sereal/SerealException.java new file mode 100644 index 000000000..60ba10c2b --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealException.java @@ -0,0 +1,12 @@ +package com.booking.sereal; + +@SuppressWarnings("serial") +public class SerealException extends Exception { + public SerealException(String msg) { + super(msg); + } + + public SerealException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/booking/sereal/SerealHeader.java b/src/main/java/com/booking/sereal/SerealHeader.java new file mode 100644 index 000000000..afecdef58 --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealHeader.java @@ -0,0 +1,80 @@ +package com.booking.sereal; + +public interface SerealHeader { + + // 0x6c72733d but little endian for some reason + int MAGIC = (0x6c) + (0x72 << 8) + (0x73 << 16) + (0x3d << 24); + + // 0x6c72f33d but little endian for some reason + int MAGIC_V3 = (0x6c) + (0x72 << 8) + (0xf3 << 16) + (0x3d << 24); + + byte SRL_MASK_SHORT_BINARY_LEN = (byte) 31; // lower 5 bits + + byte SRL_ENCODING_NONE = 0; + byte SRL_ENCODING_SNAPPY_LEGACY = 1; + byte SRL_ENCODING_SNAPPY = 2; + byte SRL_ENCODING_ZLIB = 3; + byte SRL_ENCODING_ZSTD = 4; + + /* + Note: Despite this interface already being named SerealHeader we still use SRL_HDR_ + as a prefix so grepping will show both these and the C ones. + +=for autoupdater start + +* NOTE this section is autoupdated by Sereal.git:Perl/shared/author_tools/update_from_header.pl */ + byte SRL_HDR_POS = (byte) 0; /* 0 0x00 0b00000000 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_POS_LOW = (byte) 0; /* 0 0x00 0b00000000 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_POS_HIGH = (byte) 15; /* 15 0x0f 0b00001111 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_NEG = (byte) 16; /* 16 0x10 0b00010000 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_NEG_LOW = (byte) 16; /* 16 0x10 0b00010000 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_NEG_HIGH = (byte) 31; /* 31 0x1f 0b00011111 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_VARINT = (byte) 32; /* 32 0x20 0b00100000 - Varint variable length integer */ + byte SRL_HDR_ZIGZAG = (byte) 33; /* 33 0x21 0b00100001 - Zigzag variable length integer */ + byte SRL_HDR_FLOAT = (byte) 34; /* 34 0x22 0b00100010 */ + byte SRL_HDR_DOUBLE = (byte) 35; /* 35 0x23 0b00100011 */ + byte SRL_HDR_LONG_DOUBLE = (byte) 36; /* 36 0x24 0b00100100 */ + byte SRL_HDR_UNDEF = (byte) 37; /* 37 0x25 0b00100101 None - Perl undef var; eg my $var= undef; */ + byte SRL_HDR_BINARY = (byte) 38; /* 38 0x26 0b00100110 - binary/(latin1) string */ + byte SRL_HDR_STR_UTF8 = (byte) 39; /* 39 0x27 0b00100111 - utf8 string */ + byte SRL_HDR_REFN = (byte) 40; /* 40 0x28 0b00101000 - ref to next item */ + byte SRL_HDR_REFP = (byte) 41; /* 41 0x29 0b00101001 - ref to previous item stored at offset */ + byte SRL_HDR_HASH = (byte) 42; /* 42 0x2a 0b00101010 [ ...] - count followed by key/value pairs */ + byte SRL_HDR_ARRAY = (byte) 43; /* 43 0x2b 0b00101011 [ ...] - count followed by items */ + byte SRL_HDR_OBJECT = (byte) 44; /* 44 0x2c 0b00101100 - class, object-item */ + byte SRL_HDR_OBJECTV = (byte) 45; /* 45 0x2d 0b00101101 - offset of previously used classname tag - object-item */ + byte SRL_HDR_ALIAS = (byte) 46; /* 46 0x2e 0b00101110 - alias to item defined at offset */ + byte SRL_HDR_COPY = (byte) 47; /* 47 0x2f 0b00101111 - copy of item defined at offset */ + byte SRL_HDR_WEAKEN = (byte) 48; /* 48 0x30 0b00110000 - Weaken the following reference */ + byte SRL_HDR_REGEXP = (byte) 49; /* 49 0x31 0b00110001 */ + byte SRL_HDR_OBJECT_FREEZE = (byte) 50; /* 50 0x32 0b00110010 - class, object-item. Need to call "THAW" method on class after decoding */ + byte SRL_HDR_OBJECTV_FREEZE = (byte) 51; /* 51 0x33 0b00110011 - (OBJECTV_FREEZE is to OBJECT_FREEZE as OBJECTV is to OBJECT) */ + byte SRL_HDR_NO = (byte) 52; /* 52 0x34 0b00110100 SvIsBOOL() == PL_No, 5.36 and later only (json false) */ + byte SRL_HDR_YES = (byte) 53; /* 53 0x35 0b00110101 SvIsBOOL() == PL_Yes, 5.36 and later only (json true) */ + byte SRL_HDR_RESERVED = (byte) 54; /* 54 0x36 0b00110110 */ + byte SRL_HDR_RESERVED_LOW = (byte) 54; /* 54 0x36 0b00110110 */ + byte SRL_HDR_RESERVED_HIGH = (byte) 55; /* 55 0x37 0b00110111 */ + byte SRL_HDR_FLOAT_128 = (byte) 56; /* 56 0x38 0b00111000 quadmath _float128 */ + byte SRL_HDR_CANONICAL_UNDEF = (byte) 57; /* 57 0x39 0b00111001 undef (PL_sv_undef) - "the" Perl undef (see notes) */ + byte SRL_HDR_FALSE = (byte) 58; /* 58 0x3a 0b00111010 false (PL_sv_no) */ + byte SRL_HDR_TRUE = (byte) 59; /* 59 0x3b 0b00111011 true (PL_sv_yes) */ + byte SRL_HDR_MANY = (byte) 60; /* 60 0x3c 0b00111100 - repeated tag (not done yet, will be implemented in version 3) */ + byte SRL_HDR_PACKET_START = (byte) 61; /* 61 0x3d 0b00111101 (first byte of magic string in header) */ + byte SRL_HDR_EXTEND = (byte) 62; /* 62 0x3e 0b00111110 - for additional tags */ + byte SRL_HDR_PAD = (byte) 63; /* 63 0x3f 0b00111111 (ignored tag, skip to next byte) */ + byte SRL_HDR_ARRAYREF = (byte) 64; /* 64 0x40 0b01000000 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_ARRAYREF_LOW = (byte) 64; /* 64 0x40 0b01000000 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_ARRAYREF_HIGH = (byte) 79; /* 79 0x4f 0b01001111 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_HASHREF = (byte) 80; /* 80 0x50 0b01010000 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_HASHREF_LOW = (byte) 80; /* 80 0x50 0b01010000 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_HASHREF_HIGH = (byte) 95; /* 95 0x5f 0b01011111 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_SHORT_BINARY = (byte) 96; /* 96 0x60 0b01100000 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_SHORT_BINARY_LOW = (byte) 96; /* 96 0x60 0b01100000 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_SHORT_BINARY_HIGH = (byte) 127; /* 127 0x7f 0b01111111 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_TRACK_FLAG = (byte) 128; /* 128 0x80 0b10000000 if this bit is set track the item */ +/* +* NOTE the above section is auto-updated by Sereal.git:Perl/shared/author_tools/update_from_header.pl + +=for autoupdater stop + */ +} diff --git a/src/main/java/com/booking/sereal/SerealToken.java b/src/main/java/com/booking/sereal/SerealToken.java new file mode 100644 index 000000000..466cd7a6d --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealToken.java @@ -0,0 +1,138 @@ +package com.booking.sereal; + +/** + * Enumeration for Sereal token types, used by {@link com.booking.sereal.TokenDecoder}. + */ +public enum SerealToken { + /** Returned when there is no token. */ + NONE, + + /** + * An integer value up to 64 bits. + */ + LONG, + + /** + * A positive integer value larger than {@link java.lang.Long#MAX_VALUE}. + *

+ * Java does not have unsigned values, so the value is returned as a signed (negative) {@code long} + * with the same bit pattern as the unsigned value. + */ + UNSIGNED_LONG, + + /** + * A binary/ISO-8859-1 string. + */ + BINARY, + + /** + * An array value. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + ARRAY_START, + + /** + * Returned after the last element of an array. + */ + ARRAY_END, + + /** + * An hash value. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + HASH_START, + + /** + * Returned after the last element of an hash. + */ + HASH_END, + + /** + * Returned after the last element of the Sereal document. + */ + END, + + /** + * A 32-bit IEEE-754 floating point number. + */ + FLOAT, + + /** + * A 64-bit IEEE-754 floating point number. + */ + DOUBLE, + + /** + * Canonical {@code true} value. + */ + TRUE, + + /** + * Canonical {@code false} value. + */ + FALSE, + + /** + * An {@code undef} value. + */ + UNDEF, + + /** + * An UTF-8 encoded string. + */ + UTF8, + + /** + * A Perl reference, pointing to the object following the {@code REFN} token. + */ + REFN, + + /** + * A Perl reference, pointing to an object already encountered at a previous offset. + */ + REFP, + + /** + * A Perl object. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + OBJECT_START, + + /** + * Returned after the end of a Perl object. + */ + OBJECT_END, + + /** + * Sereal copy tag, referring to a previous value. + */ + COPY, + + /** + * Sereal alias tag, referring to a previous value. + */ + ALIAS, + + /** + * Marks the following reference as a weak reference. + */ + WEAKEN, + + /** + * Regular expression value. + */ + REGEXP, + + /** + * Canonical {@code undef} value. + */ + CANONICAL_UNDEF, + + /** + * Returned after the end of a sub-decode. + */ + SUBDECODE_END; +} diff --git a/src/main/java/com/booking/sereal/TokenDecoder.java b/src/main/java/com/booking/sereal/TokenDecoder.java new file mode 100644 index 000000000..aef8bd412 --- /dev/null +++ b/src/main/java/com/booking/sereal/TokenDecoder.java @@ -0,0 +1,923 @@ +package com.booking.sereal; + +import com.github.luben.zstd.Zstd; +import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import org.xerial.snappy.Snappy; + +/** + * A low-level stream decoder for Sereal. + *

+ * It provides a token stream, with accessor to retrieve token data (e.g. the integer value associated with a {@code LONG} + * token). + *

+ * The decoder performs some basic sanity checks, but it does not perform higher-level validation, such as checking + * that hash keys are strings, array values are scalars, or weaken tokens are followed by a reference. + *

+ * Example: + * 

+ * {@code
+ *   decoder.setData(bytes);
+ *   decoder.prepareDecodeBody();
+ *
+ *   while (decoder.nextToken() != SerealToken.END) {
+ *     if (decoder.nextToken() == SerealToken.LONG) {
+ *       System.outprintln("Long value " + decoder.longValue() + " at offset " + decoder.tokenOffset());
+ *     }
+ *   }
+ * }
+ *
+ */ +public class TokenDecoder { + private static class Context { + private final Context outer; + private final int type; + private int total, remaining; + private int originalPosition; + + Context(Context outer, int type, int remaining) { + this.outer = outer; + this.type = type; + this.remaining = this.total = remaining; + } + + Context(Context outer, int type, int remaining, int originalPosition) { + this.outer = outer; + this.type = type; + this.remaining = this.total = remaining; + this.originalPosition = originalPosition; + } + } + + private static class AllowedEncoding { + private final EncoderOptions.CompressionType compressionType; + private final int encoding, minVersion, maxVersion; + + AllowedEncoding(EncoderOptions.CompressionType compressionType, int encoding, int minVersion, int maxVersion) { + this.compressionType = compressionType; + this.encoding = encoding; + this.minVersion = minVersion; + this.maxVersion = maxVersion; + } + } + + private static final DecoderOptions DEFAULT_OPTIONS = new DecoderOptions(); + private static final AllowedEncoding[] ALLOWED_ENCODINGS = new AllowedEncoding[] { + new AllowedEncoding(EncoderOptions.CompressionType.NONE, 0, 1, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.SNAPPY, 1, 1, 1), + new AllowedEncoding(EncoderOptions.CompressionType.SNAPPY, 2, 2, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.ZLIB, 3, 3, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.ZSTD, 4, 4, Integer.MAX_VALUE), + }; + + private static final int CONTEXT_ROOT = 0; + private static final int CONTEXT_HASH = 1; + private static final int CONTEXT_ARRAY = 2; + private static final int CONTEXT_OBJECT = 3; + private static final int CONTEXT_SUBDECODE = 4; + + private byte[] data; + private int position, end; + private byte[] bodyData; + private int bodyPosition, bodySize; + private ByteArray originalData; + private int protocolVersion = -1; + private int encoding = -1; + private int baseOffset = Integer.MAX_VALUE; + private int userHeaderPosition = -1; + private int userHeaderSize = -1; + private Inflater inflater; + private byte[] bigintBuffer; + + private Context currentContext; + private SerealToken currentToken = SerealToken.NONE; + private boolean isSecondTime; + private int trackOffset, tokenOffset; + private long longValue; + private float floatValue; + private double doubleValue; + private int binarySliceStart, binarySliceEnd; + private boolean binaryIsUtf8; + private int backreferenceOffset; + + /** Create a new {@code TokenDecoder} with default options. */ + public TokenDecoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create an new {@code TokenDecoder} with the specified options. + * + * @param options {@link DecoderOptions} to use + */ + public TokenDecoder(DecoderOptions options) { + } + + private void checkHeaderSuffix() { + int suffixSize = (int) readVarint(); + + userHeaderSize = 0; + userHeaderPosition = position + suffixSize; + if (protocolVersion >= 2 && suffixSize > 0) { + byte bitfield = data[position++]; + + if ((bitfield & 0x01) == 0x01) { + userHeaderPosition = position; + userHeaderSize = suffixSize - 1; + } + } + } + + private void checkNoEOD() throws SerealException { + if ((end - position) <= 0) { + throw new SerealException("Unexpected end of data at byte " + position); + } + } + + private void checkMagicProtoAndFlags() throws SerealException { + int magic = + ((data[position] & 0xff) << 24) + + ((data[position + 1] & 0xff) << 16) + + ((data[position + 2] & 0xff) << 8) + + (data[position + 3] & 0xff); + protocolVersion = data[position + 4] & 15; // 4 bits for version + if (magic != SerealHeader.MAGIC && magic != SerealHeader.MAGIC_V3) { + throw new SerealException( + String.format("Invalid Sereal header (%08x): doesn't match magic", magic)); + } else if (protocolVersion < 1 || protocolVersion > 4) { + throw new SerealException( + String.format("Invalid Sereal header: unsupported protocol version %d", protocolVersion)); + } else if (magic == SerealHeader.MAGIC && protocolVersion > 2) { + throw new SerealException( + String.format("Invalid Sereal header: magic v1 with protocol version %d", protocolVersion)); + } else if (magic == SerealHeader.MAGIC_V3 && protocolVersion < 3) { + throw new SerealException( + String.format("Invalid Sereal header: magic v3 with protocol version %d", protocolVersion)); + } + + encoding = (data[position + 4] >> 4) & 0xf; + if (encoding > 4) { + throw new SerealException("Unsupported Sereal body encoding " + encoding); + } + for (AllowedEncoding allowedEncoding : ALLOWED_ENCODINGS) { + if (allowedEncoding.encoding == encoding) { + if (protocolVersion < allowedEncoding.minVersion || protocolVersion > allowedEncoding.maxVersion) { + throw new SerealException(String.format("Unsupported encoding %d (%s) for Sereal protocol %d", + encoding, allowedEncoding.compressionType, protocolVersion)); + } + } + } + position += 5; + } + + /** + * @return {@code true} if the document has a Sereal header. + * + * @throws SerealException header cannot be parsed. + */ + public boolean hasHeader() throws SerealException { + parseHeader(); + + return userHeaderSize > 0; + } + + /** + * @return Size of the Sereal header, if present, 0 otherwise. + * + * @throws SerealException header cannot be parsed. + */ + public int headerSize() throws SerealException { + parseHeader(); + + return userHeaderSize > 0 ? userHeaderSize : 0; + } + + /** + * Set up the decoder to iterate over the Sereal header. + *

+ * This function will fail if {@link TokenDecoder#hasHeader()} returned {@code false}. + * + * @throws SerealException Token cannot be decoded. + */ + public void prepareDecodeHeader() throws SerealException { + parseHeader(); + + if (protocolVersion == 1) { + throw new SerealException("Sereal user header not supported in protocol version 1"); + } + if (userHeaderSize <= 0) { + throw new SerealException("Sereal user header not present"); + } + + data = originalData.array; + position = userHeaderPosition; + end = userHeaderPosition + userHeaderSize; + + // because offsets start at 1 + baseOffset = position - 1; + + currentContext = new Context(null, CONTEXT_ROOT, 1); + } + + private void parseHeader() throws SerealException { + if (originalData == null) { + throw new SerealException("No data set"); + } + if (userHeaderSize >= 0) { + return; + } + if (originalData.length < 7) { + throw new SerealException("Invalid Sereal document: total size is too small"); + } + + data = originalData.array; + end = originalData.start + originalData.length; + position = originalData.start; + + checkMagicProtoAndFlags(); + checkHeaderSuffix(); + } + + /** + * Set up the decoder to iterate over the Sereal document body. + * + * @throws SerealException Token cannot be decoded. + */ + public void prepareDecodeBody() throws SerealException { + parseHeader(); + + if (bodyData == null) { + if (encoding != 0) { + position = userHeaderPosition + userHeaderSize; + end = originalData.length; + if (encoding == SerealHeader.SRL_ENCODING_SNAPPY_LEGACY || encoding == SerealHeader.SRL_ENCODING_SNAPPY) { + uncompressSnappy(); + } else if (encoding == SerealHeader.SRL_ENCODING_ZLIB) { + uncompressZlib(); + } else if (encoding == SerealHeader.SRL_ENCODING_ZSTD) { + uncompressZstd(); + } + } else { + bodyData = data; + bodyPosition = userHeaderPosition + userHeaderSize; + bodySize = originalData.start + originalData.length; + } + } + + data = bodyData; + position = bodyPosition; + end = bodySize; + + if (protocolVersion == 1) { + baseOffset = 0; + } else { + // because offsets start at 1 + baseOffset = position - 1; + } + + currentContext = new Context(null, CONTEXT_ROOT, 1); + } + + private void uncompressSnappy() throws SerealException { + int len = originalData.length - (position - originalData.start); + int pos = encoding == SerealHeader.SRL_ENCODING_SNAPPY_LEGACY ? position : originalData.start; + + if (encoding == SerealHeader.SRL_ENCODING_SNAPPY) { + len = (int) readVarint(); + } + byte[] uncompressed; + try { + if (!Snappy.isValidCompressedBuffer( + originalData.array, position, len)) + throw new SerealException("Invalid snappy data"); + uncompressed = + new byte + [pos + + Snappy.uncompressedLength( + originalData.array, position, len)]; + Snappy.uncompress( + originalData.array, position, len, uncompressed, pos); + } catch (IOException e) { + throw new SerealException(e); + } + this.bodyData = uncompressed; + this.bodyPosition = pos; + this.bodySize = uncompressed.length; + } + + private void uncompressZlib() throws SerealException { + if (inflater == null) { + inflater = new Inflater(); + } + inflater.reset(); + + long uncompressedLength = readVarint(); + long compressedLength = readVarint(); + inflater.setInput(originalData.array, position, (int) compressedLength); + byte[] uncompressed = new byte[(int) uncompressedLength]; + try { + int inflatedSize = inflater.inflate(uncompressed); + if (inflatedSize != uncompressedLength || !inflater.finished()) { + throw new SerealException("Error in zlib-compressed data"); + } + } catch (DataFormatException e) { + throw new SerealException(e); + } + this.bodyData = uncompressed; + this.bodyPosition = 0; + this.bodySize = uncompressed.length; + } + + private void uncompressZstd() throws SerealException { + int len = (int) readVarint(); + + byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len); + long decompressedSize = Zstd.decompressedSize(compressedData); + if (decompressedSize > Integer.MAX_VALUE) { + throw new SerealException("Decompressed size exceeds integer MAX_VALUE: " + decompressedSize); + } + + byte[] uncompressed = new byte[(int) decompressedSize]; + long status = Zstd.decompress(uncompressed, compressedData); + if (Zstd.isError(status)) { + throw new SerealException(Zstd.getErrorName(status)); + } + this.bodyData = uncompressed; + this.bodyPosition = 0; + this.bodySize = uncompressed.length; + } + + private void readBinary() { + int length = (int) readVarint(); + binaryIsUtf8 = false; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + // top bit set (0x80) means next byte is 7 bits more more varint + private long readVarint() { + long uv = 0; + int lshift = 0; + + byte b = data[position++]; + while ((position < end) && (b < 0)) { + uv |= ((long) b & 127) << lshift; // add 7 bits + lshift += 7; + b = data[position++]; + } + uv |= (long) b << lshift; // add final (or first if there is only 1) + + return uv; + } + + /** + * Iterate over the Sereal document returning the next token. + *

+ * Depending on the token returned, accessors can be used to retrieve additional + * information about the token. + * + * @return The next token + * + * @throws SerealException Token cannot be decoded. + */ + public SerealToken nextToken() throws SerealException { + if (currentContext.remaining == 0) { + Context popped = currentContext; + currentContext = popped.outer; + switch (popped.type) { + case CONTEXT_ROOT: + return (currentToken = SerealToken.END); + case CONTEXT_ARRAY: + return (currentToken = SerealToken.ARRAY_END); + case CONTEXT_HASH: + return (currentToken = SerealToken.HASH_END); + case CONTEXT_OBJECT: + return (currentToken = SerealToken.OBJECT_END); + case CONTEXT_SUBDECODE: + position = popped.originalPosition; + return (currentToken = SerealToken.SUBDECODE_END); + } + } + + checkNoEOD(); + + byte tag = data[position++]; + + tokenOffset = position - 1 - baseOffset; + trackOffset = 0; + if ((tag & SerealHeader.SRL_HDR_TRACK_FLAG) != 0) { + tag = (byte) (tag & ~SerealHeader.SRL_HDR_TRACK_FLAG); + trackOffset = tokenOffset; + } + + currentContext.remaining--; + + if (tag <= SerealHeader.SRL_HDR_POS_HIGH) { + longValue = (long) tag; + return (currentToken = SerealToken.LONG); + } else if (tag <= SerealHeader.SRL_HDR_NEG_HIGH) { + longValue = (long) (tag - 32); + return (currentToken = SerealToken.LONG); + } else if ((tag & SerealHeader.SRL_HDR_SHORT_BINARY_LOW) == SerealHeader.SRL_HDR_SHORT_BINARY_LOW) { + readShortBinary(tag); + return (currentToken = SerealToken.BINARY); + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + if (isSecondTime) { + isSecondTime = false; + trackOffset = 0; + int numKeys = tag & 0xf; + currentContext = new Context(currentContext, CONTEXT_HASH, numKeys * 2); + return (currentToken = SerealToken.HASH_START); + } else { + isSecondTime = true; + --position; + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + } + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + if (isSecondTime) { + isSecondTime = false; + trackOffset = 0; + int length = tag & 0xf; + currentContext = new Context(currentContext, CONTEXT_ARRAY, length); + return (currentToken = SerealToken.ARRAY_START); + } else { + isSecondTime = true; + --position; + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + } + } else { + switch (tag) { + case SerealHeader.SRL_HDR_VARINT: + longValue = readVarint(); + return (currentToken = (longValue >= 0 ? SerealToken.LONG : SerealToken.UNSIGNED_LONG)); + case SerealHeader.SRL_HDR_ZIGZAG: + longValue = readZigzag(); + return (currentToken = SerealToken.LONG); + case SerealHeader.SRL_HDR_FLOAT: + int floatBits = + ((data[position + 3] & 0xff) << 24) + + ((data[position + 2] & 0xff) << 16) + + ((data[position + 1] & 0xff) << 8) + + (data[position] & 0xff); + position += 4; + floatValue = Float.intBitsToFloat(floatBits); + return (currentToken = SerealToken.FLOAT); + case SerealHeader.SRL_HDR_DOUBLE: + long doubleBits = + ((long) (data[position + 7] & 0xff) << 56) + + ((long) (data[position + 6] & 0xff) << 48) + + ((long) (data[position + 5] & 0xff) << 40) + + ((long) (data[position + 4] & 0xff) << 32) + + ((long) (data[position + 3] & 0xff) << 24) + + ((long) (data[position + 2] & 0xff) << 16) + + ((long) (data[position + 1] & 0xff) << 8) + + ((long) (data[position] & 0xff)); + position += 8; + doubleValue = Double.longBitsToDouble(doubleBits); + return (currentToken = SerealToken.DOUBLE); + case SerealHeader.SRL_HDR_TRUE: + return (currentToken = SerealToken.TRUE); + case SerealHeader.SRL_HDR_FALSE: + return (currentToken = SerealToken.FALSE); + case SerealHeader.SRL_HDR_UNDEF: + return (currentToken = SerealToken.UNDEF); + case SerealHeader.SRL_HDR_CANONICAL_UNDEF: + return (currentToken = SerealToken.CANONICAL_UNDEF); + case SerealHeader.SRL_HDR_BINARY: + readBinary(); + return (currentToken = SerealToken.BINARY); + case SerealHeader.SRL_HDR_STR_UTF8: + readUTF8(); + return (currentToken = SerealToken.UTF8); + case SerealHeader.SRL_HDR_REFN: + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + case SerealHeader.SRL_HDR_REFP: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.REFP); + case SerealHeader.SRL_HDR_OBJECT: + currentContext = new Context(currentContext, CONTEXT_OBJECT, 1); + readString(); + return (currentToken = SerealToken.OBJECT_START); + case SerealHeader.SRL_HDR_OBJECTV: + currentContext = new Context(currentContext, CONTEXT_OBJECT, 1); + readStringCopy(); + return (currentToken = SerealToken.OBJECT_START); + case SerealHeader.SRL_HDR_COPY: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.COPY); + case SerealHeader.SRL_HDR_ALIAS: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.ALIAS); + case SerealHeader.SRL_HDR_WEAKEN: + currentContext.remaining++; + return (currentToken = SerealToken.WEAKEN); + case SerealHeader.SRL_HDR_HASH: + int numKeys = (int) readVarint(); + currentContext = new Context(currentContext, CONTEXT_HASH, numKeys * 2); + return (currentToken = SerealToken.HASH_START); + case SerealHeader.SRL_HDR_ARRAY: + int length = (int) readVarint(); + currentContext = new Context(currentContext, CONTEXT_ARRAY, length); + return (currentToken = SerealToken.ARRAY_START); + case SerealHeader.SRL_HDR_REGEXP: + readRegexp(); + return (currentToken = SerealToken.REGEXP); + case SerealHeader.SRL_HDR_PAD: + currentContext.remaining++; + return nextToken(); + default: + throw new SerealException("Tag not supported: " + tag); + } + } + } + + /** + * Starts a nested decoding context at the specified position. + *

+ * Position must point at the start of a Sereal tag. + *

+ * After the tag has been completely parsed (including any nested values), {@link #nextToken} + * returns {@link SerealToken#SUBDECODE_END}, and the decoder position is restored to the + * one before this call. + * + * @param offset Sereal document offset. + * + * @throws SerealException Token cannot be decoded. + */ + public void startSubDecode(int offset) throws SerealException { + currentContext = new Context(currentContext, CONTEXT_SUBDECODE, 1, position); + position = offset + baseOffset; + } + + /** + * @return After a call to {@link #nextToken}, return the current token. + */ + public SerealToken currentToken() { + return currentToken; + } + + /** + * @return The element count of the current parsing context. + *

+ * Between {@link SerealToken#ARRAY_START} and {@link SerealToken#ARRAY_END} returns the number of elements in the array. + *

+ * Between {@link SerealToken#HASH_START} and {@link SerealToken#HASH_END} returns the number of + * keys + values in the hash (always a multiple of 2). + *

+ * Between {@link SerealToken#OBJECT_START} and {@link SerealToken#OBJECT_END} returns 1. + *

+ * At the top-level of a sub-parse, returns 1. + */ + public int elementCount() { + return currentContext.total; + } + + /** + * @return Whether the offset of the current tag is the target of a reference. + *

+ * When it returns a non-zero value, it is the offset of a Sereal tag that is going to be references + * by a later {@link SerealToken#ALIAS} or {@link SerealToken#REFP} token. + */ + public int trackOffset() { + return trackOffset; + } + + /** + * The offset of the last token returned by {@link TokenDecoder#nextToken()}. + *

+ * Defined for all tokens except for {@link SerealToken#ARRAY_END}, {@link SerealToken#HASH_END} + * and {@link SerealToken#OBJECT_END}. + * + * @return a token offset that can be used for {@link #startSubDecode(int)} + */ + public int tokenOffset() { + return tokenOffset; + } + + /** + * @return Decoder position in the Sereal document. + *

+ * Defined for all tokens. + */ + public int currentOffset() { + return position - 1 - baseOffset; + } + + /** + * @return Current {@code long} value. + *

+ * Defined for {@link SerealToken#LONG} and {@link SerealToken#UNSIGNED_LONG}. + */ + public long longValue() { + return longValue; + } + + /** + * @return Current {@code long} value as a {@link java.math.BigInteger}. + *

+ * Defined for {@link SerealToken#LONG} and {@link SerealToken#UNSIGNED_LONG}. + */ + public BigInteger bigintValue() { + if (currentToken == SerealToken.UNSIGNED_LONG) { + if (bigintBuffer == null) { + bigintBuffer = new byte[8]; + } + long temp = longValue; + for (int i = 7; i >= 0; --i) { + bigintBuffer[i] = (byte) (temp & 0xff); + temp >>= 8; + } + return new BigInteger(1, bigintBuffer); + } else { + return BigInteger.valueOf(longValue); + } + } + + /** + * @return Current {@code float} value. + *

+ * Defined for {@link SerealToken#FLOAT}. + */ + public float floatValue() { + return floatValue; + } + + /** + * @return Current {@code double} value. + *

+ * Defined for {@link SerealToken#DOUBLE}. + */ + public double doubleValue() { + return doubleValue; + } + + /** + * All binary/string values are slices of this array. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + * + * @return Token binary data + */ + public byte[] decoderBuffer() { + return data; + } + + /** + * @return Start offset of the current binary/string in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceStart() { + return binarySliceStart; + } + + /** + * @return End offset of the current binary/string in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceEnd() { + return binarySliceEnd; + } + + /** + * @return Length of the current binary/string. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceLength() { + return binarySliceEnd - binarySliceStart; + } + + /** + * @return Start offset of the current regexp flags in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceStart() { + return (int) longValue; + } + + /** + * @return End offset of the current regexp flags in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceEnd() { + return backreferenceOffset; + } + + /** + * @return Length of the regexp flags string. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceLength() { + return regexpFlagsSliceEnd() - regexpFlagsSliceStart(); + } + + /** + * @return {@code true} if the byte slice between {@link TokenDecoder#binarySliceStart()}/{@link TokenDecoder#binarySliceEnd()} is UTF-8 encoded. + *

+ * Always {@code true} for {@link SerealToken#UTF8}, always {@code false} for {@link SerealToken#BINARY}. + */ + public boolean binaryIsUtf8() { + return binaryIsUtf8; + } + + /** + * @return Offset into the Sereal document. + *

+ * Defined for {@link SerealToken#REFP}, {@link SerealToken#ALIAS} and {@link SerealToken#COPY} token. + */ + public int backreferenceOffset() { + return backreferenceOffset; + } + + /** @return {@code true} if the current token is a hash key, {@code false} otherwise. */ + public boolean isHashKey() { + Context context = currentContext; + while (context.type == CONTEXT_SUBDECODE) { + context = context.outer; + } + return context.type == CONTEXT_HASH && (context.remaining & 1) == 1; + } + + private void readShortBinary(byte tag) { + int length = tag & SerealHeader.SRL_MASK_SHORT_BINARY_LEN; + binaryIsUtf8 = false; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + private void readStringCopy() throws SerealException { + int originalPosition = (int) readVarint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + readString(); + position = currentPosition; // go back to where we were + } + + private void readUTF8() { + int length = (int) readVarint(); + binaryIsUtf8 = true; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + private long readZigzag() { + long n = readVarint(); + + return (n >>> 1) ^ (-(n & 1)); // note the unsigned right shift + } + + private void readRegexp() throws SerealException { + readString(); + + boolean savedBinaryIfUtf8 = binaryIsUtf8; + int savedBinarySliceStart = binarySliceStart, savedBinarySliceEnd = binarySliceEnd; + + readString(); + + longValue = binarySliceStart; + backreferenceOffset = binarySliceEnd; + + binaryIsUtf8 = savedBinaryIfUtf8; + binarySliceStart = savedBinarySliceStart; + binarySliceEnd = savedBinarySliceEnd; + } + + private void readString() throws SerealException { + checkNoEOD(); + + byte tag = data[position++]; + + if ((tag & SerealHeader.SRL_HDR_SHORT_BINARY_LOW) == SerealHeader.SRL_HDR_SHORT_BINARY_LOW) { + readShortBinary(tag); + } else if (tag == SerealHeader.SRL_HDR_BINARY) { + readBinary(); + } else if (tag == SerealHeader.SRL_HDR_STR_UTF8) { + readUTF8(); + } else if (tag == SerealHeader.SRL_HDR_COPY) { + readStringCopy(); + } else { + throw new SerealException("Tag " + tag + " is not a string tag"); + } + } + + /** + * Set the Sereal data to be decoded. + *

+ * After calling this method, blob is owned by the decoder until the next call to {@code setData} or {@link #reset()}. + * + * @param blob Sereal data. + */ + public void setData(ByteArray blob) { + reset(); + originalData = blob; + } + + /** + * Set the Sereal data to be decoded. + *

+ * After calling this method, blob is owned by the decoder until the next call to {@code setData} or {@link #reset()}. + * + * @param blob Sereal data. + */ + public void setData(byte[] blob) { + setData(new ByteArray(blob)); + } + + /** Discard all internal state. */ + public void reset() { + currentToken = SerealToken.NONE; + currentContext = null; + originalData = null; + data = bodyData = null; + protocolVersion = encoding = -1; + baseOffset = Integer.MAX_VALUE; + userHeaderPosition = userHeaderSize = -1; + isSecondTime = false; + } + + /** + * Decode a string of bytes as Perl regexp modifiers, and return the corresponding {@link java.util.regex.Pattern} flags. + *

+ * It only handles {@code m, s, i, x} flags. + * + * @param buffer Buffer containing the modifier characters. + * @param start Start of the range containing the modifier characters. + * @param end Start of the range containing the modifier characters. + * @return The decoded modifiers as {@link java.util.regex.Pattern} flags + * @throws SerealException on an unrecognized modifier + */ + public static int decodeRegexpFlags(byte[] buffer, int start, int end) throws SerealException { + int flags = 0; + + for (int i = start; i < end; ++i) { + byte value = buffer[i]; + switch (value) { + case 'm': + flags = flags | Pattern.MULTILINE; + break; + case 's': + flags = flags | Pattern.DOTALL; + break; + case 'i': + flags = flags | Pattern.CASE_INSENSITIVE; + break; + case 'x': + flags = flags | Pattern.COMMENTS; + break; + case 'p': + // ignored + break; + default: + throw new SerealException(String.format("Unknown regexp modifier: '%c'", value)); + } + } + + return flags; + } + + // used for testing + SerealToken poisonNextToken() throws SerealException { + poisonFields(); + return nextToken(); + } + + private void poisonFields() { + trackOffset = (int) (Math.random() * Integer.MAX_VALUE); + tokenOffset = (int) (Math.random() * Integer.MAX_VALUE); + longValue = (long) (Math.random() * Integer.MAX_VALUE); + floatValue = (float) (Math.random() * Float.MAX_VALUE); + doubleValue = Math.random() * Double.MAX_VALUE; + binarySliceStart = (int) (Math.random() * Integer.MAX_VALUE); + binarySliceEnd = (int) (Math.random() * Integer.MAX_VALUE);; + binaryIsUtf8 = Math.random() > 0.5; + backreferenceOffset = (int) (Math.random() * Integer.MAX_VALUE);; + } + + /** + * Close the decoder to recycle the resources. + *

+ * Returns the native memory used by inflater explicitly before the garbage collector. + */ + public void close() { + if (inflater != null) { + inflater.end(); + inflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/TokenEncoder.java b/src/main/java/com/booking/sereal/TokenEncoder.java new file mode 100644 index 000000000..f323fe861 --- /dev/null +++ b/src/main/java/com/booking/sereal/TokenEncoder.java @@ -0,0 +1,1261 @@ +package com.booking.sereal; + +import com.booking.sereal.EncoderOptions.CompressionType; +import com.github.luben.zstd.Zstd; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.regex.Pattern; +import java.util.zip.Deflater; +import org.xerial.snappy.Snappy; + +/** + * A low-level stream encoder for Sereal. + *

+ * Using this class correctly requires understanding of the Sereal specification and how the Perl encoder + * handles the produced Sereal. + *

+ * Example: + * 

+ * {@code
+ *   encoder.startDocument();
+ *
+ *   encoder.startArray(2);
+ *   encoder.appendString("Hello, Sereal");
+ *   encoder.appendLong(4);
+ *   encoder.endArray();
+ *
+ *   encoder.endDocument();
+ *
+ *   byte[] data = encoder.getData();
+ * }
+ *
+ */ +public class TokenEncoder { + private static class Context { + private final Context outer; + private final int type; + private final int expectedCount; + private final int position; + private int count; + + Context(Context outer, int type, int position, int expectedCount) { + this.outer = outer; + this.type = type; + this.position = position; + this.expectedCount = expectedCount; + } + } + + private static final int CONTEXT_ROOT = 0; + private static final int CONTEXT_HASH = 1; + private static final int CONTEXT_ARRAY = 2; + private static final int CONTEXT_OBJECT = 3; + private static final int CONTEXT_WEAKEN = 4; + private static final int CONTEXT_INITIAL = 5; + private static final int CONTEXT_HEADER = 6; + private static final int CONTEXT_FINAL = 7; + + private static final EncoderOptions DEFAULT_OPTIONS = new EncoderOptions(); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private final int MAX_VARINT_LENGTH = 10; + private final byte[] HEADER = + new byte[] { + (byte) (SerealHeader.MAGIC >> 24), + (byte) (SerealHeader.MAGIC >> 16), + (byte) (SerealHeader.MAGIC >> 8), + (byte) (SerealHeader.MAGIC), + }; + private final byte[] HEADER_V3 = + new byte[] { + (byte) (SerealHeader.MAGIC_V3 >> 24), + (byte) (SerealHeader.MAGIC_V3 >> 16), + (byte) (SerealHeader.MAGIC_V3 >> 8), + (byte) (SerealHeader.MAGIC_V3), + }; + + private final byte protocolVersion, encoding; + private final CompressionType compressionType; + private final CharsetEncoder utf8Encoder = StandardCharsets.UTF_8.newEncoder(); + private Deflater deflater; + private final int compressionThreshold; + private Context currentContext; + private byte[] bytes = new byte[1024], compressedBytes = EMPTY_BYTE_ARRAY; + private int size = 0, compressedSize = 0, trackOffset = 0; + private int headerSize, headerOffset; + private final int zstdCompressionLevel; + private boolean hasHeader; + + /** Create an new {@code TokenEncoder} with default options. */ + public TokenEncoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create an new {@code TokenEncoder} with the specified options. + * + * @param options {@link EncoderOptions} to use. + */ + public TokenEncoder(EncoderOptions options) { + protocolVersion = (byte) options.protocolVersion(); + compressionType = options.compressionType(); + zstdCompressionLevel = options.zstdCompressionLevel(); + compressionThreshold = (int) options.compressionThreshold(); + + if (compressionType == CompressionType.SNAPPY) { + encoding = protocolVersion == 1 ? SerealHeader.SRL_ENCODING_SNAPPY_LEGACY : SerealHeader.SRL_ENCODING_SNAPPY; + } else { + encoding = compressionType.encoding; + } + + if (encoding == 3) { + deflater = new Deflater(options.zlibCompressionLevel()); + } else { + deflater = null; + } + + start(); + } + + /** @return Sereal protocol version used by this encoder. */ + public int protocolVersion() { + return protocolVersion; + } + + /** @return {@code true} after the root element has been completely written. */ + public boolean isComplete() { + return currentContext.type == CONTEXT_ROOT && currentContext.count == 1; + } + + /** + * @return Sereal document offset of last written token. + *

+ * The returned value can be used for the target of {@link TokenEncoder#appendCopy(int)}, + * {@link TokenEncoder#appendRefPrevious(int)} or {@link TokenEncoder#appendAlias(int)}. + */ + public int trackOffsetLastValue() { + return trackOffset - headerOffset; + } + + /** + * @return Sereal document offset of the next token that will be written. + *

+ * The returned value can be used for the target of {@link TokenEncoder#appendCopy(int)}, + * {@link TokenEncoder#appendRefPrevious(int)} or {@link TokenEncoder#appendAlias(int)}. + */ + public int trackOffsetNextValue() { + return size - headerOffset; + } + + /** Reset internal state as it was right after construction. */ + public void reset() { + size = compressedSize = headerSize = headerOffset = trackOffset = 0; + hasHeader = false; + start(); + } + + /** + * @return Get a reference to the encoded document. + *

+ * The contents of the buffer will become invalid after calling any of the mutator methods. + */ + public ByteArray getDataReference() { + checkParts(); + if (compressedSize != 0) { + return new ByteArray(compressedBytes, compressedSize); + } else { + return new ByteArray(bytes, size); + } + } + + /** @return Get a copy of the encoded document. */ + public byte[] getData() { + checkParts(); + if (compressedSize != 0) { + return Arrays.copyOf(compressedBytes, compressedSize); + } else { + return Arrays.copyOf(bytes, size); + } + } + + private void checkParts() { + if (currentContext.outer != null) { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Missing endHash/endArray/endObject call"); + } else { + throw new IllegalStateException("Missing endHeader/endDocument call"); + } + } + if (currentContext.type == CONTEXT_INITIAL || currentContext.type == CONTEXT_HEADER) { + throw new IllegalStateException("Missing startDocument/endDocument calls"); + } else if (currentContext.type != CONTEXT_FINAL) { + throw new IllegalStateException(""); + } + } + + private static void prepareHeader( + byte[] originBytes, byte[] compressedBytes, int headerSize, int sizeLength) { + System.arraycopy(originBytes, 0, compressedBytes, 0, headerSize); + if (sizeLength > 0) { + // varint-encoded 0, filling all space + for (int i = headerSize; i < headerSize + sizeLength - 1; i++) { + compressedBytes[i] = (byte) 0x80; + } + compressedBytes[headerSize + sizeLength - 1] = 0; + } + } + + private static void finishHeader( + byte[] compressedBytes, long compressedSize, int headerSize, int sizeLength) { + int after = encodeVarint(compressedSize, compressedBytes, headerSize); + if (after != headerSize + sizeLength) { + compressedBytes[after - 1] |= (byte) 0x80; + } + } + + private void start() { + if (protocolVersion >= 3) { + appendBytesUnsafe(HEADER_V3); + } else { + appendBytesUnsafe(HEADER); + } + appendByteUnsafe((byte) ((encoding << 4) | protocolVersion)); + currentContext = new Context(null, CONTEXT_INITIAL, 0, 0); + } + + /** + * Set up the encoder to emit data to the Sereal header. + *

+ * Sereal header is optional, and if present it must be emitted before the main document. + * + * @throws SerealException header cannot be encoded. + */ + public void startHeader() throws SerealException { + if (protocolVersion == 1) { + throw new SerealException("Can't encode user header in Sereal protocol version 1"); + } else if (currentContext.outer != null) { + throw new IllegalStateException("startHeader called while already inside startHeader/startDocumentt"); + } else if (currentContext.type != CONTEXT_INITIAL) { + if (currentContext.type == CONTEXT_HEADER) { + throw new IllegalStateException("startHeader called twice"); + } else { + throw new IllegalStateException("startHeader called after emitting document body"); + } + } + + currentContext = new Context(currentContext, CONTEXT_ROOT, size, 1); + // be optimistic about encoded header size + size += 2; // one for the size, one for 8bit bitfield + // because offsets start at 1 + headerOffset = size - 1; + } + + /** + * Complete encoding of the Sereal header. + * + * @throws SerealException header cannot be encoded. + */ + public void endHeader() throws SerealException { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + hasHeader = true; + headerSize = size; + + int originalSize = currentContext.position; + int suffixSize = (size - originalSize - 1); + if (suffixSize < 128) { + bytes[originalSize] = (byte) suffixSize; + bytes[originalSize + 1] = 0x01; + } else { + // we were too optimistic + int sizeLength = varintLength(suffixSize); + + // make space + ensureAvailable(sizeLength - 1); + System.arraycopy( + bytes, + originalSize + 2, + bytes, + originalSize + sizeLength + 1, + suffixSize - 1); + size += sizeLength - 1; + headerSize = size; + + // now write size and 8bit bitfield + encodeVarint(suffixSize, bytes, originalSize); + bytes[originalSize + sizeLength] = 0x01; + } + currentContext = new Context(null, CONTEXT_HEADER, -1, 0); + } + + /** + * Set up the encoder to emit data for the Sereal body. + * + * @throws SerealException document cannot be encoded. + */ + public void startDocument() throws SerealException { + if (currentContext.outer != null) { + throw new IllegalStateException("startDocument called while already inside startHeader/startDocument"); + } + if (currentContext.type == CONTEXT_FINAL) { + throw new IllegalStateException("startDocument called twice"); + } + + currentContext = new Context(null, CONTEXT_FINAL, -1, 0); + if (!hasHeader) { + appendByteUnsafe((byte) 0x00); + headerSize = size; + } else { + if (headerSize != size) { + throw new SerealException("Can't append data between header and body"); + } + } + + currentContext = new Context(currentContext, CONTEXT_ROOT, -1, 1); + if (protocolVersion > 1) { + // because offsets start at 1 + headerOffset = headerSize - 1; + } else { + headerOffset = 0; + } + } + + /** + * Complete encoding of the Sereal body. + * + * @throws SerealException document cannot be encoded. + */ + public void endDocument() throws SerealException { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (!compressionType.equals(CompressionType.NONE) && size - headerSize > compressionThreshold) { + if (compressionType.equals(CompressionType.SNAPPY)) { + compressSnappy(); + } else if (compressionType.equals(CompressionType.ZLIB)) { + compressZlib(); + } else if (compressionType.equals(CompressionType.ZSTD)) { + compressZstd(); + } + } else { + // we did not do compression after all + markNotCompressed(); + } + currentContext = currentContext.outer; + } + + private void checkCount(Context context) throws SerealException { + if (context.expectedCount != -1 && context.count != context.expectedCount) { + throw new SerealException("Bad value count"); + } + } + + /** + * Append a Sereal {@code REFN} tag. + * + * @throws SerealException reference cannot be encoded. + */ + public void appendRefNext() throws SerealException { + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REFN); + } + + /** + * Append a Sereal {@code WEAKEN} tag. + *

+ * The next value appended needs to be some kind of reference. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendWeaken() throws SerealException { + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_WEAKEN); + } + + /** + * Convenience method to check or force the next emitted value to be a weak reference. + *

+ * After emitting the value, {@link TokenEncoder#endWeaken()} must be called. + *

+ * If the emitted value is not a reference and {@code forceReference} is {@code false}, an exception is thrown. + *

+ * If the emitted value is not a reference and {@code forceReference} is {@code true}, the value + * is forced into a reference by using {@code REFN}. + * + * @param forceReference {@code true} to force the value as a reference, {@code false} otherwise + * + * @throws SerealException tag cannot be encoded. + */ + public void startWeaken(boolean forceReference) throws SerealException { + ensureAvailable(2); + appendByteUnsafe(SerealHeader.SRL_HDR_WEAKEN); + + if (forceReference) { + appendByteUnsafe(SerealHeader.SRL_HDR_PAD); + } + currentContext = new Context(currentContext, CONTEXT_WEAKEN, size - 1,1); + } + + /** + * Completes writing a weak reference started with {@link TokenEncoder#startWeaken(boolean)}. + * + * @throws SerealException tag cannot be encoded. + */ + public void endWeaken() throws SerealException { + if (currentContext.type != CONTEXT_WEAKEN) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (bytes[currentContext.position] == SerealHeader.SRL_HDR_PAD) { + if (!isRefTag(bytes[currentContext.position + 1])) { + bytes[currentContext.position] = SerealHeader.SRL_HDR_REFN; + } + } else { + if (!isRefTag(bytes[currentContext.position + 1])) { + throw new SerealException("Internal error while encoding weak reference"); + } + } + currentContext = currentContext.outer; + currentContext.count++; + } + + /** + * Append a Sereal {@code REFP} tag. + * + * @param offset Sereal document offset of the target of the reference. + * + * @throws SerealException reference cannot be encoded. + */ + public void appendRefPrevious(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REFP); + appendVarint(offset); + setTrackBit(offset); + } + + /** + * Append a Sereal {@code ALIAS} tag. + * + * @param offset Sereal document offset of the target of the alias. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendAlias(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_ALIAS); + appendVarint(offset); + setTrackBit(offset); + } + + /** + * Append a Sereal {@code COPY} tag. + * + * @param offset Sereal document offset of the target of the copy. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendCopy(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(offset); + } + + /** + * Append an integer value. + *

+ * Depending on the value, uses one one of {@code POS_*}, {@code NEG_*}, {@code VARINT} or {@code ZIGZAG} Sereal tags. + * + * @param l Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendLong(long l) throws SerealException { + currentContext.count++; + trackOffset = size; + if (l < 0) { + if (l > -17) { + appendByte((byte) (SerealHeader.SRL_HDR_NEG_LOW | (l + 32))); + } else { + appendZigZag(l); + } + } else { + if (l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + } + + /** + * Append an unsigned integer value. Negative value are encoded as the positive 640bit value with the same bit pattern. + *

+ * Depending on the value, uses one one of {@code POS_*} or {@code VARINT} Sereal tags. + * + * @param l Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendUnsignedLong(long l) throws SerealException { + currentContext.count++; + trackOffset = size; + if (l >= 0 && l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + + /** + * Append a Sereal {@code FLOAT} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendFloat(float f) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(5); + appendByteUnsafe(SerealHeader.SRL_HDR_FLOAT); + int floatBits = Float.floatToIntBits(f); + for (int i = 0; i < 4; ++i) { + appendByteUnsafe((byte) (floatBits & 0xff)); + floatBits >>= 8; + } + } + + /** + * Append a Sereal {@code DOUBLE} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendDouble(double d) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(9); + appendByteUnsafe(SerealHeader.SRL_HDR_DOUBLE); + long doubleBits = Double.doubleToLongBits(d); + for (int i = 0; i < 8; ++i) { + appendByteUnsafe((byte) (doubleBits & 0xff)); + doubleBits >>= 8; + } + } + + /** + * Append a Sereal {@code TRUE} or {@code FALSE} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendBoolean(boolean b) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(b ? SerealHeader.SRL_HDR_TRUE : SerealHeader.SRL_HDR_FALSE); + } + + /** + * Append a Sereal {@code UNDEF} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendUndef() throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_UNDEF); + } + + /** + * Append a Sereal {@code CANONICAL_UNDEF} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendCanonicalUndef() throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_CANONICAL_UNDEF); + } + + /** + * Append an binary/ISO 8859-1 value. + *

+ * Depending on the value, uses one of {@code SHORT_BINARY_*} or {@code BINARY} Sereal tags. + * + * @param bytes Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendBinary(byte[] bytes) throws SerealException { + appendBinary(bytes, 0, bytes.length); + } + + /** + * Append an binary/ISO 8859-1 value. + *

+ * Depending on the value, uses one of {@code SHORT_BINARY_*} or {@code BINARY} Sereal tags. + * + * @param bytes Value to be appended. + * @param offset Index of the first byte to append. + * @param length Number of bytes to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendBinary(byte[] bytes, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + appendBinaryInternal(bytes, offset, length); + } + + private void appendBinaryInternal(byte[] bytes, int offset, int length) throws SerealException { + if (length <= SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(bytes, offset, length); + } else { + appendLongBinary(bytes, offset, length); + } + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string the value to be appended + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(CharSequence string) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string)); + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(char[] string) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string)); + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string Value to be appended. + * @param offset Index of the first character to append. + * @param length Number of characters to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(char[] string, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string, offset, length)); + } + + private void appendCharBuffer(CharBuffer string) throws SerealException { + int maxLength = string.length() * 3; + int varintLenght = varintLength(maxLength); + ensureAvailable( maxLength + varintLenght + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_STR_UTF8); + utf8Encoder.reset(); + int stringStart = size + varintLenght; + ByteBuffer out = ByteBuffer.wrap(bytes, stringStart, bytes.length - stringStart); + CoderResult result = utf8Encoder.encode(string, out, true); + if (result.isError()) { + throw new SerealException(result.toString()); + } + int actualLength = out.position() - stringStart; + for (int varintEnd = encodeVarint(actualLength, bytes, size); varintEnd < stringStart; ++varintEnd) { + bytes[varintEnd - 1] |= (byte) 0x80; + bytes[varintEnd] = (byte) 0x00; + } + size += varintLenght + actualLength; + } + + /** + * Append a Sereal {@code UTF8} tag. + *

+ * The passed-in value is assumed to be valid UTF-8, no check is performed. + * + * @param utf8 Value to be appended. + * @param offset Index of the first byte to append. + * @param length Number of bytes to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendUTF8(byte[] utf8, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable( length + MAX_VARINT_LENGTH + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_STR_UTF8); + appendVarint(length); + appendBytesUnsafe(utf8, offset, length); + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * Pattern flags other than {@link java.util.regex.Pattern#MULTILINE}, {@link java.util.regex.Pattern#DOTALL}, + * {@link java.util.regex.Pattern#CASE_INSENSITIVE} and {@link java.util.regex.Pattern#COMMENTS} are + * silently ignored. + * + * @throws SerealException value cannot be encoded. + */ + public void appendRegexp(Pattern pattern) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendCharBuffer(CharBuffer.wrap(pattern.pattern())); + ensureAvailable(5); + int nextFlag = size + 1; + int flags = pattern.flags(); + + if ((flags & Pattern.MULTILINE) != 0) { + bytes[nextFlag++] = 'm'; + } + if ((flags & Pattern.DOTALL) != 0) { + bytes[nextFlag++] = 's'; + } + if ((flags & Pattern.CASE_INSENSITIVE) != 0) { + bytes[nextFlag++] = 'i'; + } + if ((flags & Pattern.COMMENTS) != 0) { + bytes[nextFlag++] = 'x'; + } + + bytes[size] = (byte) ((nextFlag - size - 1) | SerealHeader.SRL_HDR_SHORT_BINARY); + size = nextFlag; + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * The pattern is encoded using a Sereal {@code UTF8} tag. + */ + public void appendRegexpString(String pattern, byte[] flags) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendCharBuffer(CharBuffer.wrap(pattern)); + appendBinaryInternal(flags, 0, flags.length); + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * The pattern is encoded using a Sereal {@code BINARY} or {@code SHORT_BINARY_*} tag. + */ + public void appendRegexpBinary(byte[] pattern, byte[] flags) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendBinaryInternal(pattern, 0, pattern.length); + appendBinaryInternal(flags, 0, flags.length); + } + + /** + * Start writing an hash reference. + *

+ * Whenever possible, use {@link TokenEncoder#startHash(int)}. + */ + public void startHash() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + currentContext = new Context(currentContext, CONTEXT_HASH, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an hash reference. + *

+ * Depending on the key count, uses one of {@code HASHREF_*} or {@code REFN + HASH} Sereal tags. + * + * @param count Number of keys in the hash. + */ + public void startHash(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + if (count <= 15) { + appendByteUnsafe((byte) (SerealHeader.SRL_HDR_HASHREF | count)); + } else { + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + appendVarint(count); + } + currentContext = new Context(currentContext, CONTEXT_HASH, size, count); + } + + /** + * Start writing an hash value. + *

+ * Whenever possible, use {@link TokenEncoder#startHashValue(int)}. + */ + public void startHashValue() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + currentContext = new Context(currentContext, CONTEXT_HASH, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an hash value. + *

+ * Uses a Sereal {@code HASH} tag without a preceding {@code REFN}. + * + * @param count Number of keys in the hash. + */ + public void startHashValue(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + appendVarint(count); + currentContext = new Context(currentContext, CONTEXT_HASH, size, count); + } + + /** + * Complete writing an hash value or hash reference. + */ + public void endHash() throws SerealException { + if (currentContext.type != CONTEXT_HASH) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + if ((currentContext.count & 0x1) != 0) { + throw new SerealException("Odd value count in hash"); + } + currentContext.count >>= 1; + checkCount(currentContext); + if (currentContext.expectedCount == -1) { + if (currentContext.count > 127) { + int lenght = varintLength(currentContext.count); + ensureAvailable(lenght); + System.arraycopy(bytes, currentContext.position + 1, + bytes, currentContext.position + lenght, + size - currentContext.position - 1); + size += lenght - 1; + } + encodeVarint(currentContext.count, bytes, currentContext.position); + } + currentContext = currentContext.outer; + } + + /** + * Start writing an array reference. + *

+ * Whenever possible, use {@link TokenEncoder#startArray(int)}. + */ + public void startArray() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an array reference. + *

+ * Depending on the value count, uses one of {@code ARRAYREF_*} or {@code REFN + ARRAY} Sereal tags. + * + * @param count Number of elements in the array. + */ + public void startArray(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + if (count <= 15) { + appendByteUnsafe((byte) (SerealHeader.SRL_HDR_ARRAYREF | count)); + } else { + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + } + currentContext = new Context(currentContext, CONTEXT_ARRAY, size, count); + } + + /** + * Start writing an array value. + *

+ * Whenever possible, use {@link TokenEncoder#startArrayValue(int)}. + */ + public void startArrayValue() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an array value. + *

+ * Uses a Sereal {@code ARRAY} tag without a preceding {@code REFN}. + */ + public void startArrayValue(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size, count); + appendVarint(count); + } + + /** + * Complete writing an array value or array reference. + */ + public void endArray() throws SerealException { + if (currentContext.type != CONTEXT_ARRAY) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (currentContext.expectedCount == -1) { + if (currentContext.count > 127) { + int lenght = varintLength(currentContext.count); + ensureAvailable(lenght); + System.arraycopy(bytes, currentContext.position + 1, + bytes, currentContext.position + lenght, + size - currentContext.position - 1); + size += lenght - 1; + } + encodeVarint(currentContext.count, bytes, currentContext.position); + } + currentContext = currentContext.outer; + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as a {@code SHORT_BINARY_*} or {@code BINARY} tag. + */ + public void startObject(byte[] className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendBinaryInternal(className, 0, className.length); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as an {@code UTF8} tag. + */ + public void startObject(CharSequence className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendCharBuffer(CharBuffer.wrap(className)); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as an {@code UTF8} tag. + */ + public void startObject(char[] className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendCharBuffer(CharBuffer.wrap(className)); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as a {@code COPY} tag. + */ + public void startObject(int classnameOffset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(classnameOffset); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECTV} tag. + */ + public void startObjectV(int classnameOffset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECTV); + appendVarint(classnameOffset); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Complete writing an object value. + */ + public void endObject() throws SerealException { + if (currentContext.type != CONTEXT_OBJECT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + currentContext = currentContext.outer; + } + + private void markNotCompressed() { + compressedSize = 0; + bytes[4] &= (byte) 0xf; + } + + private void compressSnappy() throws SerealException { + int maxSize = Snappy.maxCompressedLength(size - headerSize); + int sizeLength = encoding == SerealHeader.SRL_ENCODING_SNAPPY ? varintLength(maxSize) : 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + maxSize) > compressedBytes.length) { + compressedBytes = new byte[headerSize + sizeLength + maxSize]; + } + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + int compressed; + try { + compressed = + Snappy.compress( + bytes, headerSize, size - headerSize, compressedBytes, headerSize + sizeLength); + } catch (IOException e) { + throw new SerealException(e); + } + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + if (encoding == 2) { + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + } + + // from miniz.c + private int zlibMaxSize(int sourceLen) { + return Math.max( + 128 + (sourceLen * 110) / 100, 128 + sourceLen + ((sourceLen / (31 * 1024)) + 1) * 5); + } + + private void compressZlib() { + deflater.reset(); + + int sourceSize = size - headerSize; + int maxSize = zlibMaxSize(sourceSize); + int sizeLength = varintLength(sourceSize); + int sizeLength2 = varintLength(maxSize); + int pos = 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + sizeLength2 + maxSize) > compressedBytes.length) { + compressedBytes = new byte[headerSize + sizeLength + sizeLength2 + maxSize]; + } + + System.arraycopy(bytes, 0, compressedBytes, 0, headerSize); + pos += headerSize; + pos = encodeVarint(sourceSize, compressedBytes, pos); + + // varint-encoded 0, filling all space + int encodedSizePos = pos; + for (int max = pos + sizeLength2 - 1; pos < max; ) { + compressedBytes[pos++] = (byte) 128; + } + compressedBytes[pos++] = 0; + + deflater.setInput(bytes, headerSize, sourceSize); + deflater.finish(); + + int compressed = deflater.deflate(compressedBytes, pos, compressedBytes.length - pos); + compressedSize = headerSize + sizeLength + sizeLength2 + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + int after = encodeVarint(compressed, compressedBytes, encodedSizePos); + if (after != headerSize + sizeLength + sizeLength2) { + compressedBytes[after - 1] |= (byte) 0x80; + } + } + + private void compressZstd() throws SerealException { + long maxSize = Zstd.compressBound(size - headerSize); + int sizeLength = varintLength(maxSize); + + if (headerSize + sizeLength + maxSize > Integer.MAX_VALUE) { + throw new SerealException( + "Compressed data size exceeds integer MAX_VALUE: " + (headerSize + maxSize)); + } + if (headerSize + sizeLength + maxSize > compressedBytes.length) { + compressedBytes = new byte[(int) (headerSize + sizeLength + maxSize)]; + } + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + long compressed = + Zstd.compressUsingDict( + compressedBytes, + headerSize + sizeLength, + bytes, + headerSize, + size - headerSize, + new byte[0], + zstdCompressionLevel); + if (Zstd.isError(compressed)) { + throw new SerealException(Zstd.getErrorName(compressed)); + } + compressedSize = headerSize + sizeLength + (int) compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + + private void appendShortBinary(byte[] latin1, int offset, int length) throws SerealException { + // length of string + appendByte((byte) (length | SerealHeader.SRL_HDR_SHORT_BINARY)); + + // save it + appendBytes(latin1, offset, length); + } + + private void appendLongBinary(byte[] latin1, int offset, int lenght) { + // length of string + appendByte(SerealHeader.SRL_HDR_BINARY); + appendBytesWithLength(latin1, offset, lenght); + } + + private void appendBytesWithLength(byte[] in, int offset, int length) { + appendVarint(length); + appendBytes(in, offset, length); + } + + private void appendVarint(long n) { + ensureAvailable(MAX_VARINT_LENGTH); + size = encodeVarint(n, bytes, size); + } + + private void appendZigZag(long n) { + ensureAvailable(MAX_VARINT_LENGTH + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_ZIGZAG); + size = encodeVarint((n << 1) ^ (n >> 63), bytes, size); // note the signed right shift + } + + private void setTrackBit(int offset) { + bytes[offset + headerOffset] |= (byte) 0x80; + } + + private static boolean isRefTag(byte tag) { + // the first branch is the common case, the other two branchs are unlikely + if (tag == SerealHeader.SRL_HDR_REFN || + tag == SerealHeader.SRL_HDR_REFP) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + return true; + } + + return false; + } + + private static int varintLength(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + n >>>= 7; + length++; + } + + return length + 1; + } + + private static int encodeVarint(long n, byte[] buffer, int pos) { + while (Long.compareUnsigned(n, 127) > 0) { + buffer[pos++] = (byte) ((n & 127) | 128); + n >>>= 7; + } + buffer[pos++] = (byte) n; + + return pos; + } + + private void ensureAvailable(int required) { + long total = required + size; + + if (total > bytes.length) { + bytes = Arrays.copyOf(bytes, (int) (total * 3 / 2)); + } + } + + private void appendBytes(byte[] data, int offset, int lenght) { + ensureAvailable(lenght); + appendBytesUnsafe(data, offset, lenght); + } + + private void appendBytesUnsafe(byte[] data) { + System.arraycopy(data, 0, bytes, size, data.length); + size += data.length; + } + + private void appendBytesUnsafe(byte[] data, int offset, int length) { + System.arraycopy(data, offset, bytes, size, length); + size += length; + } + + private void appendByte(byte data) { + ensureAvailable(1); + appendByteUnsafe(data); + } + + private void appendByteUnsafe(byte data) { + bytes[size] = data; + size++; + } + + // used for testing + void poisonBuffer() { + for (int i = size; i < bytes.length; ++i) { + bytes[i] = (byte) (Math.random() * 256); + } + } + + /** + * Close the encoder to recycle the resources, it must be called by the client at the end of the + * use, otherwise, NullPointerException will be thrown when you reuse the ZLIB encoder. + *

+ * Returns the native memory used by deflater explicitly before the garbage collector. + */ + public void close() { + if (deflater != null) { + deflater.end(); + deflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/TypeMapper.java b/src/main/java/com/booking/sereal/TypeMapper.java new file mode 100644 index 000000000..c07469c58 --- /dev/null +++ b/src/main/java/com/booking/sereal/TypeMapper.java @@ -0,0 +1,14 @@ +package com.booking.sereal; + +import java.util.List; +import java.util.Map; + +public interface TypeMapper { + public boolean useObjectArray(); + + public List makeArray(int size); + + public Map makeMap(int size); + + public Object makeObject(String className, Object data); +} diff --git a/src/main/java/com/booking/sereal/Utils.java b/src/main/java/com/booking/sereal/Utils.java new file mode 100644 index 000000000..6df684835 --- /dev/null +++ b/src/main/java/com/booking/sereal/Utils.java @@ -0,0 +1,189 @@ +package com.booking.sereal; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.lang.reflect.Array; +import java.util.*; +import java.util.regex.Pattern; + +public class Utils { + private static final char[] hexDigits = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', + }; + // so we can not overflow our stack with ciruclar refs + private static Set already_output = new HashSet(); + + public static String join(String[] parts, String seperator) { + return join(Arrays.asList(parts), seperator); + } + + public static String join(List things, String separator) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < things.size(); i++) { + sb.append(things.get(i).toString()); + if (i < things.size() - 1) { + sb.append(separator); + } + } + return sb.toString(); + } + + public static String dump(Object o) { + + String d = dump(o, 0); + already_output.clear(); + return d; + } + + @SuppressWarnings({"rawtypes", "deprecation"}) + private static String dump(Object o, int indent) { + + if (o != null && already_output.contains(System.identityHashCode(o))) { + return "@" + System.identityHashCode(o); + } else if (o != null) { + already_output.add(System.identityHashCode(o)); + } + + String ind = ""; + for (int i = 0; i < indent; i++) { + ind += "\t"; + } + if (o == null) { + return "(NULL)"; + } else if (o instanceof Map) { + StringBuilder sb = new StringBuilder(ind + "Map@" + System.identityHashCode(o) + " {\n"); + Map map = (Map) o; + Object[] array = map.keySet().toArray(); + try { + Arrays.sort(array); + } catch (ClassCastException e) { + // In case where the array can't be sorted (if the keys don't + // implement Comparable) + } + for (Object key : array) { + sb.append(ind + dump(key, indent + 1)); + sb.append(" => "); + sb.append(dump(map.get(key), indent + 1)); + sb.append("\n"); + } + sb.append(ind).append("}"); + return sb.toString(); + } else if (o instanceof List) { + List l = (List) o; + String type = l.isEmpty() ? "" : l.get(0).getClass().getName(); + StringBuilder sb = new StringBuilder(ind + "List<" + type + ">[\n"); + for (Object li : l) { + sb.append(dump(li, indent + 1)); + sb.append("\n"); + } + sb.append(ind).append("]"); + return sb.toString(); + } else if (o.getClass().isArray()) { + StringBuilder sb = new StringBuilder(ind + "Array@" + System.identityHashCode(o) + " [\n"); + int length = Array.getLength(o); + for (int i = 0; i < length; i++) { + sb.append(dump(Array.get(o, i), indent + 1)); + sb.append("\n"); + } + + sb.append(ind).append("]"); + return sb.toString(); + } else if (o instanceof Pattern) { + Pattern pat = (Pattern) o; + return "/" + + pat.pattern() + + "/" + + (((pat.flags() & Pattern.CANON_EQ) > 0) ? "c" : "") + + (((pat.flags() & Pattern.CASE_INSENSITIVE) > 0) ? "i" : "") + + (((pat.flags() & Pattern.COMMENTS) > 0) ? "x" : "") + + (((pat.flags() & Pattern.DOTALL) > 0) ? "s" : "") + + (((pat.flags() & Pattern.LITERAL) > 0) ? "q" : "") + + (((pat.flags() & Pattern.MULTILINE) > 0) ? "m" : "") + + (((pat.flags() & Pattern.UNICODE_CASE) > 0) ? "l" : "") + + (((pat.flags() & Pattern.UNIX_LINES) > 0) ? "u" : "") + + "@" + + System.identityHashCode(o); + + } else if (o instanceof PerlAlias) { + return ind + "Alias: " + dump(((PerlAlias) o).getValue(), indent); + } else if (o instanceof PerlReference) { + return ind + + "Perlref@" + + System.identityHashCode(o) + + ": " + + dump(((PerlReference) o).getValue(), indent); + } else if (o instanceof WeakReference) { + return ind + + "(weakref@" + + System.identityHashCode(o) + + ") " + + dump(((WeakReference) o).get(), 0); + } else if (o instanceof PerlObject) { + PerlObject po = (PerlObject) o; + return ind + + "Object(" + + (po.isHash() ? "hash" : (po.isArray() ? "array" : "reference")) + + "):" + + po.getName() + + "= " + + dump(po.getData(), 0); + } else { + // ad system ident hascode (which is normally memory location) so you + // can see if things point to the same + return ind + + o.getClass().getSimpleName() + + "@" + + System.identityHashCode(o) + + ": " + + o.toString(); + } + } + + public static String hexStringFromByteArray(byte[] in) { + return hexStringFromByteArray(in, -1); + } + + public static String hexStringFromByteArray(byte[] in, int group) { + StringBuilder out = + new StringBuilder(2 + in.length * 2 + (group != -1 ? (in.length / group + 1) : 0)); + out.append("0x"); + + int count = 0; + for (byte b : in) { + out.append(hexDigits[(b >> 4) & 0xf]); + out.append(hexDigits[b & 0xf]); + + if (group > 0 && (++count == group)) { + out.append(' '); + count = 0; + } + } + + return out.toString(); + } + + public static Object decodeFile(Decoder decoder, File f) throws SerealException, IOException { + if (!f.exists()) { + throw new FileNotFoundException("No such file: " + f.getCanonicalPath()); + } + + // read everything + int size = (int) f.length(); // yeah yeah truncate + byte[] buf = new byte[size]; + FileInputStream fi = new FileInputStream(f); + try { + fi.read(buf); + } finally { + fi.close(); + } + + decoder.setData(buf); + Object structure = decoder.decode(); + + return structure; + } +} diff --git a/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java b/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java new file mode 100644 index 000000000..31f4fde77 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class BytearrayCopyMap { + public static final long NOT_FOUND = -1; + + private static final byte[] NO_KEY = new byte[0]; + byte[][] keys; + private long[] values; + private int size, maxLoad, modulus; + + public BytearrayCopyMap() { + init(32); + } + + private void init(int capacity) { + keys = new byte[capacity][]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(byte[] key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(byte[] key, long value) { + int slot = findSlot(key); + + if (keys[slot] == NO_KEY) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + byte[][] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(byte[] key) { + int slot = Arrays.hashCode(key) & modulus; + while (keys[slot] != NO_KEY && !Arrays.equals(keys[slot], key)) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/IdentityMap.java b/src/main/java/com/booking/sereal/impl/IdentityMap.java new file mode 100644 index 000000000..38578f184 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/IdentityMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class IdentityMap { + public static final long NOT_FOUND = -1; + + private static final Object NO_KEY = new Object(); + private Object[] keys; + private long[] values; + private int size, maxLoad, modulus; + + public IdentityMap() { + init(32); + } + + private void init(int capacity) { + keys = new Object[capacity]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(Object key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(Object key, long value) { + int slot = findSlot(key); + + if (keys[slot] != key) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + Object[] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(Object key) { + int slot = System.identityHashCode(key) & modulus; + while (keys[slot] != NO_KEY && keys[slot] != key) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/RefpMap.java b/src/main/java/com/booking/sereal/impl/RefpMap.java new file mode 100644 index 000000000..9a8310cb0 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/RefpMap.java @@ -0,0 +1,78 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class RefpMap { + public static final Object NOT_FOUND = new Object(); + + private static final long NO_KEY = -1; + private long[] keys; + private Object[] values; + private int size, maxLoad, modulus; + + public RefpMap() { + init(32); + } + + private void init(int capacity) { + keys = new long[capacity]; + values = new Object[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final Object get(long key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(long key, Object value) { + int slot = findSlot(key); + + if (keys[slot] != key) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + long[] oldKeys = keys; + Object[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(long key) { + int slot = hash6432Shift(key) & modulus; + while (keys[slot] != NO_KEY && keys[slot] != key) slot = (slot + 1) & modulus; + return slot; + } + + // http://burtleburtle.net/bob/hash/integer.html + // not tested for the actual distribution of offsets in Sereal + private int hash6432Shift(long key) { + key = (~key) + (key << 18); // key = (key << 18) - key - 1; + key = key ^ (key >>> 31); + key = key * 21; // key = (key + (key << 2)) + (key << 4); + key = key ^ (key >>> 11); + key = key + (key << 6); + key = key ^ (key >>> 22); + return (int) key; + } +} diff --git a/src/main/java/com/booking/sereal/impl/StringCopyMap.java b/src/main/java/com/booking/sereal/impl/StringCopyMap.java new file mode 100644 index 000000000..263f63027 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/StringCopyMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class StringCopyMap { + public static final long NOT_FOUND = -1; + + private static final String NO_KEY = null; + String[] keys; + private long[] values; + private int size, maxLoad, modulus; + + public StringCopyMap() { + init(32); + } + + private void init(int capacity) { + keys = new String[capacity]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(String key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(String key, long value) { + int slot = findSlot(key); + + if (keys[slot] == NO_KEY) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + String[] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(String key) { + int slot = key == null ? 0 : key.hashCode() & modulus; + while (keys[slot] != NO_KEY && !keys[slot].equals(key)) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/package-info.java b/src/main/java/com/booking/sereal/impl/package-info.java new file mode 100644 index 000000000..d8d7eb8ab --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/package-info.java @@ -0,0 +1,4 @@ +/** + * Internal helper classes used by Sereal implementation. + */ +package com.booking.sereal.impl; \ No newline at end of file diff --git a/src/main/java/com/booking/sereal/package-info.java b/src/main/java/com/booking/sereal/package-info.java new file mode 100644 index 000000000..887c3d7ee --- /dev/null +++ b/src/main/java/com/booking/sereal/package-info.java @@ -0,0 +1,14 @@ +/** + * Encoder/Decoder interface for Sereal + *

+ * See {@code com.booking.sereal.jackson.SerealObjectMapper} for a higher-level, easier to use interface. + *

+ * The main entry points are {@link com.booking.sereal.Encoder} and {@link com.booking.sereal.Decoder}, + * which allow simple encoding and decoding of nested hash/array data structures, and mirror + * the functionality available in the Perl implementation. + *

+ * {@link com.booking.sereal.TokenEncoder} and {@link com.booking.sereal.TokenDecoder} offer a low-level + * interface that can be used to build custom encoders/decoders, but is generally inconvenient and + * error-prone when used directly. + */ +package com.booking.sereal; \ No newline at end of file diff --git a/src/main/java/org/perlonjava/runtime/mro/DFS.java b/src/main/java/org/perlonjava/runtime/mro/DFS.java index 2579e1cc2..00d5c241a 100644 --- a/src/main/java/org/perlonjava/runtime/mro/DFS.java +++ b/src/main/java/org/perlonjava/runtime/mro/DFS.java @@ -96,7 +96,7 @@ private static void populateIsaMapWithCycleDetection(String className, // Get current @ISA array - FORCE fresh read RuntimeArray isaArray = InheritanceResolver.getIsaArrayForClass(className); List parents = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : InheritanceResolver.visibleArrayElements(isaArray)) { String parentName = entity.toString(); // FIXED: Skip empty or null parent names if (parentName != null && !parentName.isEmpty()) { diff --git a/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java b/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java index d5a153852..602be25d5 100644 --- a/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java +++ b/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java @@ -157,7 +157,7 @@ private static boolean hasIsaChanged(String className, MroRuntimeState state) { // Build current ISA list List currentIsa = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : visibleArrayElements(isaArray)) { String parentName = entity.toString(); if (parentName != null && !parentName.isEmpty()) { currentIsa.add(parentName); @@ -342,7 +342,7 @@ private static void populateIsaMapHelper(String className, // Retrieve @ISA array for the given class RuntimeArray isaArray = getIsaArrayForClass(className); List parents = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : visibleArrayElements(isaArray)) { String parentName = entity.toString(); // Handle undef elements as "main" for Perl compatibility if (parentName == null || parentName.equals("")) { @@ -369,6 +369,26 @@ private static void populateIsaMapHelper(String className, currentPath.remove(className); } + /** + * Return the Perl-visible contents of an array. + * + *

Most package {@code @ISA} arrays are ordinary arrays, but Perl permits + * them to be tied. Method lookup must honor the tie's {@code FETCHSIZE} and + * {@code FETCH} methods; reading {@link RuntimeArray#elements} directly sees + * only the empty backing list used by a tied array.

+ */ + static List visibleArrayElements(RuntimeArray array) { + if (array.type != RuntimeArray.TIED_ARRAY) { + return array.elements; + } + List visible = new ArrayList<>(); + int size = TieArray.tiedFetchSize(array).getInt(); + for (int i = 0; i < size; i++) { + visible.add(array.get(i)); + } + return visible; + } + /** * Searches for a method in the class hierarchy starting from a specific index. * Uses method caching to improve performance for both found and not-found methods. diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java b/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java new file mode 100644 index 000000000..2f33626c5 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java @@ -0,0 +1,247 @@ +package org.perlonjava.runtime.perlmodule; + +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Cache::FastMmap's XS primitive API implemented with a JVM shared map. + * + *

The Perl layer retains expiry, serialization, callbacks, and atomic-operation + * semantics. This backend replaces only the native mmap page store, which is not + * loadable on the JVM.

+ */ +public class CacheFastMmap extends PerlModuleBase { + private static final Map> SHARED = new ConcurrentHashMap<>(); + private static volatile long timeOverride; + + private static final class Entry { + RuntimeScalar value; + long expireOn; + int flags; + Long modseq; + long lastAccess; + } + + private static final class State { + final Map params = new ConcurrentHashMap<>(); + ConcurrentHashMap entries = new ConcurrentHashMap<>(); + boolean locked; + long reads; + long hits; + } + + public CacheFastMmap() { super("Cache::FastMmap", false); } + + public static void initialize() { + CacheFastMmap module = new CacheFastMmap(); + try { + for (String name : new String[]{ + "fc_new", "fc_set_param", "fc_init", "fc_hash", "fc_lock", "fc_unlock", + "fc_is_locked", "fc_read", "fc_write", "fc_delete", "fc_tombstone", + "fc_get_keys", "fc_get_page_details", "fc_reset_page_details", "fc_expunge", + "fc_set_time_override", "fc_close"}) { + module.registerMethod(name, null); + } + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList fc_new(RuntimeArray args, int ctx) { + return new RuntimeScalar(new State()).getList(); + } + + public static RuntimeList fc_set_param(RuntimeArray args, int ctx) { + state(args.get(0)).params.put(args.get(1).toString(), new RuntimeScalar(args.get(2))); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_init(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + RuntimeScalar share = state.params.get("share_file"); + if (share != null && share.defined().getBoolean()) { + String file = share.toString(); + boolean init = state.params.getOrDefault("init_file", new RuntimeScalar(0)).getBoolean(); + state.entries = SHARED.computeIfAbsent(file, ignored -> new ConcurrentHashMap<>()); + if (init) state.entries.clear(); + try { + Path path = Path.of(file); + if (path.getParent() != null) Files.createDirectories(path.getParent()); + Files.write(path, new byte[0], StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (Exception e) { + throw new IllegalStateException("Unable to initialize cache share file " + file, e); + } + } + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_hash(RuntimeArray args, int ctx) { + String key = args.get(1).toString(); + int hash = key.hashCode() & 0x7fffffff; + int pages = state(args.get(0)).params.getOrDefault("num_pages", new RuntimeScalar(89)).getInt(); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(hash % Math.max(1, pages))); + out.add(new RuntimeScalar(hash)); + return out; + } + + public static RuntimeList fc_lock(RuntimeArray args, int ctx) { + state(args.get(0)).locked = true; + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_unlock(RuntimeArray args, int ctx) { + state(args.get(0)).locked = false; + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_is_locked(RuntimeArray args, int ctx) { + return new RuntimeScalar(state(args.get(0)).locked).getList(); + } + + public static RuntimeList fc_read(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + state.reads++; + Entry entry = state.entries.get(args.get(2).toString()); + if (entry != null && expired(entry)) { + state.entries.remove(args.get(2).toString(), entry); + entry = null; + } + RuntimeList out = new RuntimeList(); + if (entry == null) { + out.add(new RuntimeScalar()); + out.add(new RuntimeScalar(0)); + out.add(new RuntimeScalar(0)); + out.add(new RuntimeScalar()); + out.add(new RuntimeScalar()); + } else { + state.hits++; + entry.lastAccess = now(); + out.add(new RuntimeScalar(entry.value)); + out.add(new RuntimeScalar(entry.flags)); + out.add(new RuntimeScalar(1)); + out.add(entry.expireOn < 0 ? new RuntimeScalar() : new RuntimeScalar(entry.expireOn)); + out.add(entry.modseq == null ? new RuntimeScalar() : new RuntimeScalar(entry.modseq)); + } + return out; + } + + public static RuntimeList fc_write(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + String key = args.get(2).toString(); + long expireOn = args.get(4).getLong(); + int flags = args.get(5).getInt(); + Long modseq = args.size() > 6 && args.get(6).defined().getBoolean() + ? args.get(6).getLong() : null; + Entry old = state.entries.get(key); + if (old != null && old.modseq != null && (modseq == null || modseq < old.modseq)) { + return new RuntimeScalar(-1).getList(); + } + Entry entry = new Entry(); + entry.value = new RuntimeScalar(args.get(3)); + entry.expireOn = expireOn; + entry.flags = flags; + entry.modseq = modseq; + entry.lastAccess = now(); + state.entries.put(key, entry); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_delete(RuntimeArray args, int ctx) { + Entry removed = state(args.get(0)).entries.remove(args.get(2).toString()); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(removed != null)); + out.add(new RuntimeScalar(removed == null ? 0 : removed.flags)); + return out; + } + + public static RuntimeList fc_tombstone(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + String key = args.get(2).toString(); + long modseq = args.get(4).getLong(); + Entry old = state.entries.get(key); + if (old != null && old.modseq != null && old.modseq > modseq) return new RuntimeScalar(0).getList(); + Entry tombstone = new Entry(); + tombstone.value = new RuntimeScalar(); + tombstone.expireOn = args.get(3).getLong(); + tombstone.modseq = modseq; + tombstone.lastAccess = now(); + state.entries.put(key, tombstone); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_get_keys(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + int mode = args.size() > 1 ? args.get(1).getInt() : 0; + RuntimeList out = new RuntimeList(); + state.entries.forEach((key, entry) -> { + if (expired(entry)) return; + if (mode == 0) { + out.add(new RuntimeScalar(key)); + } else { + RuntimeHash detail = new RuntimeHash(); + detail.put("key", new RuntimeScalar(key)); + detail.put("last_access", new RuntimeScalar(entry.lastAccess)); + detail.put("expire_on", entry.expireOn < 0 ? new RuntimeScalar() : new RuntimeScalar(entry.expireOn)); + detail.put("flags", new RuntimeScalar(entry.flags)); + if (mode >= 2) detail.put("value", new RuntimeScalar(entry.value)); + out.add(detail.createAnonymousReference()); + } + }); + return out; + } + + public static RuntimeList fc_get_page_details(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(state.reads)); + out.add(new RuntimeScalar(state.hits)); + return out; + } + + public static RuntimeList fc_reset_page_details(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + state.reads = state.hits = 0; + return new RuntimeList(); + } + + public static RuntimeList fc_expunge(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + int mode = args.get(1).getInt(); + RuntimeList removed = new RuntimeList(); + state.entries.entrySet().removeIf(item -> { + Entry entry = item.getValue(); + boolean remove = mode == 1 || expired(entry); + if (remove && args.get(2).getBoolean()) { + RuntimeHash detail = new RuntimeHash(); + detail.put("key", new RuntimeScalar(item.getKey())); + detail.put("value", new RuntimeScalar(entry.value)); + detail.put("expire_on", new RuntimeScalar(entry.expireOn)); + detail.put("flags", new RuntimeScalar(entry.flags)); + removed.add(detail.createAnonymousReference()); + } + return remove; + }); + return removed; + } + + public static RuntimeList fc_set_time_override(RuntimeArray args, int ctx) { + timeOverride = args.isEmpty() ? 0 : args.get(0).getLong(); + return new RuntimeList(); + } + + public static RuntimeList fc_close(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static State state(RuntimeScalar scalar) { + if (scalar.type == RuntimeScalarType.JAVAOBJECT && scalar.value instanceof State state) return state; + throw new IllegalArgumentException("Invalid Cache::FastMmap native cache handle"); + } + + private static long now() { return timeOverride != 0 ? timeOverride : System.currentTimeMillis() / 1000L; } + private static boolean expired(Entry entry) { return entry.expireOn > 0 && entry.expireOn <= now(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java new file mode 100644 index 000000000..ab0f6c7d4 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java @@ -0,0 +1,85 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.openssl.PEMParser; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.runtimetypes.*; + +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; + +/** Crypt::OpenSSL::Verify implemented with the existing Bouncy Castle/JCA stack. */ +public class CryptOpenSSLVerify extends PerlModuleBase { + private static final String CLASS_NAME = "Crypt::OpenSSL::Verify"; + private static final String STATE_KEY = "_verify_certificates"; + + public CryptOpenSSLVerify() { super(CLASS_NAME, false); } + + public static void initialize() { + CryptOpenSSLVerify module = new CryptOpenSSLVerify(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("verify", null); + module.registerMethod("register_verify_cb", "noop", null); + module.registerMethod("ctx_error_code", "zero", null); + module.registerMethod("__X509_cleanup", "noop", null); + module.registerMethod("DESTROY", "noop", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + try { + List certificates = new ArrayList<>(); + if (args.size() > 1 && args.get(1).defined().getBoolean()) { + String pem = Files.readString(Path.of(args.get(1).toString())); + try (PEMParser parser = new PEMParser(new StringReader(pem))) { + Object value; + while ((value = parser.readObject()) != null) { + if (value instanceof X509CertificateHolder holder) { + certificates.add(new JcaX509CertificateConverter().setProvider("BC") + .getCertificate(holder)); + } + } + } + } + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(certificates)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } catch (Exception e) { + return CryptOpenSSLX509.fail("Unable to load CA certificates: " + e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private static List certificates(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.value instanceof List) return (List) state.value; + throw new IllegalArgumentException("Invalid Crypt::OpenSSL::Verify object"); + } + + public static RuntimeList verify(RuntimeArray args, int ctx) { + X509Certificate target = CryptOpenSSLX509.certificate(args.get(1)); + for (X509Certificate issuer : certificates(args.get(0))) { + if (!target.getIssuerX500Principal().equals(issuer.getSubjectX500Principal())) continue; + try { + target.verify(issuer.getPublicKey()); + return new RuntimeScalar(1).getList(); + } catch (Exception ignored) { + // Try the remaining candidate issuers. + } + } + return CryptOpenSSLX509.fail("certificate verify failed"); + } + + public static RuntimeList noop(RuntimeArray args, int ctx) { return new RuntimeList(); } + public static RuntimeList zero(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java new file mode 100644 index 000000000..940a76550 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java @@ -0,0 +1,161 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.Extensions; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.openssl.PEMParser; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +import java.io.ByteArrayInputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.Locale; + +/** Crypt::OpenSSL::X509 compatibility backed by Bouncy Castle and JCA. */ +public class CryptOpenSSLX509 extends PerlModuleBase { + static final String CLASS_NAME = "Crypt::OpenSSL::X509"; + static final String STATE_KEY = "_x509_certificate"; + private static final DateTimeFormatter OPENSSL_TIME = + DateTimeFormatter.ofPattern("MMM dd HH:mm:ss yyyy 'GMT'", Locale.US).withZone(ZoneOffset.UTC); + + static { + if (java.security.Security.getProvider("BC") == null) { + java.security.Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); + } + } + + public CryptOpenSSLX509() { super(CLASS_NAME, false); } + + public static void initialize() { + CryptOpenSSLX509 module = new CryptOpenSSLX509(); + try { + module.registerMethod("new_from_string", null); + module.registerMethod("new_from_file", null); + module.registerMethod("subject", null); + module.registerMethod("issuer", null); + module.registerMethod("notBefore", null); + module.registerMethod("notAfter", null); + module.registerMethod("checkend", null); + module.registerMethod("extensions_by_oid", null); + module.registerMethod("as_string", null); + module.registerMethod("DESTROY", "destroy", null); + module.registerMethod("__X509_cleanup", "destroy", null); + CryptOpenSSLX509Extension.initialize(); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_from_string(RuntimeArray args, int ctx) { + if (args.size() < 2) return fail("Usage: Crypt::OpenSSL::X509->new_from_string(string)"); + try { + return object(parse(args.get(1).toString())).getList(); + } catch (Exception e) { + return fail("Unable to parse X509 certificate: " + e.getMessage()); + } + } + + public static RuntimeList new_from_file(RuntimeArray args, int ctx) { + if (args.size() < 2) return fail("Usage: Crypt::OpenSSL::X509->new_from_file(file)"); + try { + return object(parse(java.nio.file.Files.readString(java.nio.file.Path.of(args.get(1).toString())))).getList(); + } catch (Exception e) { + return fail("Unable to parse X509 certificate file: " + e.getMessage()); + } + } + + static X509Certificate parse(String text) throws Exception { + try (PEMParser parser = new PEMParser(new StringReader(text))) { + Object value = parser.readObject(); + if (value instanceof X509CertificateHolder holder) { + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + } + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + return (X509Certificate) factory.generateCertificate( + new ByteArrayInputStream(text.getBytes(StandardCharsets.ISO_8859_1))); + } + + static RuntimeScalar object(X509Certificate certificate) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(certificate)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref; + } + + static X509Certificate certificate(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.type == RuntimeScalarType.JAVAOBJECT + && state.value instanceof X509Certificate certificate) return certificate; + throw new IllegalArgumentException("Invalid Crypt::OpenSSL::X509 object"); + } + + public static RuntimeList subject(RuntimeArray args, int ctx) { + return new RuntimeScalar(certificate(args.get(0)).getSubjectX500Principal().getName()).getList(); + } + + public static RuntimeList issuer(RuntimeArray args, int ctx) { + return new RuntimeScalar(certificate(args.get(0)).getIssuerX500Principal().getName()).getList(); + } + + public static RuntimeList notBefore(RuntimeArray args, int ctx) { + return new RuntimeScalar(OPENSSL_TIME.format(certificate(args.get(0)).getNotBefore().toInstant())).getList(); + } + + public static RuntimeList notAfter(RuntimeArray args, int ctx) { + return new RuntimeScalar(OPENSSL_TIME.format(certificate(args.get(0)).getNotAfter().toInstant())).getList(); + } + + public static RuntimeList checkend(RuntimeArray args, int ctx) { + long seconds = args.size() > 1 ? args.get(1).getLong() : 0; + Date when = new Date(System.currentTimeMillis() + seconds * 1000L); + X509Certificate cert = certificate(args.get(0)); + boolean invalid = when.before(cert.getNotBefore()) || when.after(cert.getNotAfter()); + return new RuntimeScalar(invalid).getList(); + } + + public static RuntimeList extensions_by_oid(RuntimeArray args, int ctx) { + try { + X509CertificateHolder holder = new X509CertificateHolder(certificate(args.get(0)).getEncoded()); + RuntimeHash hash = new RuntimeHash(); + Extensions extensions = holder.getExtensions(); + if (extensions != null) { + for (ASN1ObjectIdentifier oid : extensions.getExtensionOIDs()) { + hash.put(oid.getId(), CryptOpenSSLX509Extension.object(extensions.getExtension(oid))); + } + } + return hash.createAnonymousReference().getList(); + } catch (Exception e) { + return fail("Unable to read X509 extensions: " + e.getMessage()); + } + } + + public static RuntimeList as_string(RuntimeArray args, int ctx) { + try { + String base64 = java.util.Base64.getMimeEncoder(64, new byte[]{'\n'}) + .encodeToString(certificate(args.get(0)).getEncoded()); + return new RuntimeScalar("-----BEGIN CERTIFICATE-----\n" + base64 + + "\n-----END CERTIFICATE-----\n").getList(); + } catch (Exception e) { + return fail("Unable to encode X509 certificate: " + e.getMessage()); + } + } + + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + static RuntimeList fail(String message) { + return WarnDie.die(new RuntimeScalar(message), new RuntimeScalar("\n")).getList(); + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java new file mode 100644 index 000000000..6752749f2 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java @@ -0,0 +1,63 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.runtimetypes.*; + +/** Methods on Crypt::OpenSSL::X509::Extension objects. */ +public class CryptOpenSSLX509Extension extends PerlModuleBase { + private static final String CLASS_NAME = "Crypt::OpenSSL::X509::Extension"; + private static final String STATE_KEY = "_x509_extension"; + + public CryptOpenSSLX509Extension() { super(CLASS_NAME, false); } + + public static void initialize() throws NoSuchMethodException { + CryptOpenSSLX509Extension module = new CryptOpenSSLX509Extension(); + module.registerMethod("value", null); + module.registerMethod("to_string", null); + module.registerMethod("critical", null); + module.registerMethod("extendedKeyUsage", null); + module.registerMethod("DESTROY", "destroy", null); + } + + static RuntimeScalar object(Extension extension) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(extension)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref; + } + + private static Extension extension(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.value instanceof Extension extension) return extension; + throw new IllegalArgumentException("Invalid X509 extension object"); + } + + public static RuntimeList value(RuntimeArray args, int ctx) { + return new RuntimeScalar("#" + java.util.HexFormat.of().withUpperCase() + .formatHex(extension(args.get(0)).getExtnValue().getOctets())).getList(); + } + + public static RuntimeList to_string(RuntimeArray args, int ctx) { + Extension extension = extension(args.get(0)); + if (Extension.extendedKeyUsage.equals(extension.getExtnId())) { + KeyPurposeId[] usages = ExtendedKeyUsage.getInstance(extension.getParsedValue()).getUsages(); + StringBuilder out = new StringBuilder(); + for (KeyPurposeId usage : usages) { + if (!out.isEmpty()) out.append(", "); + out.append(usage.getId()); + } + return new RuntimeScalar(out.toString()).getList(); + } + return value(args, ctx); + } + + public static RuntimeList extendedKeyUsage(RuntimeArray args, int ctx) { return to_string(args, ctx); } + public static RuntimeList critical(RuntimeArray args, int ctx) { + return new RuntimeScalar(extension(args.get(0)).isCritical()).getList(); + } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java b/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java index 49c18c7f2..a535aee82 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java @@ -28,6 +28,7 @@ public static void initialize() { // and the modules fall through to their pure-Perl implementations. dynaLoader.registerMethod("dl_findfile", "dl_empty", null); dynaLoader.registerMethod("dl_load_file", "dl_empty", null); + dynaLoader.registerMethod("dl_load_flags", "dl_load_flags", null); dynaLoader.registerMethod("dl_find_symbol", "dl_empty", null); dynaLoader.registerMethod("dl_find_symbol_anywhere", "dl_empty", null); dynaLoader.registerMethod("dl_install_xsub", "dl_empty", null); @@ -74,6 +75,10 @@ public static RuntimeList dl_empty(RuntimeArray args, int ctx) { return new RuntimeList(); } + public static RuntimeList dl_load_flags(RuntimeArray args, int ctx) { + return new RuntimeScalar(0).getList(); + } + public static RuntimeList dl_error(RuntimeArray args, int ctx) { return new RuntimeScalar("DynaLoader is not supported in PerlOnJava").getList(); } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java b/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java index ddb6a5ca1..3daeb0d74 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java @@ -102,11 +102,27 @@ public static RuntimeList request(RuntimeArray args, int ctx) throws Exception { responseMap.put("headers", responseHeaders.createReference()); return responseMap.createReference().getList(); - } catch (IOException | InterruptedException e) { - throw new RuntimeException("HTTP request failed", e); + } catch (IOException e) { + return transportFailure(url, e).getList(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return transportFailure(url, e).getList(); } } + private static RuntimeScalar transportFailure(String url, Exception error) { + RuntimeHash response = new RuntimeHash(); + response.put("success", new RuntimeScalar(false)); + response.put("status", new RuntimeScalar(599)); + response.put("reason", new RuntimeScalar("Internal Exception")); + String message = error.getMessage(); + response.put("content", new RuntimeScalar( + message == null || message.isEmpty() ? error.getClass().getSimpleName() : message)); + response.put("url", new RuntimeScalar(url)); + response.put("headers", new RuntimeHash().createReference()); + return response.createReference(); + } + private static String getStatusReason(int statusCode) { return switch (statusCode) { case 100 -> "Continue"; @@ -329,4 +345,4 @@ public static RuntimeList mirror(RuntimeArray args, int ctx) throws Exception { return responseMap.createReference().getList(); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java b/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java index 8fbb1eda8..41b2199c2 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java @@ -72,6 +72,7 @@ public static void initialize() { internals.registerMethod("jperl_cv_start_location", "jperlCvStartLocation", "$"); internals.registerMethod("jperl_cv_deparse_info", "jperlCvDeparseInfo", "$"); internals.registerMethod("jperl_cv_is_constant", "jperlCvIsConstant", "$"); + internals.registerMethod("jperl_mark_pseudo_constant", "jperlMarkPseudoConstant", "$$"); internals.registerMethod("jperl_end_av_ref", "jperlEndAvRef", ""); internals.registerMethod("jperl_b_object_2svref", "jperlBObject2svref", "$"); internals.registerMethod("jperl_set_closed_over", "jperlSetClosedOver", null); @@ -982,4 +983,21 @@ public static RuntimeList jperlCvIsConstant(RuntimeArray args, int ctx) { boolean isConst = code.constantValue != null || code.isConstantCv; return new RuntimeScalar(isConst ? 1 : 0).getList(); } + + /** Preserve constant.pm's scalar-reference proxy in the stash hash view. */ + public static RuntimeList jperlMarkPseudoConstant(RuntimeArray args, int ctx) { + if (args.size() >= 2) { + RuntimeScalar proxy = args.get(1); + if (proxy.type == RuntimeScalarType.REFERENCE + && proxy.value instanceof RuntimeScalar target) { + proxy = target; + } + if (proxy.type == RuntimeScalarType.READONLY_SCALAR + && proxy.value instanceof RuntimeScalar target) { + proxy = target; + } + GlobalVariable.setGlobalPseudoConstant(args.get(0).toString(), proxy); + } + return new RuntimeList(); + } } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java new file mode 100644 index 000000000..ff8063ce7 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java @@ -0,0 +1,101 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Decoder; +import com.booking.sereal.DecoderOptions; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.charset.StandardCharsets; + +/** Sereal::Decoder XS compatibility layer backed by Sereal's official Java codec. */ +public class SerealDecoder extends PerlModuleBase { + private static final String CLASS_NAME = "Sereal::Decoder"; + private static final String STATE_KEY = "_sereal_decoder"; + + public SerealDecoder() { super(CLASS_NAME, false); } + + public static void initialize() { + SerealDecoder module = new SerealDecoder(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("decode", "decode_sereal", null); + module.registerMethod("decode_sereal", null); + module.registerMethod("sereal_decode_with_object", null); + module.registerMethod("looks_like_sereal", null); + module.registerMethod("scalar_looks_like_sereal", null); + module.registerMethod("bytes_consumed", null); + module.registerMethod("flags", null); + module.registerMethod("DESTROY", "destroy", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + private static Decoder newDecoder() { + return new Decoder(new DecoderOptions().perlReferences(true).preserveUndef(true)); + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(newDecoder())); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } + + public static RuntimeList decode_sereal(RuntimeArray args, int ctx) { + int blobIndex = isObject(args) ? 1 : 0; + Decoder decoder = isObject(args) ? state(args.get(0)) : newDecoder(); + return decode(decoder, args, blobIndex); + } + + public static RuntimeList sereal_decode_with_object(RuntimeArray args, int ctx) { + return decode(state(args.get(0)), args, 1); + } + + private static RuntimeList decode(Decoder decoder, RuntimeArray args, int blobIndex) { + if (args.size() <= blobIndex) return WarnDie.die( + new RuntimeScalar("Usage: decode_sereal(blob)"), new RuntimeScalar("\n")).getList(); + try { + byte[] bytes = args.get(blobIndex).toString().getBytes(StandardCharsets.ISO_8859_1); + decoder.setData(bytes); + return SerealRuntimeConverter.fromJava(decoder.decode()).getList(); + } catch (Exception e) { + return WarnDie.die(new RuntimeScalar("Sereal decode failed: " + e.getMessage()), + new RuntimeScalar("\n")).getList(); + } + } + + public static RuntimeList looks_like_sereal(RuntimeArray args, int ctx) { + int index = isObject(args) ? 1 : 0; + return likely(args, index); + } + + public static RuntimeList scalar_looks_like_sereal(RuntimeArray args, int ctx) { return likely(args, 0); } + + private static RuntimeList likely(RuntimeArray args, int index) { + if (args.size() <= index) return new RuntimeScalar(0).getList(); + String value = args.get(index).toString(); + boolean match = value.length() >= 4 && value.charAt(0) == '=' + && (value.substring(1, 4).equals("srl") || value.charAt(1) == (char) 0xF3); + return new RuntimeScalar(match).getList(); + } + + public static RuntimeList bytes_consumed(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList flags(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static boolean isObject(RuntimeArray args) { + return !args.isEmpty() && RuntimeScalarType.blessedId(args.get(0)) != 0; + } + + private static Decoder state(RuntimeScalar self) { + RuntimeScalar stored = self.hashDeref().get(STATE_KEY); + if (stored != null && stored.type == RuntimeScalarType.JAVAOBJECT && stored.value instanceof Decoder decoder) { + return decoder; + } + WarnDie.die(new RuntimeScalar("Invalid Sereal::Decoder object"), new RuntimeScalar("\n")); + return null; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java new file mode 100644 index 000000000..0b0179ea3 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java @@ -0,0 +1,89 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Encoder; +import com.booking.sereal.EncoderOptions; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +/** Sereal::Encoder XS compatibility layer backed by Sereal's official Java codec. */ +public class SerealEncoder extends PerlModuleBase { + private static final String CLASS_NAME = "Sereal::Encoder"; + private static final String STATE_KEY = "_sereal_encoder"; + + public SerealEncoder() { super(CLASS_NAME, false); } + + public static void initialize() { + SerealEncoder module = new SerealEncoder(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("encode", "encode_sereal", null); + module.registerMethod("encode_sereal", null); + module.registerMethod("sereal_encode_with_object", null); + module.registerMethod("encode_sereal_with_header_data", null); + module.registerMethod("flags", null); + module.registerMethod("DESTROY", "destroy", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(new Encoder(options()))); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } + + public static RuntimeList encode_sereal(RuntimeArray args, int ctx) { + int valueIndex = isObject(args) ? 1 : 0; + Encoder encoder = isObject(args) ? state(args.get(0)) : new Encoder(options()); + return encode(encoder, args, valueIndex, -1); + } + + public static RuntimeList sereal_encode_with_object(RuntimeArray args, int ctx) { + return encode(state(args.get(0)), args, 1, -1); + } + + public static RuntimeList encode_sereal_with_header_data(RuntimeArray args, int ctx) { + return encode(new Encoder(options()), args, 0, 1); + } + + private static RuntimeList encode(Encoder encoder, RuntimeArray args, int valueIndex, int headerIndex) { + if (args.size() <= valueIndex) return WarnDie.die( + new RuntimeScalar("Usage: encode_sereal(value)"), new RuntimeScalar("\n")).getList(); + try { + Object value = SerealRuntimeConverter.toJava(args.get(valueIndex)); + if (headerIndex >= 0 && args.size() > headerIndex) { + encoder.write(value, SerealRuntimeConverter.toJava(args.get(headerIndex))); + } else { + encoder.write(value); + } + return SerealRuntimeConverter.byteScalar(encoder.getData()).getList(); + } catch (Exception e) { + return WarnDie.die(new RuntimeScalar("Sereal encode failed: " + e.getMessage()), + new RuntimeScalar("\n")).getList(); + } + } + + public static RuntimeList flags(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static boolean isObject(RuntimeArray args) { + return !args.isEmpty() && RuntimeScalarType.blessedId(args.get(0)) != 0; + } + + private static EncoderOptions options() { + return new EncoderOptions().protocolVersion(4).perlReferences(true); + } + + private static Encoder state(RuntimeScalar self) { + RuntimeScalar stored = self.hashDeref().get(STATE_KEY); + if (stored != null && stored.type == RuntimeScalarType.JAVAOBJECT && stored.value instanceof Encoder encoder) { + return encoder; + } + WarnDie.die(new RuntimeScalar("Invalid Sereal::Encoder object"), new RuntimeScalar("\n")); + return null; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java new file mode 100644 index 000000000..9765a6e97 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java @@ -0,0 +1,146 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Latin1String; +import com.booking.sereal.PerlObject; +import com.booking.sereal.PerlReference; +import com.booking.sereal.PerlUndef; +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.perlonjava.runtime.runtimetypes.RuntimeScalarType.*; + +/** Converts PerlOnJava runtime values to and from the official Sereal Java model. */ +final class SerealRuntimeConverter { + private SerealRuntimeConverter() {} + + static Object toJava(RuntimeScalar scalar) { + return toJavaScalar(scalar, new IdentityHashMap<>()); + } + + private static Object toJavaScalar(RuntimeScalar scalar, IdentityHashMap seen) { + while (scalar.type == READONLY_SCALAR && scalar.value instanceof RuntimeScalar inner) { + scalar = inner; + } + return switch (scalar.type) { + case UNDEF -> PerlUndef.CANONICAL; + case INTEGER -> scalar.getLong(); + case DOUBLE -> scalar.getDouble(); + case BOOLEAN -> scalar.getBoolean(); + case BYTE_STRING -> new Latin1String(scalar.toString()); + case STRING, VSTRING, DUALVAR -> scalar.toString(); + case ARRAYREFERENCE -> referenceValue(scalar, scalar.arrayDeref(), seen); + case HASHREFERENCE -> referenceValue(scalar, scalar.hashDeref(), seen); + case REFERENCE -> referenceValue(scalar, scalar.scalarDeref(), seen); + default -> scalar.toString(); + }; + } + + private static Object referenceValue(RuntimeScalar scalar, RuntimeBase referent, + IdentityHashMap seen) { + Object converted = seen.get(referent); + if (converted == null) { + if (referent instanceof RuntimeArray array) { + List list = new ArrayList<>(array.size()); + PerlReference reference = new PerlReference(list); + seen.put(referent, reference); + for (RuntimeScalar value : array) list.add(toJavaScalar(value, seen)); + converted = reference; + } else if (referent instanceof RuntimeHash hash) { + Map map = new LinkedHashMap<>(); + PerlReference reference = new PerlReference(map); + seen.put(referent, reference); + for (Map.Entry entry : hash.elements.entrySet()) { + map.put(entry.getKey(), toJavaScalar(entry.getValue(), seen)); + } + converted = reference; + } else if (referent instanceof RuntimeScalar target) { + PerlReference placeholder = new PerlReference(null); + seen.put(referent, placeholder); + placeholder.setValue(toJavaScalar(target, seen)); + converted = placeholder; + } else { + converted = new PerlReference(referent.toString()); + seen.put(referent, converted); + } + } + int blessId = RuntimeScalarType.blessedId(scalar); + if (blessId != 0) { + String className = NameNormalizer.getBlessStr(blessId); + converted = new PerlObject(className == null ? "__ANON__" : className, converted); + } + return converted; + } + + static RuntimeScalar fromJava(Object value) { + return fromJava(value, new IdentityHashMap<>()); + } + + private static RuntimeScalar fromJava(Object value, IdentityHashMap seen) { + if (value == null || value instanceof PerlUndef) return new RuntimeScalar(); + RuntimeScalar prior = seen.get(value); + if (prior != null) return prior; + if (value instanceof PerlObject object) { + RuntimeScalar result = fromJava(object.getData(), seen); + if (RuntimeScalarType.isReference(result) && result.value instanceof RuntimeBase base) { + base.blessId = NameNormalizer.getBlessId(object.getName()); + } + return result; + } + if (value instanceof PerlReference reference) { + Object referent = reference.getValue(); + RuntimeScalar target = fromJava(referent, seen); + // Java's PerlReference wraps both aggregate references and scalar + // references. Aggregate conversion already returns a Perl reference; + // only scalar referents need one more level of indirection. + RuntimeScalar result = !(referent instanceof PerlReference) + && (target.type == ARRAYREFERENCE || target.type == HASHREFERENCE) + ? target : target.createReference(); + seen.put(value, result); + return result; + } + if (value instanceof Map map) { + RuntimeHash hash = new RuntimeHash(); + RuntimeScalar result = hash.createAnonymousReference(); + seen.put(value, result); + for (Map.Entry entry : map.entrySet()) { + hash.put(entry.getKey().toString(), fromJava(entry.getValue(), seen)); + } + return result; + } + if (value instanceof List list) { + RuntimeArray array = new RuntimeArray(); + RuntimeScalar result = array.createAnonymousReference(); + seen.put(value, result); + for (Object item : list) RuntimeArray.push(array, fromJava(item, seen)); + return result; + } + if (value instanceof Object[] arrayValue) { + RuntimeArray array = new RuntimeArray(); + RuntimeScalar result = array.createAnonymousReference(); + seen.put(value, result); + for (Object item : arrayValue) RuntimeArray.push(array, fromJava(item, seen)); + return result; + } + if (value instanceof byte[] bytes) return byteScalar(bytes); + if (value instanceof Latin1String latin1) return byteScalar(latin1.getBytes()); + if (value instanceof Boolean bool) return new RuntimeScalar(bool); + if (value instanceof Byte || value instanceof Short || value instanceof Integer) { + return new RuntimeScalar(((Number) value).intValue()); + } + if (value instanceof Long number) return new RuntimeScalar(number); + if (value instanceof Number number) return new RuntimeScalar(number.doubleValue()); + return new RuntimeScalar(value.toString()); + } + + static RuntimeScalar byteScalar(byte[] bytes) { + RuntimeScalar scalar = new RuntimeScalar(new String(bytes, StandardCharsets.ISO_8859_1)); + scalar.type = BYTE_STRING; + return scalar; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java b/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java index 88e5008ca..7e8b34c2a 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java @@ -35,6 +35,7 @@ public static void initialize() { module.registerMethod("time", ""); module.registerMethod("sleep", null); module.registerMethod("alarm", null); + module.registerMethod("ualarm", null); } catch (NoSuchMethodException e) { System.err.println("Warning: Missing Time::HiRes method: " + e.getMessage()); } @@ -88,4 +89,11 @@ public static RuntimeList alarm(RuntimeArray args, int ctx) { // Implement alarm functionality if needed return new RuntimeScalar(0).getList(); } + + public static RuntimeList ualarm(RuntimeArray args, int ctx) { + // Match the existing alarm compatibility behavior. Registering the + // function is important even where JVM signal delivery is unavailable: + // callers commonly use ualarm(0) to cancel an optional timeout. + return new RuntimeScalar(0).getList(); + } } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java b/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java index 065ea3a8d..479202460 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java @@ -14,6 +14,8 @@ import javax.xml.transform.*; import javax.xml.transform.dom.*; import javax.xml.transform.stream.*; +import javax.xml.validation.Schema; +import javax.xml.validation.Validator; import javax.xml.xpath.*; import org.w3c.dom.*; import org.xml.sax.*; @@ -40,6 +42,7 @@ public class XMLLibXML extends PerlModuleBase { private static final String OPTS_KEY = "_parser_opts"; private static final String XPC_KEY = "_xpc_state"; private static final String READER_KEY = "_reader_state"; + private static final String RELAXNG_KEY = "_relaxng_schema"; /** Pseudo-namespace for functions registered without namespace ("{}name"). */ private static final String NONS_NS = "http://perlonjava.org/xpc-nons"; @@ -422,6 +425,15 @@ public static void initialize() { module.registerMethodInPackage(xpePkg, "new", "xpeNew"); module.registerMethodInPackage(xpePkg, "expression", "xpeExpression"); + // XML::LibXML::RelaxNG. Jing supplies the standard JAXP RELAX NG + // provider; the compiled schema is retained in the blessed Perl + // object just as DOM nodes retain their backing Java object. + String rngPkg = "XML::LibXML::RelaxNG"; + module.registerMethodInPackage(rngPkg, "parse_buffer", "relaxNGParseBuffer"); + module.registerMethodInPackage(rngPkg, "parse_location", "relaxNGParseLocation"); + module.registerMethodInPackage(rngPkg, "parse_document", "relaxNGParseDocument"); + module.registerMethodInPackage(rngPkg, "validate", "relaxNGValidate"); + setupISA(); } catch (NoSuchMethodException e) { @@ -506,6 +518,70 @@ public static Node getNode(RuntimeScalar self) { throw new RuntimeException("Not a valid XML::LibXML node (missing " + NODE_KEY + " key)"); } + private static RuntimeScalar wrapRelaxNG(Schema schema) { + RuntimeHash hash = new RuntimeHash(); + hash.put(RELAXNG_KEY, new RuntimeScalar(schema)); + RuntimeScalar ref = hash.createReferenceWithTrackedElements(); + return ReferenceOperators.bless(ref, new RuntimeScalar("XML::LibXML::RelaxNG")); + } + + private static Schema getRelaxNG(RuntimeScalar self) { + RuntimeScalar state = self.hashDerefRaw().get(RELAXNG_KEY); + if (state != null && state.type == RuntimeScalarType.JAVAOBJECT + && state.value instanceof Schema schema) return schema; + throw new PerlDieException(new RuntimeScalar("Invalid XML::LibXML::RelaxNG object\n")); + } + + private static Schema compileRelaxNG(Source source) throws SAXException { + // Jing does not publish a META-INF JAXP provider entry, so instantiate + // its standard XML-syntax factory explicitly. + return new com.thaiopensource.relaxng.jaxp.XMLSyntaxSchemaFactory().newSchema(source); + } + + private static RuntimeList relaxNGError(Exception error) { + Throwable cause = error; + while (cause.getCause() != null && cause.getCause() != cause) cause = cause.getCause(); + String message = cause.getMessage(); + if (message == null || message.isEmpty()) message = cause.toString(); + throw new PerlDieException(new RuntimeScalar(message + "\n")); + } + + public static RuntimeList relaxNGParseBuffer(RuntimeArray args, int ctx) { + try { + return wrapRelaxNG(compileRelaxNG(new StreamSource(new StringReader(args.get(1).toString())))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGParseLocation(RuntimeArray args, int ctx) { + try { + return wrapRelaxNG(compileRelaxNG(new StreamSource(args.get(1).toString()))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGParseDocument(RuntimeArray args, int ctx) { + try { + String schemaText = serializeNode(getNode(args.get(1)), false, true); + return wrapRelaxNG(compileRelaxNG(new StreamSource(new StringReader(schemaText)))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGValidate(RuntimeArray args, int ctx) { + try { + Validator validator = getRelaxNG(args.get(0)).newValidator(); + String documentText = serializeNode(getNode(args.get(1)), false, true); + validator.validate(new StreamSource(new StringReader(documentText))); + return scalarZero.getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + /** * Update the Java Node stored in a Perl XML::LibXML node object. * Needed after Document.renameNode() which may return a new Node instance. diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java index f73cc97a3..cb6863bd4 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java @@ -1217,6 +1217,18 @@ public static boolean hasGlobalPseudoConstant(String key) { return resolvedKey != key && globalPseudoConstants().containsKey(resolvedKey); } + public static RuntimeScalar getGlobalPseudoConstant(String key) { + if (key == null) { + return null; + } + RuntimeScalar scalar = globalPseudoConstants().get(key); + if (scalar != null) { + return scalar; + } + String resolvedKey = resolveAliasedFqn(key); + return resolvedKey != key ? globalPseudoConstants().get(resolvedKey) : null; + } + /** * Retrieves a global array by its key, initializing it if necessary. * diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java index 804b52653..8ae8e0c1d 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java @@ -9,6 +9,7 @@ Handling pipes (e.g., |- or -| modes). */ import org.perlonjava.runtime.io.*; +import org.perlonjava.runtime.HintHashRegistry; import org.perlonjava.runtime.operators.IOOperator; import org.perlonjava.runtime.operators.WarnDie; import org.perlonjava.runtime.perlmodule.Warnings; @@ -720,7 +721,7 @@ public static RuntimeIO open(String fileName, String mode) { // Use SeekableJarHandle to support seek operations (needed by Module::Metadata) fh.ioHandle = new SeekableJarHandle(is); addHandle(fh.ioHandle); - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } return fh; @@ -761,7 +762,7 @@ public static RuntimeIO open(String fileName, String mode) { } // Apply any I/O layers - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } @@ -772,7 +773,19 @@ public static RuntimeIO open(String fileName, String mode) { return fh; } - private boolean applyOpenLayers(String ioLayers) { + private boolean applyOpenLayers(String ioLayers, String mode) { + if (ioLayers == null || ioLayers.isEmpty()) { + Map hints = HintHashRegistry.getCurrentCallSiteHintHash(); + String key = mode != null && mode.contains(">") ? "open>" : "open<"; + String lexicalLayer = hints == null ? null : hints.get(key); + // open.pm defaults are lexical compiler hints. An unrelated + // package's use open must not leak through the process-global + // ${^OPEN} compatibility scalar into this call site. + // ":" asks binmode() for the platform default (:crlf on Windows, + // :raw elsewhere) without consulting the legacy process-global + // ${^OPEN} value. + ioLayers = lexicalLayer == null || lexicalLayer.isEmpty() ? ":" : lexicalLayer; + } RuntimeScalar status = binmode(ioLayers); if (status.getBoolean()) { return true; @@ -879,7 +892,7 @@ public static RuntimeIO open(RuntimeScalar scalarRef, String mode) { // PerlIO::scalar is not a real OS file descriptor, so a plain scalar // open should not inherit the platform text layer such as Windows :crlf. if (!ioLayers.isEmpty()) { - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } } @@ -972,7 +985,7 @@ public static RuntimeIO openPipe(RuntimeList runtimeList) { // Apply any I/O layers (excluding the already-processed :noshell) if (!ioLayers.isEmpty()) { - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java index ebd702499..a3104b319 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java @@ -27,7 +27,7 @@ public RuntimeStashEntry(String globName, boolean isDefined) { // assigned reference even though globDeref() still provides the // symbol's complete typeglob. type = REFERENCE; - value = GlobalVariable.getGlobalVariable(globName); + value = GlobalVariable.getGlobalPseudoConstant(globName); } // System.out.println("Stash Entry create: " + globName + " " + isDefined); } @@ -43,6 +43,23 @@ public RuntimeGlob createDetachedCopy() { return this; } + /** + * A constant.pm proxy is a scalar-reference value stored directly in the + * stash hash, not a typeglob value. Taking a reference to that hash value + * must therefore preserve the scalar shape instead of forcing the + * GLOBREFERENCE representation used by ordinary stash entries. + */ + @Override + public RuntimeScalar createReference() { + if (GlobalVariable.hasGlobalPseudoConstant(this.globName)) { + RuntimeScalar reference = new RuntimeScalar(); + reference.type = REFERENCE; + reference.value = this; + return reference; + } + return super.createReference(); + } + /** * Override globDeref for stash entries to return a plain RuntimeGlob. * This ensures that *{$stash{name}} = \$value goes through RuntimeGlob.set() @@ -128,6 +145,15 @@ public RuntimeScalar set(RuntimeScalar value) { code.constantValue = value.scalarDeref().getList(); GlobalVariable.defineGlobalCodeRef(this.globName).set( new RuntimeScalar(code)); + // Perl 5 exposes a constant.pm entry through the stash hash + // as the assigned scalar-reference proxy, even though the + // same symbol also has a callable CODE slot. Keep the proxy + // metadata alongside our concrete constant CV so stash + // introspection can distinguish constants from ordinary + // subroutines. + GlobalVariable.setGlobalPseudoConstant(this.globName, targetScalar); + this.type = REFERENCE; + this.value = targetScalar; notifyCodeSlotChanged(); } else { // Ordinary scalar references alias the stash scalar slot, diff --git a/src/main/perl/lib/DynaLoader.pm b/src/main/perl/lib/DynaLoader.pm index 2fb1f6f10..0ab83947d 100644 --- a/src/main/perl/lib/DynaLoader.pm +++ b/src/main/perl/lib/DynaLoader.pm @@ -41,6 +41,9 @@ BEGIN { unless (defined &dl_load_file) { *dl_load_file = sub { return }; } + unless (defined &dl_load_flags) { + *dl_load_flags = sub { return 0 }; + } unless (defined &dl_find_symbol) { *dl_find_symbol = sub { return }; } diff --git a/src/main/perl/lib/HTTP/Tiny.pm b/src/main/perl/lib/HTTP/Tiny.pm index e105ae7dc..a01404ea8 100644 --- a/src/main/perl/lib/HTTP/Tiny.pm +++ b/src/main/perl/lib/HTTP/Tiny.pm @@ -227,15 +227,13 @@ sub _set_proxies { for my $sub_name ( qw/get head put post patch delete/ ) { my $req_method = uc $sub_name; - for my $accessor ( @attributes ) { - my $sym_ref = qualify_to_ref($sub_name, __PACKAGE__); - *{$sym_ref} = sub { - my ($self, $url, $args) = @_; - @_ == 2 || (@_ == 3 && ref $args eq 'HASH') - or _croak("Usage: \$http->$sub_name(URL, [HASHREF])\n"); - return $self->request($req_method, $url, $args || {}); - }; - } + my $sym_ref = qualify_to_ref($sub_name, __PACKAGE__); + *{$sym_ref} = sub { + my ($self, $url, $args) = @_; + @_ == 2 || (@_ == 3 && ref $args eq 'HASH') + or _croak("Usage: \$http->$sub_name(URL, [HASHREF])\n"); + return $self->request($req_method, $url, $args || {}); + }; } #pod =method post_form diff --git a/src/main/perl/lib/constant.pm b/src/main/perl/lib/constant.pm index f8d58e399..b5aec754f 100644 --- a/src/main/perl/lib/constant.pm +++ b/src/main/perl/lib/constant.pm @@ -165,6 +165,10 @@ sub import { Internals::SvREADONLY($scalar, 1); if (!exists $symtab->{$name}) { $symtab->{$name} = \$scalar; + # PerlOnJava stores a concrete CV for callable constant + # lookup; retain the scalar-reference proxy separately so + # stash introspection still sees Perl's PCS representation. + Internals::jperl_mark_pseudo_constant($full_name, \$scalar); ++$flush_mro->{$pkg}; } else { diff --git a/src/main/perl/lib/open.pm b/src/main/perl/lib/open.pm index 4a9d62c4d..8c21108ef 100644 --- a/src/main/perl/lib/open.pm +++ b/src/main/perl/lib/open.pm @@ -60,7 +60,6 @@ sub import { } } - ${^OPEN} = join( "\0", defined $in ? $in : '', defined $out ? $out : '' ); $^H{'open<'} = $in if defined $in; $^H{'open>'} = $out if defined $out; diff --git a/src/test/resources/unit/constant_stash_proxy.t b/src/test/resources/unit/constant_stash_proxy.t new file mode 100644 index 000000000..e8c4a0200 --- /dev/null +++ b/src/test/resources/unit/constant_stash_proxy.t @@ -0,0 +1,27 @@ +use strict; +use warnings; +use Scalar::Util qw(reftype); +use Test::More tests => 6; + +{ + package ConstantStashProxy; + use constant { + NUMBER => 3, + ARRAY => [qw(a b c)], + }; +} + +for my $name (qw(NUMBER ARRAY)) { + no strict 'refs'; + my $stash = \%ConstantStashProxy::; + my $entry = \$stash->{$name}; + + ok(reftype(${$entry}) eq ($name eq 'ARRAY' ? 'REF' : 'SCALAR'), + "$name is exposed as a proxy constant in the stash"); + ok(defined &{"ConstantStashProxy::$name"}, + "$name remains available through the CODE slot"); +} + +is(ConstantStashProxy::NUMBER(), 3, 'scalar proxy constant remains callable'); +is_deeply(ConstantStashProxy::ARRAY(), [qw(a b c)], + 'reference proxy constant remains callable'); diff --git a/src/test/resources/unit/dynaloader_load_flags.t b/src/test/resources/unit/dynaloader_load_flags.t new file mode 100644 index 000000000..540ee5b8e --- /dev/null +++ b/src/test/resources/unit/dynaloader_load_flags.t @@ -0,0 +1,10 @@ +use strict; +use warnings FATAL => 'all'; +use Test::More; + +require DynaLoader; + +ok(DynaLoader->can('dl_load_flags'), 'DynaLoader exposes dl_load_flags'); +is(DynaLoader->dl_load_flags, 0, 'default dynamic loader flags are zero'); + +done_testing; diff --git a/src/test/resources/unit/http_tiny_method_installation.t b/src/test/resources/unit/http_tiny_method_installation.t new file mode 100644 index 000000000..ea8024387 --- /dev/null +++ b/src/test/resources/unit/http_tiny_method_installation.t @@ -0,0 +1,19 @@ +use strict; +use warnings FATAL => 'all'; +use Test::More; + +require HTTP::Tiny; + +{ + package Local::HTTP::Tiny; + our @ISA = ('HTTP::Tiny'); + sub request { return $_[1] } +} + +my $client = bless {}, 'Local::HTTP::Tiny'; +for my $method (qw(get head put post patch delete)) { + ok(HTTP::Tiny->can($method), "HTTP::Tiny installs $method once without warnings"); + is($client->$method('http://example.test/'), uc($method), "$method dispatches the correct HTTP verb"); +} + +done_testing; diff --git a/src/test/resources/unit/http_tiny_transport_failure.t b/src/test/resources/unit/http_tiny_transport_failure.t new file mode 100644 index 000000000..770a8c4bd --- /dev/null +++ b/src/test/resources/unit/http_tiny_transport_failure.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 4; +use HTTP::Tiny; + +my $response = HTTP::Tiny->new(timeout => 1)->get('http://127.0.0.1:1/'); +ok(!$response->{success}, 'transport failure is not successful'); +is($response->{status}, 599, 'transport failure uses HTTP::Tiny status 599'); +is($response->{reason}, 'Internal Exception', 'transport failure has standard reason'); +ok(exists $response->{content}, 'transport failure includes diagnostic content'); diff --git a/src/test/resources/unit/open_pragma_lexical_io.t b/src/test/resources/unit/open_pragma_lexical_io.t new file mode 100644 index 000000000..dc4dcabb7 --- /dev/null +++ b/src/test/resources/unit/open_pragma_lexical_io.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use Test::More tests => 2; +use Fcntl qw(O_WRONLY O_CREAT O_TRUNC); + +my $encoded_file = "/tmp/perlonjava-open-encoded-$$"; +my $raw_file = "/tmp/perlonjava-open-raw-$$"; + +{ + use open ':encoding(UTF-8)'; + sysopen(my $handle, $encoded_file, O_WRONLY | O_CREAT | O_TRUNC) or die $!; + print {$handle} chr(0xF3); + close $handle; +} + +{ + no open; + sysopen(my $handle, $raw_file, O_WRONLY | O_CREAT | O_TRUNC) or die $!; + print {$handle} chr(0xF3); + close $handle; +} + +sub file_hex { + my ($file) = @_; + open my $handle, '<:raw', $file or die $!; + local $/; + return unpack('H*', <$handle>); +} + +is(file_hex($encoded_file), 'c3b3', 'use open applies to its lexical scope'); +is(file_hex($raw_file), 'f3', 'open defaults do not leak into another scope'); +unlink $encoded_file, $raw_file; diff --git a/src/test/resources/unit/tied_isa_method_lookup.t b/src/test/resources/unit/tied_isa_method_lookup.t new file mode 100644 index 000000000..9dd4b7eb9 --- /dev/null +++ b/src/test/resources/unit/tied_isa_method_lookup.t @@ -0,0 +1,33 @@ +use strict; +use warnings; +use Test::More tests => 4; + +{ + package TiedISA::Parent; + sub inherited { 42 } +} + +{ + package TiedISA::Array; + + sub TIEARRAY { + my ($class, $values) = @_; + return bless [ @{$values} ], $class; + } + + sub FETCHSIZE { scalar @{$_[0]} } + sub FETCH { $_[0]->[$_[1]] } +} + +{ + package TiedISA::Child; + our @ISA = ('TiedISA::Parent'); + my @parents = @ISA; + tie @ISA, 'TiedISA::Array', \@parents; +} + +is_deeply([ @TiedISA::Child::ISA ], ['TiedISA::Parent'], 'tied @ISA exposes its parent'); +ok(TiedISA::Child->isa('TiedISA::Parent'), 'isa follows tied @ISA'); +my $method = TiedISA::Child->can('inherited'); +ok($method, 'can finds a method through tied @ISA'); +is($method->(), 42, 'inherited method remains callable'); diff --git a/src/test/resources/unit/time_hires_ualarm.t b/src/test/resources/unit/time_hires_ualarm.t new file mode 100644 index 000000000..5801277fd --- /dev/null +++ b/src/test/resources/unit/time_hires_ualarm.t @@ -0,0 +1,8 @@ +use strict; +use warnings; +use Test::More tests => 2; + +use Time::HiRes qw(ualarm); + +ok(defined &ualarm, 'ualarm is exported on request'); +cmp_ok(ualarm(0), '>=', 0, 'ualarm can cancel a timer'); diff --git a/src/test/resources/unit/xml_libxml_relaxng.t b/src/test/resources/unit/xml_libxml_relaxng.t new file mode 100644 index 000000000..76e13f1af --- /dev/null +++ b/src/test/resources/unit/xml_libxml_relaxng.t @@ -0,0 +1,22 @@ +use strict; +use warnings; +use Test::More; + +eval { require XML::LibXML; 1 } + or plan skip_all => 'XML::LibXML is not installed'; + +my $schema = XML::LibXML::RelaxNG->new(string => <<'RNG'); + + + +RNG + +ok($schema, 'compiled a RELAX NG schema'); +my $parser = XML::LibXML->new; +is($schema->validate($parser->parse_string('ok')), 0, + 'valid document returns zero'); +my $valid = eval { $schema->validate($parser->parse_string('')); 1 }; +ok(!$valid, 'invalid document throws'); +like($@, qr/(?:expect|allow|valid|element)/i, 'validation error is meaningful'); + +done_testing;