From d5511ff3d7f3b42de896410fe5d266aee5886d77 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 16:40:38 +0200 Subject: [PATCH 1/3] fix: expand jcpan compiler and Java module tooling Fix tied inheritance, constant stash proxies, lexical open layers, HTTP::Tiny transport behavior, Time::HiRes exports, and DynaLoader compatibility. Add Java-backed Sereal, Cache::FastMmap, OpenSSL X509, OpenSSL verification, and RelaxNG support for CPAN dependency closure. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- build.gradle | 3 + dev/design/jcpan-compiler-tooling-followup.md | 69 + gradle/libs.versions.toml | 6 + .../java/com/booking/sereal/ByteArray.java | 32 + .../java/com/booking/sereal/DeSereal.java | 26 + src/main/java/com/booking/sereal/Decoder.java | 914 ++++++++++++ .../com/booking/sereal/DecoderOptions.java | 271 ++++ .../com/booking/sereal/DefaultTypeMapper.java | 28 + src/main/java/com/booking/sereal/Encoder.java | 1014 +++++++++++++ .../com/booking/sereal/EncoderOptions.java | 203 +++ .../java/com/booking/sereal/Latin1String.java | 75 + .../java/com/booking/sereal/PerlAlias.java | 18 + .../java/com/booking/sereal/PerlObject.java | 40 + .../com/booking/sereal/PerlReference.java | 27 + .../java/com/booking/sereal/PerlUndef.java | 11 + .../com/booking/sereal/SerealException.java | 12 + .../java/com/booking/sereal/SerealHeader.java | 80 ++ .../java/com/booking/sereal/SerealToken.java | 138 ++ .../java/com/booking/sereal/TokenDecoder.java | 923 ++++++++++++ .../java/com/booking/sereal/TokenEncoder.java | 1261 +++++++++++++++++ .../java/com/booking/sereal/TypeMapper.java | 14 + src/main/java/com/booking/sereal/Utils.java | 189 +++ .../booking/sereal/impl/BytearrayCopyMap.java | 66 + .../com/booking/sereal/impl/IdentityMap.java | 66 + .../java/com/booking/sereal/impl/RefpMap.java | 78 + .../booking/sereal/impl/StringCopyMap.java | 66 + .../com/booking/sereal/impl/package-info.java | 4 + .../java/com/booking/sereal/package-info.java | 14 + .../java/org/perlonjava/runtime/mro/DFS.java | 2 +- .../runtime/mro/InheritanceResolver.java | 24 +- .../runtime/perlmodule/CacheFastMmap.java | 247 ++++ .../perlmodule/CryptOpenSSLVerify.java | 85 ++ .../runtime/perlmodule/CryptOpenSSLX509.java | 161 +++ .../perlmodule/CryptOpenSSLX509Extension.java | 63 + .../runtime/perlmodule/DynaLoader.java | 5 + .../runtime/perlmodule/HttpTiny.java | 22 +- .../runtime/perlmodule/Internals.java | 18 + .../runtime/perlmodule/SerealDecoder.java | 101 ++ .../runtime/perlmodule/SerealEncoder.java | 89 ++ .../perlmodule/SerealRuntimeConverter.java | 146 ++ .../runtime/perlmodule/TimeHiRes.java | 8 + .../runtime/perlmodule/XMLLibXML.java | 76 + .../runtime/runtimetypes/GlobalVariable.java | 12 + .../runtime/runtimetypes/RuntimeIO.java | 20 +- .../runtimetypes/RuntimeStashEntry.java | 28 +- src/main/perl/lib/DynaLoader.pm | 3 + src/main/perl/lib/HTTP/Tiny.pm | 16 +- src/main/perl/lib/constant.pm | 4 + src/main/perl/lib/open.pm | 1 - .../resources/unit/constant_stash_proxy.t | 27 + .../resources/unit/dynaloader_load_flags.t | 10 + .../unit/http_tiny_method_installation.t | 19 + .../unit/http_tiny_transport_failure.t | 10 + .../resources/unit/open_pragma_lexical_io.t | 32 + .../resources/unit/tied_isa_method_lookup.t | 33 + src/test/resources/unit/time_hires_ualarm.t | 8 + src/test/resources/unit/xml_libxml_relaxng.t | 22 + 57 files changed, 6918 insertions(+), 22 deletions(-) create mode 100644 dev/design/jcpan-compiler-tooling-followup.md create mode 100644 src/main/java/com/booking/sereal/ByteArray.java create mode 100644 src/main/java/com/booking/sereal/DeSereal.java create mode 100644 src/main/java/com/booking/sereal/Decoder.java create mode 100644 src/main/java/com/booking/sereal/DecoderOptions.java create mode 100644 src/main/java/com/booking/sereal/DefaultTypeMapper.java create mode 100644 src/main/java/com/booking/sereal/Encoder.java create mode 100644 src/main/java/com/booking/sereal/EncoderOptions.java create mode 100644 src/main/java/com/booking/sereal/Latin1String.java create mode 100644 src/main/java/com/booking/sereal/PerlAlias.java create mode 100644 src/main/java/com/booking/sereal/PerlObject.java create mode 100644 src/main/java/com/booking/sereal/PerlReference.java create mode 100644 src/main/java/com/booking/sereal/PerlUndef.java create mode 100644 src/main/java/com/booking/sereal/SerealException.java create mode 100644 src/main/java/com/booking/sereal/SerealHeader.java create mode 100644 src/main/java/com/booking/sereal/SerealToken.java create mode 100644 src/main/java/com/booking/sereal/TokenDecoder.java create mode 100644 src/main/java/com/booking/sereal/TokenEncoder.java create mode 100644 src/main/java/com/booking/sereal/TypeMapper.java create mode 100644 src/main/java/com/booking/sereal/Utils.java create mode 100644 src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java create mode 100644 src/main/java/com/booking/sereal/impl/IdentityMap.java create mode 100644 src/main/java/com/booking/sereal/impl/RefpMap.java create mode 100644 src/main/java/com/booking/sereal/impl/StringCopyMap.java create mode 100644 src/main/java/com/booking/sereal/impl/package-info.java create mode 100644 src/main/java/com/booking/sereal/package-info.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java create mode 100644 src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java create mode 100644 src/test/resources/unit/constant_stash_proxy.t create mode 100644 src/test/resources/unit/dynaloader_load_flags.t create mode 100644 src/test/resources/unit/http_tiny_method_installation.t create mode 100644 src/test/resources/unit/http_tiny_transport_failure.t create mode 100644 src/test/resources/unit/open_pragma_lexical_io.t create mode 100644 src/test/resources/unit/tied_isa_method_lookup.t create mode 100644 src/test/resources/unit/time_hires_ualarm.t create mode 100644 src/test/resources/unit/xml_libxml_relaxng.t diff --git a/build.gradle b/build.gradle index ca97d748f7..f065399dcb 100644 --- a/build.gradle +++ b/build.gradle @@ -214,6 +214,7 @@ dependencies { implementation libs.asm.util // ASM utilities implementation libs.icu4j // Unicode support implementation libs.jsoup // HTML5 parsing for HTML::Content::Extractor + implementation libs.jing // RELAX NG validation for XML::LibXML implementation libs.snakeyaml.engine // YAML processing implementation libs.tomlj // TOML processing implementation libs.commons.csv // CSV processing @@ -226,6 +227,8 @@ dependencies { implementation libs.sqlite.jdbc // SQLite JDBC driver implementation libs.bcprov // Bouncy Castle crypto (SHA-3, Keccak, etc.) implementation libs.bcpkix // Bouncy Castle PEM/PKCS parsing + implementation libs.snappy.java // Official Sereal Java codec compression + implementation libs.zstd.jni // Official Sereal Java codec compression implementation 'org.jruby.joni:joni:2.2.7' // Stack-safe recursive regex backend implementation 'io.netty:netty-codec-http:4.1.115.Final' // Netty HTTP codec for PSGI server diff --git a/dev/design/jcpan-compiler-tooling-followup.md b/dev/design/jcpan-compiler-tooling-followup.md new file mode 100644 index 0000000000..c687828a81 --- /dev/null +++ b/dev/design/jcpan-compiler-tooling-followup.md @@ -0,0 +1,69 @@ +# jcpan compiler and tooling follow-up + +## Goal + +Remove shared PerlOnJava compiler/runtime/tooling blockers encountered while testing Data::Checks, POE::Component::MessageQueue, Net::Server::POP3::Skeleton, Mail::BIMI, Imager::Album, Test::Mimic::Recorder, Fuse::Filesys::Virtual, and their dependencies. Distribution preferences are deliberately avoided. + +## Implementation + +### Compiler and runtime semantics + +- Tied `@ISA` values are fetched through the tie interface during method resolution. +- Constant subroutines preserve their scalar-reference stash proxy without losing the callable CODE slot. +- `use open` defaults are read from lexical `%^H` call-site snapshots instead of leaking through a process-global `${^OPEN}` value. + +### Bundled module tooling + +- `DynaLoader::dl_load_flags` is available to XS-style loaders. +- HTTP::Tiny installs each convenience verb once and returns its standard status-599 response for transport failures. +- Time::HiRes provides the requested `ualarm` export. +- XML::LibXML::RelaxNG uses Jing for schema compilation and validation. +- Sereal::Encoder and Sereal::Decoder use the official Booking.com Java codec with Snappy and Zstandard support. +- Cache::FastMmap has a JVM-backed compatibility implementation and preserves the distribution's Perl serialization, expiry, and callback layer. Its map is process-local; the share file is created for API compatibility. +- Crypt::OpenSSL::X509 and Crypt::OpenSSL::Verify use the existing Bouncy Castle/JCA stack rather than adding another crypto implementation. + +The imported Sereal Java sources are based on upstream commit `9ad81cf3023ccc456c2accd83bea2c2803a82e16`. + +## System-Perl exclusions + +The following are not treated as PerlOnJava regressions because their current distributions or tests fail under the available system Perl or require unavailable platform facilities: + +- Data::Checks: its fresh system-Perl dependency set lacks `builtin.pm`. +- Net::Server::POP3::Skeleton: the release metadata points `VERSION_FROM` at a nonexistent `lib/Tk/Carp.pm`. +- Imager::Album: its system-Perl dependency closure lacks Imager and relies on the legacy Gtk/display stack. +- Test::Mimic::Recorder: its own test fails on system Perl with a hard-coded reference-history assumption. +- Fuse::Filesys::Virtual: Fuse 0.16 refuses to configure on Darwin without OSXFUSE. +- POE::Component::MessageQueue: its remaining bind test cannot run in this sandbox; a minimal system-Perl bind fails with the same platform restriction. + +## Progress Tracking + +### Current Status: implementation and local verification complete + +### Completed Phases + +- [x] Phase 1: classify upstream/system failures (2026-08-15) + - Reproduced each exclusion with system Perl or the relevant platform configuration step. +- [x] Phase 2: compiler/runtime fixes (2026-08-15) + - Fixed tied inheritance, constant stash proxies, and lexical open-layer handling. +- [x] Phase 3: reusable Java module bridges (2026-08-15) + - Added Sereal, cache, OpenSSL/X509, and RelaxNG support using existing or upstream Java libraries. +- [x] Phase 4: regression coverage (2026-08-15) + - Added focused unit tests and validated new Perl semantics with system Perl before PerlOnJava. +- [x] Phase 5: final verification (2026-08-15) + - Full `make` passed. + - Mail::BIMI passed all 31 test programs and 82 assertions; network- and author-only tests skipped as expected. + +### Next Steps + +1. Open the pull request. +2. Verify CI. + +### Open Questions + +- A future Cache::FastMmap implementation could provide true cross-process mmap sharing; current requested tests only require same-process behavior. + +## References + +- [Sereal](https://github.com/Sereal/Sereal) +- [Jing and Trang](https://github.com/relaxng/jing-trang) +- Skills: `debug-perlonjava`, `port-cpan-module`, `port-native-module` diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b06e06cfc8..9c1999f26e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,10 +6,13 @@ commons-csv = "1.14.1" commonmark = "0.29.0" icu4j = "78.3" jsoup = "1.23.1" +jing = "20241231" junit-jupiter = "6.1.3" snakeyaml-engine = "3.1.1" +snappy-java = "1.1.10.8" sqlite-jdbc = "3.53.2.1" tomlj = "1.1.1" +zstd-jni = "1.5.7-8" [libraries] asm = { module = "org.ow2.asm:asm", version.ref = "asm" } @@ -25,12 +28,15 @@ commonmark-tables = { module = "org.commonmark:commonmark-ext-gfm-tables", versi commonmark-task-list = { module = "org.commonmark:commonmark-ext-task-list-items", version.ref = "commonmark" } icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +jing = { module = "org.relaxng:jing", version.ref = "jing" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit-jupiter" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit-jupiter" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" } snakeyaml-engine = { module = "org.snakeyaml:snakeyaml-engine", version.ref = "snakeyaml-engine" } +snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy-java" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } tomlj = { module = "org.tomlj:tomlj", version.ref = "tomlj" } +zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstd-jni" } [plugins] cyclonedx = "org.cyclonedx.bom:2.3.0" diff --git a/src/main/java/com/booking/sereal/ByteArray.java b/src/main/java/com/booking/sereal/ByteArray.java new file mode 100644 index 0000000000..cb033e30ec --- /dev/null +++ b/src/main/java/com/booking/sereal/ByteArray.java @@ -0,0 +1,32 @@ +package com.booking.sereal; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class ByteArray { + public byte[] array; + public int start; + public int length; + + public ByteArray(byte[] array) { + this(array, 0, array.length); + } + + public ByteArray(byte[] array, int length) { + this(array, 0, length); + } + + public ByteArray(byte[] array, int start, int length) { + this.array = array; + this.start = start; + this.length = length; + } + + public ByteArray(ByteBuffer buffer) { + this(buffer.array(), buffer.limit()); + } + + public void ensure(int size) { + if (size > array.length) array = Arrays.copyOf(array, size * 3 / 2); + } +} diff --git a/src/main/java/com/booking/sereal/DeSereal.java b/src/main/java/com/booking/sereal/DeSereal.java new file mode 100644 index 0000000000..ffe254fcc9 --- /dev/null +++ b/src/main/java/com/booking/sereal/DeSereal.java @@ -0,0 +1,26 @@ +package com.booking.sereal; + +import java.io.File; +import java.io.IOException; + +public class DeSereal { + /** + * @param args command arguments + * @throws IOException Indicated file cannot accessed. + * @throws SerealException Sereal data cannot be processed. + */ + public static void main(String[] args) throws IOException, SerealException { + if (args.length == 0) { + throw new UnsupportedOperationException("Usage: DeSereal test_data"); + } + + DecoderOptions decoder_options = + new DecoderOptions().perlReferences(true).perlAliases(true).preferLatin1(true); + + Decoder dec = new Decoder(decoder_options); + final File target = new File(args[0]).getCanonicalFile(); // to absorb ".." in paths + // dec.log.setLevel( Level.FINE ); + Object data = Utils.decodeFile(dec, target); + System.out.println(Utils.dump(data)); + } +} diff --git a/src/main/java/com/booking/sereal/Decoder.java b/src/main/java/com/booking/sereal/Decoder.java new file mode 100644 index 0000000000..edca41ed6c --- /dev/null +++ b/src/main/java/com/booking/sereal/Decoder.java @@ -0,0 +1,914 @@ +package com.booking.sereal; + +import com.booking.sereal.impl.RefpMap; +import com.github.luben.zstd.Zstd; + +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import org.xerial.snappy.Snappy; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +/** + * Sereal decoder with Perl-like interface. + *

+ * This class can be used to decoder Perl-like data-structures: (boxed) primitive types, strings, arrays + * and maps. + */ +public class Decoder implements SerealHeader { + + private static final DecoderOptions DEFAULT_OPTIONS = new DecoderOptions(); + private static final Charset charset_utf8 = Charset.forName("UTF-8"); + private static final Charset charset_latin1 = Charset.forName("ISO-8859-1"); + private final boolean perlRefs; + private final boolean perlAlias; + private final boolean preserveUndef; + private final boolean refuseSnappy; + private final boolean refuseZlib; + private final boolean refuseZstd; + private final boolean preferLatin1; + private final boolean forceJavaStringForByteArrayValues; + private final boolean refuseObjects; + private final boolean stripObjects; + private final TypeMapper typeMapper; + private final boolean useObjectArray; + private final int decodeBufferSize; + private final int maxSize; + + private final int maxRecursionDepth; + private final int maxNumMapEntries; + private final int maxNumArrayEntries; + private final int maxStringLength; + + private byte[] data; + private int position, end; + private ByteArray originalData; + // where we track items for REFP purposes + private RefpMap tracked = new RefpMap(); + private int protocolVersion = -1; + private int encoding = -1; + private int baseOffset = Integer.MAX_VALUE; + private long userHeaderPosition = -1; + private long userHeaderSize = -1; + private Inflater inflater; + + private int recursionDepth = 0; + + /** Create a new Decoder with default options. */ + public Decoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create a new Decoder with the specified options. + * + * @param options {@link DecoderOptions} to apply. + */ + public Decoder(DecoderOptions options) { + perlRefs = options.perlReferences(); + perlAlias = options.perlAliases(); + preserveUndef = options.preserveUndef(); + refuseSnappy = options.refuseSnappy(); + refuseZlib = options.refuseZlib(); + refuseZstd = options.refuseZstd(); + preferLatin1 = options.preferLatin1(); + forceJavaStringForByteArrayValues = options.forceJavaStringForByteArrayValues(); + refuseObjects = options.refuseObjects(); + stripObjects = options.stripObjects(); + typeMapper = options.typeMapper(); + useObjectArray = typeMapper.useObjectArray(); + decodeBufferSize = options.bufferSize(); + maxSize = options.maxBufferSize(); + + maxRecursionDepth = options.maxRecursionDepth(); + maxNumMapEntries = options.maxNumMapEntries(); + maxNumArrayEntries = options.maxNumArrayEntries(); + maxStringLength = options.maxStringLength(); + } + + private void checkHeader() throws SerealException { + + if ((end - position) < 4) { + throw new SerealException("Invalid Sereal header: too few bytes"); + } + + int magic = + ((int) (data[position] & 0xff) << 24) + + ((int) (data[position + 1] & 0xff) << 16) + + ((int) (data[position + 2] & 0xff) << 8) + + ((int) (data[position + 3] & 0xff) << 0); + position += 4; + if (magic != MAGIC && magic != MAGIC_V3) { + throw new SerealException( + String.format("Invalid Sereal header (%08x): doesn't match magic", magic)); + } + } + + private void checkHeaderSuffix() { + long suffix_size = read_varint(); + long basePosition = position; + + userHeaderSize = 0; + if (suffix_size > 0) { + byte bitfield = data[position++]; + + if ((bitfield & 0x01) == 0x01) { + userHeaderPosition = position; + userHeaderSize = suffix_size - 1; + } + } + + // skip everything in the optional suffix part + position = (int) (basePosition + suffix_size); + } + + private void checkNoEOD() throws SerealException { + + if ((end - position) <= 0) { + throw new SerealException("Unexpected end of data at byte " + position); + } + } + + private void checkProtoAndFlags() throws SerealException { + + if ((end - position) < 1) { + throw new SerealException("Invalid Sereal header: no protocol/version byte"); + } + + int protoAndFlags = data[position++]; + protocolVersion = protoAndFlags & 15; // 4 bits for version + + if (protocolVersion < 0 || protocolVersion > 4) { + throw new SerealException( + String.format("Invalid Sereal header: unsupported protocol version %d", protocolVersion)); + } + + encoding = (protoAndFlags & ~15) >> 4; + if ((encoding == 1 || encoding == 2) && refuseSnappy) { + throw new SerealException("Unsupported encoding: Snappy"); + } else if (encoding == 3 && refuseZlib) { + throw new SerealException("Unsupported encoding: Zlib"); + } else if (encoding == 4 && refuseZstd) { + throw new SerealException("Unsupported encoding: Zstd"); + } else if (encoding == 4 && protocolVersion < 4) { + throw new SerealException( + "Unsupported encoding zstd for protocol version " + protocolVersion); + } else if (encoding < 0 || encoding > 4) { + throw new SerealException("Unsupported encoding: unknown"); + } + } + + /** + * Indicates if the Sereal document has an header. + * + * @return {@code true} if the Sereal document has an header, {@code false} otherwise + * + * @throws SerealException if header cannot be parsed. + */ + public boolean hasHeader() throws SerealException { + parseHeader(); + + return userHeaderSize > 0; + } + + /** + * Size of the Sereal header. + * + * @return Size of the Sereal header, 0 if there is no header. + * + * @throws SerealException if header cannot be parsed. + */ + public long headerSize() throws SerealException { + parseHeader(); + + return userHeaderSize > 0 ? userHeaderSize : 0; + } + + /** + * Decode the Sereal document header and returns the decoded value. + * + * @return returns the header decoded value. + * + * @throws SerealException if header cannot be parsed. + */ + public Object decodeHeader() throws SerealException { + parseHeader(); + + if (userHeaderSize <= 0) throw new SerealException("Sereal user header not present"); + byte[] originalData = data; + int originalPosition = position, originalSize = end; + try { + data = originalData; + end = (int) (userHeaderPosition + userHeaderSize); + position = (int) userHeaderPosition; + + return readSingleValue(); + } finally { + data = originalData; + end = originalSize; + position = originalPosition; + resetTracked(); + } + } + + private void parseHeader() throws SerealException { + if (userHeaderSize >= 0) return; + + checkHeader(); + checkProtoAndFlags(); + checkHeaderSuffix(); + } + + /** + * Decode the Sereal document body and returns the decoded value. + * + * @return the decoded value. + * + * @throws SerealException if data cannot be parsed. + */ + public Object decode() throws SerealException { + + recursionDepth = 0; + + if (data == null) { + throw new SerealException("No data set"); + } + + parseHeader(); + + if (encoding != 0) { + if (encoding == 1 || encoding == 2) uncompressSnappy(); + else if (encoding == 3) uncompressZlib(); + else if (encoding == 4) uncompressZstd(); + if (protocolVersion == 1) baseOffset = 0; + else + // because offsets start at 1 + baseOffset = -1; + } else { + if (protocolVersion == 1) baseOffset = 0; + else + // because offsets start at 1 + baseOffset = position - 1; + } + Object out; + try { + out = readSingleValue(); + } catch (StackOverflowError error) { + throw new SerealException("StackOverflowError: Reached recursion limit during deserialization"); + } + + return out; + } + + private void uncompressSnappy() throws SerealException { + int len = originalData.length - (position - originalData.start); + int pos = protocolVersion == 1 ? position : originalData.start; + + if (encoding == 2) { + len = (int) read_varint(); + } + + byte[] uncompressed; + try { + if (!Snappy.isValidCompressedBuffer(originalData.array, position, len)) { + throw new SerealException("Invalid snappy data"); + } + int uncompressedLength = Snappy.uncompressedLength(originalData.array, position, len); + if (uncompressedLength > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + uncompressed = new byte[pos + uncompressedLength]; + Snappy.uncompress(originalData.array, position, len, uncompressed, pos); + } catch (IOException e) { + throw new SerealException(e); + } + this.data = uncompressed; + this.position = pos; + this.end = uncompressed.length; + } + + private void uncompressZlib() throws SerealException { + if (inflater == null) { + inflater = new Inflater(); + } + inflater.reset(); + + long uncompressedLength = read_varint(); + if (uncompressedLength > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + + long compressedLength = read_varint(); + inflater.setInput(originalData.array, position, (int) compressedLength); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + byte[] buffer = new byte[this.decodeBufferSize]; + while (!inflater.finished()) { + if (outputStream.size() > this.maxSize) { + throw new SerealException("The uncompressed size is larger than the allowed maximum size"); + } else if (outputStream.size() > uncompressedLength) { + throw new SerealException("The uncompressed size is larger than the expected size"); + } + int count = inflater.inflate(buffer); + if (count == 0 ) { + break; + } + outputStream.write(buffer, 0, count); + } + this.data = outputStream.toByteArray(); + } catch (DataFormatException | IOException e) { + throw new SerealException(e); + } + this.position = 0; + this.end = this.data.length; + } + + private void uncompressZstd() throws SerealException { + int len = (int) read_varint(); + + byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len); + long decompressedSize = Zstd.decompressedSize(compressedData); + + if (decompressedSize > this.maxSize) { + throw new SerealException("The expected uncompressed size is larger than the allowed maximum size"); + } + if (decompressedSize > Integer.MAX_VALUE) { + throw new SerealException("Decompressed size exceeds integer MAX_VALUE: " + decompressedSize); + } + byte[] uncompressed = new byte[(int) decompressedSize]; + long status = Zstd.decompress(uncompressed, compressedData); + if (Zstd.isError(status)) { + throw new SerealException(Zstd.getErrorName(status)); + } + this.data = uncompressed; + this.position = 0; + this.end = uncompressed.length; + } + + /** + * Decode a Sereal ARRAY tag to a native Java arrya + * + * @param length number of items in the array + * @param track we might need to track since array elements could refer to us + */ + private Object[] readNativeArray(int length, int track) throws SerealException { + if (maxNumArrayEntries != 0 && length > maxNumArrayEntries) { + throw new SerealException("Got input array with " + length + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + Object[] out = new Object[length]; + if (track != 0) { // track ourself + track_stuff(track, out); + } + + for (int i = 0; i < length; i++) { + out[i] = readSingleValue(); + } + + return out; + } + + /** + * Decode a Sereal ARRAY tag to a native Java List + * + * @param length number of items in the list + * @param track we might need to track since array elements could refer to us + */ + private List readList(int length, int track) throws SerealException { + + if (maxNumArrayEntries != 0 && length > maxNumArrayEntries) { + throw new SerealException("Got input array with " + length + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + List out = typeMapper.makeArray(length); + if (track != 0) { // track ourself + track_stuff(track, out); + } + + for (int i = 0; i < length; i++) { + out.add(readSingleValue()); + } + + return out; + } + + /** + * Reads a byte array, but was called read_binary in C, so for grepping purposes I kept the name + * + *

For some reason we call them Latin1Strings. + */ + private byte[] read_binary() { + int length = (int) read_varint(); + byte[] out = Arrays.copyOfRange(data, position, position + length); + + position += length; + + return out; + } + + private Map readMap(int num_keys, int track) throws SerealException { + + if (maxNumMapEntries != 0 && num_keys > maxNumMapEntries) { + throw new SerealException("Got input hash with " + num_keys + " entries, but the configured maximum is just " + maxNumMapEntries); + } + + Map hash = typeMapper.makeMap((int) num_keys); + if (track != 0) { // track ourself + track_stuff(track, hash); + } + + for (int i = 0; i < num_keys; i++) { + String key = readString(); + Object val = readSingleValue(); + hash.put(key, val); + } + + return hash; + } + + private Object get_tracked_item() { + long offset = read_varint(); + return tracked.get(offset); + } + + // top bit set (0x80) means next byte is 7 bits more more varint + private long read_varint() { + + long uv = 0; + int lshift = 0; + + byte b = data[position++]; + while ((position < end) && (b < 0)) { + uv |= ((long) b & 127) << lshift; // add 7 bits + lshift += 7; + b = data[position++]; + } + uv |= (long) b << lshift; // add final (or first if there is only 1) + + return uv; + } + + private Object readSingleValue() throws SerealException { + + checkNoEOD(); + + byte tag = data[position++]; + + int track = 0; + if ((tag & SRL_HDR_TRACK_FLAG) != 0) { + tag = (byte) (tag & ~SRL_HDR_TRACK_FLAG); + track = position - 1 - baseOffset; + } + + Object out; + + if (tag <= SRL_HDR_POS_HIGH) { + out = (long) tag; + } else if (tag <= SRL_HDR_NEG_HIGH) { + out = (long) (tag - 32); + } else if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + byte[] short_binary = read_short_binary(tag); + if (forceJavaStringForByteArrayValues) { + out = new String(short_binary); + } else { + out = preferLatin1 ? new Latin1String(short_binary) : short_binary; + } + } else if ((tag & SRL_HDR_HASHREF) == SRL_HDR_HASHREF) { + depthIncrement(); + + Map hash = readMap(tag & 0xf, track); + if (perlRefs) { + out = new PerlReference(hash); + } else { + out = hash; + } + + depthDecrement(); + } else if ((tag & SRL_HDR_ARRAYREF) == SRL_HDR_ARRAYREF) { + depthIncrement(); + + Object arr; + if (useObjectArray) { + arr = readNativeArray(tag & 0xf, track); + } else { + arr = readList(tag & 0xf, track); + } + if (perlRefs) { + out = new PerlReference(arr); + } else { + out = arr; + } + + depthDecrement(); + } else { + switch (tag) { + case SRL_HDR_VARINT: + long l = read_varint(); + if (l >= 0) { + out = l; + } else { + // long int greater than Long.MAX_VALUE wrapped around to negative: return a BigInteger + byte[] buffer = new byte[8]; + for (int i = 7; i >= 0; --i) { + buffer[i] = (byte) (l & 0xff); + l >>= 8; + } + out = new BigInteger(1, buffer); + } + break; + case SRL_HDR_ZIGZAG: + long zz = read_zigzag(); + out = zz; + break; + case SRL_HDR_FLOAT: + int floatBits = + ((int) (data[position + 3] & 0xff) << 24) + + ((int) (data[position + 2] & 0xff) << 16) + + ((int) (data[position + 1] & 0xff) << 8) + + ((int) (data[position] & 0xff) << 0); + position += 4; + float f = Float.intBitsToFloat(floatBits); + out = f; + break; + case SRL_HDR_DOUBLE: + long doubleBits = + ((long) (data[position + 7] & 0xff) << 56) + + ((long) (data[position + 6] & 0xff) << 48) + + ((long) (data[position + 5] & 0xff) << 40) + + ((long) (data[position + 4] & 0xff) << 32) + + ((long) (data[position + 3] & 0xff) << 24) + + ((long) (data[position + 2] & 0xff) << 16) + + ((long) (data[position + 1] & 0xff) << 8) + + ((long) (data[position] & 0xff) << 0); + position += 8; + double d = Double.longBitsToDouble(doubleBits); + out = d; + break; + case SRL_HDR_TRUE: + out = true; + break; + case SRL_HDR_FALSE: + out = false; + break; + case SRL_HDR_UNDEF: + if (preserveUndef) out = new PerlUndef(); + else out = null; + break; + case SRL_HDR_CANONICAL_UNDEF: + if (preserveUndef) out = PerlUndef.CANONICAL; + else out = null; + break; + case SRL_HDR_BINARY: + byte[] bytes = read_binary(); + if (forceJavaStringForByteArrayValues) { + out = new String(bytes); + } else { + out = preferLatin1 ? new Latin1String(bytes) : bytes; + } + break; + case SRL_HDR_STR_UTF8: + String utf8 = read_UTF8(); + out = utf8; + break; + case SRL_HDR_REFN: + if (perlRefs) { + PerlReference refn = new PerlReference(null); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, refn); + } + refn.setValue(readSingleValue()); + out = refn; + } else { + depthIncrement(); + out = readSingleValue(); + depthDecrement(); + } + break; + case SRL_HDR_REFP: + long offset_prev = read_varint(); + Object prv_value = tracked.get(offset_prev); + if (prv_value == RefpMap.NOT_FOUND) { + throw new SerealException("REFP to offset " + offset_prev + ", which is not tracked"); + } + Object prev = perlRefs ? new PerlReference(prv_value) : prv_value; + out = prev; + break; + case SRL_HDR_OBJECT: + if (refuseObjects) + throw new SerealException( + String.format( + "Encountered object in input, but the 'refuseObject' option is in effect at offset %d of input", + position)); + Object obj = readObject(); + out = obj; + break; + case SRL_HDR_OBJECTV: + if (refuseObjects) + throw new SerealException( + String.format( + "Encountered object in input, but the 'refuseObject' option is in effect at offset %d of input", + position)); + String className = readStringCopy(); + out = readObject(className); + break; + case SRL_HDR_COPY: + Object copy = read_copy(); + out = copy; + break; + case SRL_HDR_ALIAS: + Object value = get_tracked_item(); + + if (perlAlias) { + out = new PerlAlias(value); + } else { + out = value; + } + break; + case SRL_HDR_WEAKEN: + // so the next thing HAS to be a ref (afaict) which means we can track it + if (perlRefs) { + PerlReference placeHolder = new PerlReference(null); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, placeHolder); + } + placeHolder.setValue(((PerlReference) readSingleValue()).getValue()); + WeakReference wref = new WeakReference(placeHolder); + out = wref; + } else { + Object ref = readSingleValue(); + // track early for weak references + if (track != 0) { // track ourself + track_stuff(track, ref); + } + WeakReference wref = new WeakReference(ref); + out = wref; + } + break; + case SRL_HDR_HASH: + depthIncrement(); + Object hash = readMap((int) read_varint(), track); + out = hash; + depthDecrement(); + break; + case SRL_HDR_ARRAY: + depthIncrement(); + if (useObjectArray) { + out = readNativeArray((int) read_varint(), track); + } else { + out = readList((int) read_varint(), track); + } + depthDecrement(); + break; + case SRL_HDR_REGEXP: + Pattern pattern = read_regex(); + out = pattern; + break; + case SRL_HDR_PAD: + return readSingleValue(); + default: + throw new SerealException("Tag not supported: " + tag); + } + } + + if (track != 0) { // we double-track arrays ATM (but they just overwrite) + track_stuff(track, out); + } + + return out; + } + + /** + * Read a short binary ISO-8859-1 (latin1) string, the lower bits of the tag hold the length + * + * @param tag the Sereal SHORT_BINARY_* tag + */ + private byte[] read_short_binary(byte tag) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + byte[] buf = Arrays.copyOfRange(data, position, position + length); + position += length; + return buf; + } + + /** + * From the spec: Sometimes it is convenient to be able to reuse a previously emitted sequence in + * the packet to reduce duplication. For instance a data structure with many hashes with the same + * keys. The COPY tag is used for this. Its argument is a varint which is the offset of a + * previously emitted tag, and decoders are to behave as though the tag it references was inserted + * into the packet stream as a replacement for the COPY tag. + * + *

Note, that in this case the track flag is not set. It is assumed the decoder can jump back + * to reread the tag from its location alone. + * + *

Copy tags are forbidden from referring to another COPY tag, and are also forbidden from + * referring to anything containing a COPY tag, with the exception that a COPY tag used as a value + * may refer to an tag that uses a COPY tag for a classname or hash key. + */ + private Object read_copy() throws SerealException { + + int originalPosition = (int) read_varint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + Object copy = readSingleValue(); + position = currentPosition; // go back to where we were + + return copy; + } + + private String readStringCopy() throws SerealException { + int originalPosition = (int) read_varint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + String copy = readString(); + position = currentPosition; // go back to where we were + + return copy; + } + + private String read_UTF8() throws SerealException { + int length = (int) read_varint(); + int originalPosition = position; + + position += length; + + if (maxStringLength != 0 && length > maxStringLength) { + throw new SerealException("Got input string with " + length + " characters, but the configured maximum is just " + maxStringLength); + } + + return new String(data, originalPosition, length, charset_utf8); + } + + private long read_zigzag() { + + long n = read_varint(); + + return (n >>> 1) ^ (-(n & 1)); // note the unsigned right shift + } + + private Pattern read_regex() throws SerealException { + + int flags = 0; + Object str = readSingleValue(); + String regex; + if (str instanceof CharSequence) { + regex = ((CharSequence) str).toString(); + } else if (str instanceof byte[]) { + regex = (new Latin1String((byte[]) str)).toString(); + } else { + throw new SerealException("Regex has to be built from a char or byte sequence"); + } + + // now read modifiers + byte tag = data[position++]; + if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + while (length-- > 0) { + byte value = data[position++]; + switch (value) { + case 'm': + flags = flags | Pattern.MULTILINE; + break; + case 's': + flags = flags | Pattern.DOTALL; + break; + case 'i': + flags = flags | Pattern.CASE_INSENSITIVE; + break; + case 'x': + flags = flags | Pattern.COMMENTS; + break; + case 'p': + // ignored + break; + default: + throw new SerealException("Unknown regex modifier: " + value); + } + } + } else { + throw new SerealException( + "Expecting SRL_HDR_SHORT_BINARY for modifiers of regexp, got: " + tag); + } + + return Pattern.compile(regex, flags); + } + + private Object readObject() throws SerealException { + Object className = readString(); + + return readObject(className.toString()); + } + + private Object readObject(String className) throws SerealException { + Object structure = readSingleValue(); + if (stripObjects) return structure; + Object object = typeMapper.makeObject(className, structure); + return object; + } + + private String readString() throws SerealException { + checkNoEOD(); + + byte tag = data[position++]; + + if ((tag & SRL_HDR_SHORT_BINARY_LOW) == SRL_HDR_SHORT_BINARY_LOW) { + int length = tag & SRL_MASK_SHORT_BINARY_LEN; + String string = new String(data, position, length, charset_latin1); + + position += length; + + return string; + } else if (tag == SRL_HDR_BINARY) { + int length = (int) read_varint(); + String string = new String(data, position, length, charset_latin1); + + position += length; + + return string; + } else if (tag == SRL_HDR_STR_UTF8) { + return read_UTF8(); + } else if (tag == SRL_HDR_COPY) { + return readStringCopy(); + } else { + throw new SerealException("Tag " + tag + " is not a string tag"); + } + } + + /** + * Set the Sereal data to be decoded. + *

+ * The caller must not modify the data while it is owned by the decoder. + * + * @param blob Sereal data to decode. + */ + public void setData(ByteArray blob) { + reset(); + originalData = blob; + data = originalData.array; + end = originalData.start + originalData.length; + position = originalData.start; + } + + /** + * Set the Sereal data to be decoded. + *

+ * The caller must not modify the data while it is owned by the decoder. + * + * @param blob Sereal data to decode. + */ + public void setData(byte[] blob) { + reset(); + originalData = new ByteArray(blob); + data = blob; + end = blob.length; + position = 0; + } + + private void track_stuff(int pos, Object ref) { + tracked.put(pos, ref); + } + + private void reset() { + originalData = null; + data = null; + protocolVersion = encoding = -1; + baseOffset = Integer.MAX_VALUE; + userHeaderPosition = userHeaderSize = -1; + resetTracked(); + } + + private void resetTracked() { + tracked.clear(); + } + + private void depthIncrement() throws SerealException { + if (recursionDepth++ > maxRecursionDepth) { + throw new SerealException("Reached recursion limit (" + maxRecursionDepth + ") during deserialization"); + } + } + + private void depthDecrement() { + recursionDepth--; + } + + /** + * Close the decoder to recycle the resources. + *

+ * Returns the native memory used by inflater explicitly before the garbage collector. + */ + public void close() { + if (inflater != null) { + inflater.end(); + inflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/DecoderOptions.java b/src/main/java/com/booking/sereal/DecoderOptions.java new file mode 100644 index 0000000000..ef924e05aa --- /dev/null +++ b/src/main/java/com/booking/sereal/DecoderOptions.java @@ -0,0 +1,271 @@ +package com.booking.sereal; + +public class DecoderOptions { + private boolean perlRefs = false; + private boolean perlAlias = false; + private boolean preserveUndef = false; + private boolean preferLatin1 = false; + private boolean refuseSnappy = false; + private boolean refuseObjects = false; + private boolean refuseZlib = false; + private boolean refuseZstd = false; + private boolean stripObjects = false; + private boolean forceJavaStringForByteArrayValues = false; + + private int maxRecursionDepth = 10_000; + private int maxNumMapEntries = 0; + private int maxNumArrayEntries = 0; + private int maxStringLength = 0; + + // Size to use on the buffer used to read the data. Defaults to 1KB + private int decodeBufferSize = 1024; + // Maximum size allowed for the data. Defaults to 100MB + private int maxSize = 100 * 1024 * 1024; + + private TypeMapper typeMapper = new DefaultTypeMapper(); + + public boolean perlReferences() { + return perlRefs; + } + + public boolean perlAliases() { + return perlAlias; + } + + public boolean preserveUndef() { + return preserveUndef; + } + + public boolean preferLatin1() { + return preferLatin1; + } + + /** + * If set, the decoder will refuse Snappy-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Snappy is refused, {@code False} otherwise + */ + public boolean refuseSnappy() { + return refuseSnappy; + } + + /** + * If set, the decoder will refuse deserializing any objects in the input stream and + * instead throw an exception. + * + * @return {@code True} if Objects are refused, {@code False} otherwise + */ + public boolean refuseObjects() { + return refuseObjects; + } + + /** + * If set, the decoder will refuse Zlib-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Zlib is refused, {@code False} otherwise + */ + public boolean refuseZlib() { + return refuseZlib; + } + + /** + * If set, the decoder will refuse Zstd-compressed input data. This can be + * desirable for robustness. + * + * @return {@code True} if Zstd is refused, {@code False} otherwise + */ + public boolean refuseZstd() { + return refuseZstd; + } + + public boolean stripObjects() { + return stripObjects; + } + + public boolean forceJavaStringForByteArrayValues() { + return forceJavaStringForByteArrayValues; + } + + public TypeMapper typeMapper() { + return typeMapper; + } + + public int bufferSize() { + return decodeBufferSize; + } + + public int maxBufferSize() { + return maxSize; + } + + /** + * {@link Decoder} is recursive. If you pass it a Sereal document that is deeply + * nested, it will eventually exhaust the Java stack. Therefore, there is a limit on + * the depth of recursion that is accepted. It defaults to 10000 nested calls. You + * may choose to override this value with the {@link DecoderOptions#maxRecursionDepth(int)} option. + * Beware that setting it too high can cause hard crashes. + * + * Do note that the setting is somewhat approximate. Setting it to 10000 may break at + * somewhere between 9997 and 10003 nested structures depending on their types. + * + * It's also important to note that this value doesn't mirror the Perl implementation. Perl and Java + * Sereal wrappers are implemented in a different way. That means that the recursion depth marked + * as max in Perl may be different than the one needed in Java. The same applies for the Encoder + * and Decoder options. + * + * The Java stack can be controlled with different flags (for example -Xss) to change it's size. + * + * @return maximum recursion depth + */ + public int maxRecursionDepth() { + return maxRecursionDepth; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any hash/dictionary (or hash-based object) with more than + * that number of entries. This is to be able to respond quickly to any future + * hash-collision attacks on Perl's hash function. Chances are, you don't want + * or need this. For a gentle introduction to the topic from the cryptographic + * point of view, see Collision attack. + * + * This value can be override with {@link DecoderOptions#maxNumMapEntries(int)} option + * + * @return maximum number of map entries + */ + public int maxNumMapEntries() { + return maxNumMapEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any array with more than that number of entries. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link DecoderOptions#maxNumArrayEntries(int)} option + * + * @return maximum number of array entries + */ + public int maxNumArrayEntries() { + return maxNumArrayEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Decoder} will refuse + * to deserialize any string with more than that number of characters. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link DecoderOptions#maxStringLength(int)} option + * + * @return maximum supported string length + */ + public int maxStringLength() { + return maxStringLength; + } + + public DecoderOptions perlReferences(boolean perlReferences) { + this.perlRefs = perlReferences; + + return this; + } + + public DecoderOptions perlAliases(boolean perlAliases) { + this.perlAlias = perlAliases; + + return this; + } + + public DecoderOptions forceJavaStringForByteArrayValues(boolean forceJavaStringForByteArrayValues) { + this.forceJavaStringForByteArrayValues = forceJavaStringForByteArrayValues; + + return this; + } + + public DecoderOptions preserveUndef(boolean preserveUndef) { + this.preserveUndef = preserveUndef; + + return this; + } + + public DecoderOptions preferLatin1(boolean preferLatin1) { + this.preferLatin1 = preferLatin1; + + return this; + } + + public DecoderOptions refuseSnappy(boolean refuseSnappy) { + this.refuseSnappy = refuseSnappy; + + return this; + } + + public DecoderOptions refuseObjects(boolean refuseObjects) { + this.refuseObjects = refuseObjects; + + return this; + } + + public DecoderOptions stripObjects(boolean stripObjects) { + this.stripObjects = stripObjects; + + return this; + } + + public DecoderOptions refuseZlib(boolean refuseZlib) { + this.refuseZlib = refuseZlib; + + return this; + } + + public DecoderOptions refuseZstd(boolean refuseZstd) { + this.refuseZstd = refuseZstd; + + return this; + } + + public DecoderOptions typeMapper(TypeMapper typeMapper) { + this.typeMapper = typeMapper; + + return this; + } + + public DecoderOptions bufferSize(int bufferSize) { + this.decodeBufferSize = bufferSize; + + return this; + } + + public DecoderOptions maxBufferSize(int maxSize) { + this.maxSize = maxSize; + + return this; + } + + public DecoderOptions maxRecursionDepth(int maxRecursionDepth) { + this.maxRecursionDepth = maxRecursionDepth; + + return this; + } + + public DecoderOptions maxNumMapEntries(int maxNumMapEntries) { + this.maxNumMapEntries = maxNumMapEntries; + + return this; + } + + public DecoderOptions maxNumArrayEntries(int maxNumArrayEntries) { + this.maxNumArrayEntries = maxNumArrayEntries; + + return this; + } + + public DecoderOptions maxStringLength(int maxStringLength) { + this.maxStringLength = maxStringLength; + + return this; + } +} diff --git a/src/main/java/com/booking/sereal/DefaultTypeMapper.java b/src/main/java/com/booking/sereal/DefaultTypeMapper.java new file mode 100644 index 0000000000..48de5a8dc5 --- /dev/null +++ b/src/main/java/com/booking/sereal/DefaultTypeMapper.java @@ -0,0 +1,28 @@ +package com.booking.sereal; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DefaultTypeMapper implements TypeMapper { + @Override + public boolean useObjectArray() { + return false; + } + + @Override + public List makeArray(int size) { + return new ArrayList(size); + } + + @Override + public Map makeMap(int size) { + return new HashMap(size); + } + + @Override + public Object makeObject(String className, Object data) { + return new PerlObject(className, data); + } +} diff --git a/src/main/java/com/booking/sereal/Encoder.java b/src/main/java/com/booking/sereal/Encoder.java new file mode 100644 index 0000000000..390b747496 --- /dev/null +++ b/src/main/java/com/booking/sereal/Encoder.java @@ -0,0 +1,1014 @@ +package com.booking.sereal; + +import com.booking.sereal.impl.BytearrayCopyMap; +import com.booking.sereal.impl.IdentityMap; +import com.booking.sereal.impl.StringCopyMap; +import com.github.luben.zstd.Zstd; +import java.math.BigInteger; +import org.xerial.snappy.Snappy; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.lang.reflect.Array; +import java.nio.charset.Charset; +import java.util.*; +import java.util.regex.Pattern; +import java.util.zip.Deflater; + +import static com.booking.sereal.EncoderOptions.CompressionType; + +/** + * Sereal encoder with Perl-like interface. + *

+ * This class can be used to encode Perl-like data-structures: (boxed) primitive types, strings, arrays + * and maps. + */ +public class Encoder { + + private static final EncoderOptions DEFAULT_OPTIONS = new EncoderOptions(); + private static final byte[] EMPTY_ARRAY = new byte[0]; + private static final BigInteger LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger UNSIGNED_LONG_MAX_VALUE = new BigInteger(1, new byte[] { + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, + }); + + private final boolean perlRefs; + private final boolean perlAlias; + private final int protocolVersion, encoding; + private final CompressionType compressionType; + private final long compressionThreshold; + private Deflater deflater; + private final int zstdCompressionLevel; + + private final int maxRecursionDepth; + private final int maxNumMapEntries; + private final int maxNumArrayEntries; + private final int maxStringLength; + + private final byte[] HEADER = + new byte[] { + (byte) (SerealHeader.MAGIC >> 24), + (byte) (SerealHeader.MAGIC >> 16), + (byte) (SerealHeader.MAGIC >> 8), + (byte) (SerealHeader.MAGIC >> 0), + }; + private final byte[] HEADER_V3 = + new byte[] { + (byte) (SerealHeader.MAGIC_V3 >> 24), + (byte) (SerealHeader.MAGIC_V3 >> 16), + (byte) (SerealHeader.MAGIC_V3 >> 8), + (byte) (SerealHeader.MAGIC_V3 >> 0), + }; + // so we don't need to allocate this every time we encode a varint + private byte[] varint_buf = new byte[12]; + // track things we've encoded so we can emit refs and copies + private IdentityMap tracked = new IdentityMap(); + private IdentityMap aliases, maybeAliases; + private BytearrayCopyMap trackedBytearrayCopy = new BytearrayCopyMap(); + private StringCopyMap trackedStringCopy = new StringCopyMap(); + private StringCopyMap trackedClassnames = new StringCopyMap(); + // where we store the various encoded things + private byte[] bytes = new byte[1024]; + private byte[] compressedBytes = EMPTY_ARRAY; + private long size = 0; // size of everything encoded so far + private long compressedSize; + private int headerSize, headerOffset; + private Charset charset_utf8 = Charset.forName("UTF-8"); + + private int recursionDepth = 0; + + /** Create a new Encoder with default options. */ + public Encoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create a new Encoder with the specified options. + * + * @param options {@link EncoderOptions} to apply. + */ + public Encoder(EncoderOptions options) { + perlRefs = options.perlReferences(); + perlAlias = options.perlAliases(); + protocolVersion = options.protocolVersion(); + compressionType = options.compressionType(); + compressionThreshold = options.compressionThreshold(); + zstdCompressionLevel = options.zstdCompressionLevel(); + + maxRecursionDepth = options.maxRecursionDepth(); + maxNumMapEntries = options.maxNumMapEntries(); + maxNumArrayEntries = options.maxNumArrayEntries(); + maxStringLength = options.maxStringLength(); + + switch (protocolVersion) { + case 4: + encoding = + compressionType.equals(CompressionType.ZSTD) + ? 4 + : compressionType.equals(CompressionType.ZLIB) + ? 3 + : compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 3: + encoding = + compressionType.equals(CompressionType.ZLIB) + ? 3 + : compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 2: + encoding = compressionType.equals(CompressionType.SNAPPY) ? 2 : 0; + break; + case 1: + encoding = compressionType.equals(CompressionType.SNAPPY) ? 1 : 0; + break; + default: + encoding = 0; + break; + } + if (encoding == 3) deflater = new Deflater(options.zlibCompressionLevel()); + else deflater = null; + + if (perlAlias) { + aliases = new IdentityMap(); + maybeAliases = new IdentityMap(); + } + } + + private static void prepareHeader( + byte[] originBytes, byte[] compressedBytes, int headerSize, int sizeLength) { + System.arraycopy(originBytes, 0, compressedBytes, 0, headerSize); + // varint-encoded 0, filling all space + for (int i = headerSize; i < headerSize + sizeLength - 1; i++) compressedBytes[i] = (byte) 128; + compressedBytes[headerSize + sizeLength - 1] = 0; + } + + private static void finishHeader( + byte[] compressedBytes, long compressedSize, int headerSize, int sizeLength) { + int after = encodeVarint(compressedSize, compressedBytes, headerSize); + if (after != headerSize + sizeLength) compressedBytes[after - 1] |= (byte) 0x80; + } + + private static int varintLength(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + n >>>= 7; + length++; + } + + return length + 1; + } + + private static int encodeVarint(long n, byte[] buffer, int pos) { + while (n > 127) { + buffer[pos++] = (byte) ((n & 127) | 128); + n >>= 7; + } + buffer[pos++] = (byte) n; + + return pos; + } + + // write header and version/encoding + private void init(Object header, boolean hasHeader) throws SerealException { + if (protocolVersion >= 3) appendBytesUnsafe(HEADER_V3); + else appendBytesUnsafe(HEADER); + appendByteUnsafe((byte) ((encoding << 4) | protocolVersion)); + + if (hasHeader) { + encodeUserHeader(header); + } else { + appendByteUnsafe((byte) 0x00); + } + + headerSize = (int) size; + if (protocolVersion > 1) + // because offsets start at 1 + headerOffset = headerSize - 1; + else headerOffset = 0; + } + + private void encodeUserHeader(Object header) throws SerealException { + long originalSize = size; + + // be optimistic about encoded header size + size += 2; // one for the size, one for 8bit bitfield + encode(header); + + int suffixSize = (int) (size - originalSize - 1); + if (suffixSize < 128) { + bytes[(int) originalSize] = (byte) suffixSize; + bytes[(int) originalSize + 1] = 0x01; + } else { + // we were too optimistic + int sizeLength = varintLength(suffixSize); + + // make space + ensureAvailable(sizeLength - 1); + System.arraycopy( + bytes, + (int) originalSize + 2, + bytes, + (int) originalSize + sizeLength + 1, + suffixSize - 1); + size += sizeLength - 1; + + // now write size and 8bit bitfield + encodeVarint(suffixSize, bytes, (int) originalSize); + bytes[(int) originalSize + sizeLength] = 0x01; + } + + resetTracked(); + } + + /** + * After a call to {@code write()}, returns a reference of the encoded data. + *

+ * The reference is only valid until the next call to {@code write()}. + * + * @return reference to the encoded data. + */ + public ByteArray getDataReference() { + if (compressedSize != 0) { + return new ByteArray(compressedBytes, (int) compressedSize); + } else { + return new ByteArray(bytes, (int) size); + } + } + + /** + * After a call to {@code write()}, returns a copy of the encoded data. + * + * @return copy of the encoded data. + */ + public byte[] getData() { + if (compressedSize != 0) { + return Arrays.copyOf(compressedBytes, (int) compressedSize); + } else { + return Arrays.copyOf(bytes, (int) size); + } + } + + private void markNotCompressed() { + compressedSize = 0; + bytes[4] &= (byte) 0xf; + } + + private void compressSnappy() throws SerealException { + int maxSize = Snappy.maxCompressedLength((int) size - headerSize); + int sizeLength = encoding == 2 ? varintLength(maxSize) : 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + maxSize) > compressedBytes.length) + compressedBytes = new byte[headerSize + sizeLength + maxSize]; + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + int compressed; + try { + compressed = + Snappy.compress( + bytes, headerSize, (int) size - headerSize, compressedBytes, headerSize + sizeLength); + } catch (IOException e) { + throw new SerealException(e); + } + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + if (encoding == 2) { + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + } + + // from miniz.c + private int zlibMaxSize(int sourceLen) { + return Math.max( + 128 + (sourceLen * 110) / 100, 128 + sourceLen + ((sourceLen / (31 * 1024)) + 1) * 5); + } + + private void compressZlib() { + deflater.reset(); + + int sourceSize = (int) size - headerSize; + int maxSize = zlibMaxSize(sourceSize); + int sizeLength = varintLength(sourceSize); + int sizeLength2 = varintLength(maxSize); + int pos = 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + sizeLength2 + maxSize) > compressedBytes.length) + compressedBytes = new byte[headerSize + sizeLength + sizeLength2 + maxSize]; + + System.arraycopy(bytes, 0, compressedBytes, 0, headerSize); + pos += headerSize; + pos = encodeVarint(sourceSize, compressedBytes, pos); + + // varint-encoded 0, filling all space + int encodedSizePos = pos; + for (int max = pos + sizeLength2 - 1; pos < max; ) compressedBytes[pos++] = (byte) 128; + compressedBytes[pos++] = 0; + + deflater.setInput(bytes, headerSize, sourceSize); + deflater.finish(); + + int compressed = deflater.deflate(compressedBytes, pos, compressedBytes.length - pos); + compressedSize = headerSize + sizeLength + sizeLength2 + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + int after = encodeVarint(compressed, compressedBytes, encodedSizePos); + if (after != headerSize + sizeLength + sizeLength2) compressedBytes[after - 1] |= (byte) 0x80; + } + + private void compressZstd() throws SerealException { + long maxSize = Zstd.compressBound((int) size - headerSize); + int sizeLength = varintLength(maxSize); + + if (headerSize + sizeLength + maxSize > Integer.MAX_VALUE) + throw new SerealException( + "Compressed data size exceeds integer MAX_VALUE: " + (headerSize + maxSize)); + if (headerSize + sizeLength + maxSize > compressedBytes.length) + compressedBytes = new byte[(int) (headerSize + sizeLength + maxSize)]; + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + long compressed = + Zstd.compressUsingDict( + compressedBytes, + headerSize + sizeLength, + bytes, + headerSize, + (int) size - headerSize, + new byte[0], + zstdCompressionLevel); + if (Zstd.isError(compressed)) throw new SerealException(Zstd.getErrorName(compressed)); + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + + /** + * Write an integer as a varint + * + *

Note: sometimes the next thing while decoding is know to be a varint, sometimes there must + * be a tag that denotes the next item *is* a varint. So don't forget to write that tag. + * + * @param n positive integer + */ + private void appendVarint(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + varint_buf[length++] = (byte) ((n & 127) | 128); + n >>>= 7; + } + varint_buf[length++] = (byte) n; + + appendBytes(varint_buf, length); + } + + private void setTrackBit(long offset) { + bytes[(int) offset + headerOffset] |= (byte) 0x80; + } + + /** + * Encode a number as zigzag + * + * @param n nageative integer + */ + private void appendZigZag(long n) { + appendByte(SerealHeader.SRL_HDR_ZIGZAG); + appendVarint((n << 1) ^ (n >> 63)); // note the signed right shift + } + + /** + * Encode a short ascii string + * + * @param latin1 String to encode as US-ASCII bytes + * @throws SerealException if the string is not short enough + */ + private void appendShortBinary(byte[] latin1) throws SerealException { + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(latin1); + if (copyOffset != BytearrayCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + int length = latin1.length; + long location = size; + + if (length > 31) { + throw new SerealException("Cannot create short binary for " + latin1 + ": too long"); + } + + // length of string + appendByte((byte) (length | SerealHeader.SRL_HDR_SHORT_BINARY)); + + // save it + appendBytes(latin1); + + trackForCopy(latin1, location); + } + + /** + * Encode a long ascii string + * + * @param latin1 String to encode as US-ASCII bytes + */ + private void appendBinary(byte[] latin1) { + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(latin1); + if (copyOffset != BytearrayCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + int length = latin1.length; + long location = size; + + // length of string + appendByte(SerealHeader.SRL_HDR_BINARY); + appendBytesWithLength(latin1); + + trackForCopy(latin1, location); + } + + private void appendCopy(long location) { + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(location); + } + + /** + * Encode a regex + * + * @param p regex pattern. Only support flags "smix": DOTALL | MULTILINE | CASE_INSENSITIVE | + * COMMENTS + * @throws SerealException if the pattern is longer that a short binary string + */ + private void appendRegex(Pattern p) throws SerealException { + + byte[] flags = new byte[4]; + int flags_size = 0; + if ((p.flags() & Pattern.MULTILINE) != 0) flags[flags_size++] = 'm'; + if ((p.flags() & Pattern.DOTALL) != 0) flags[flags_size++] = 's'; + if ((p.flags() & Pattern.CASE_INSENSITIVE) != 0) flags[flags_size++] = 'i'; + if ((p.flags() & Pattern.COMMENTS) != 0) flags[flags_size++] = 'x'; + + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendStringType(new Latin1String(p.pattern())); + appendByte((byte) (flags_size | SerealHeader.SRL_HDR_SHORT_BINARY)); + appendBytes(flags, flags_size); + } + + /** + * Encodes a byte array emitting both length and data + */ + private void appendBytesWithLength(byte[] in) { + appendVarint(in.length); + appendBytes(in); + } + + private void appendBoolean(boolean b) { + appendByte(b ? SerealHeader.SRL_HDR_TRUE : SerealHeader.SRL_HDR_FALSE); + } + + /** + * Create a new Sereal document containing the specified value in the body. + *

+ * Each call to this method overwrites the current Sereal document. + * + * @param obj Object to encode + * + * @return new Sereal document + * + * @throws SerealException Object could not be encoded. + */ + public Encoder write(Object obj) throws SerealException { + return write(obj, null, false); + } + + /** + * Create a new Sereal document with the given header and body. + *

+ * Each call to this method overwrites the current Sereal document. + * + * @param obj Object to encode + * @param header Sereal header + * + * @return new Sereal document + * + * @throws SerealException Object could not be encoded. + */ + public Encoder write(Object obj, Object header) throws SerealException { + return write(obj, header, true); + } + + private Encoder write(Object obj, Object header, boolean hasHeader) throws SerealException { + if (hasHeader && protocolVersion == 1) + throw new SerealException("Can't encode user header in Sereal protocol version 1"); + + reset(); + init(header, hasHeader); + try { + encode(obj); + } catch (StackOverflowError error) { + throw new SerealException("StackOverflowError: Reached recursion limit during serialization"); + } + + if (size - headerSize > compressionThreshold) { + if (compressionType.equals(CompressionType.SNAPPY)) compressSnappy(); + else if (compressionType.equals(CompressionType.ZLIB)) compressZlib(); + else if (compressionType.equals(CompressionType.ZSTD)) compressZstd(); + } else { + // we did not do compression after all + markNotCompressed(); + } + + return this; + } + + @SuppressWarnings("unchecked") + private void encode(Object obj) throws SerealException { + // track it (for ALIAS tags) + long location = size; + + if (perlAlias) { + long aliasOffset = aliases.get(obj); + if (aliasOffset != IdentityMap.NOT_FOUND) { + appendAlias(aliasOffset); + return; + } else { + maybeAliases.put(obj, location - headerOffset); + } + } + + Class type = obj == null ? PerlUndef.class : obj.getClass(); + + // this needs to be first for obvious reasons :) + if (type == PerlUndef.class) { + if (protocolVersion == 3 && obj == PerlUndef.CANONICAL) + appendByte(SerealHeader.SRL_HDR_CANONICAL_UNDEF); + else appendByte(SerealHeader.SRL_HDR_UNDEF); + return; + } + + // this is ugly :) + if (type == Long.class || type == Integer.class || type == Byte.class) { + appendNumber(((Number) obj).longValue()); + } else if (type == Boolean.class) { + appendBoolean((Boolean) obj); + } else if (type == String.class) { + appendStringType((String) obj); + } else if (type == Latin1String.class) { + appendStringType((Latin1String) obj); + } else if (type == byte[].class) { + appendStringType((byte[]) obj); + } else if (type.isArray()) { + if (perlRefs || !tryAppendRefp(obj)) { + if (obj instanceof Object[]) { + if (!(obj instanceof String[])) { + depthIncrement(); + } + appendArray((Object[]) obj); + if (!(obj instanceof String[])) { + depthDecrement(); + } + } else { + appendArray(obj); + } + } + } else if (type == HashMap.class || obj instanceof Map) { + depthIncrement(); + if (perlRefs || !tryAppendRefp(obj)) appendMap((Map) obj); + depthDecrement(); + } else if (type == ArrayList.class || obj instanceof List) { + depthIncrement(); + if (perlRefs || !tryAppendRefp(obj)) appendArray((List) obj); + depthDecrement(); + } else if (type == Pattern.class) { + appendRegex((Pattern) obj); + } else if (type == Double.class) { + appendDouble((Double) obj); + } else if (type == Float.class) { + appendFloat((Float) obj); + } else if (type == BigInteger.class) { + appendBigInteger((BigInteger) obj); + } else if (type == PerlReference.class) { + PerlReference ref = (PerlReference) obj; + long trackedRef = getTrackedItem(ref.getValue()); + + if (trackedRef != IdentityMap.NOT_FOUND) { + appendRefp(trackedRef); + } else { + appendRef(ref); + } + } else if (type == WeakReference.class) { + Object value = ((WeakReference) obj).get(); + boolean isRef = isDefinitelyReference(value); + long currentOffset = size; + + appendByte(SerealHeader.SRL_HDR_WEAKEN); + + if (!isRef) { + appendByte(SerealHeader.SRL_HDR_PAD); + } + encode(value); + if (!isRef) { + if (!isRefTag(bytes[(int) currentOffset + 2])) { + bytes[(int) currentOffset + 1] = SerealHeader.SRL_HDR_REFN; + } + } else { + if (!isRefTag(bytes[(int) currentOffset + 1])) { + throw new SerealException("Internal error while encoding weak reference"); + } + } + } else if (type == PerlAlias.class) { + Object value = ((PerlAlias) obj).getValue(); + + if (perlAlias) { + long maybeAlias = maybeAliases.get(value); + long alias = aliases.get(value); + + if (alias != IdentityMap.NOT_FOUND) { + appendAlias(alias); + } else if (maybeAlias != IdentityMap.NOT_FOUND) { + appendAlias(maybeAlias); + aliases.put(value, maybeAlias); + } else { + encode(value); + aliases.put(value, location - headerOffset); + } + } else { + encode(value); + } + } else if (type == PerlObject.class) { + PerlObject po = (PerlObject) obj; + + appendPerlObject(po.getName(), po.getData()); + } + + if (size == location) { // didn't write anything + throw new SerealException( + "Don't know how to encode: " + type.getName() + " = " + obj.toString()); + } + } + + private void appendPerlObject(String className, Object data) throws SerealException { + long nameOffset = trackedClassnames.get(className); + if (nameOffset != StringCopyMap.NOT_FOUND) { + + appendByte(SerealHeader.SRL_HDR_OBJECTV); + appendVarint(nameOffset); + } else { + appendByte(SerealHeader.SRL_HDR_OBJECT); + trackedClassnames.put(className, size - headerOffset); + appendStringType(className); + } + + // write the data structure + encode(data); + } + + /** + * @param obj object that might have been already encoded earlier in the bytestream + * @return location of object in bytestream, or {@link IdentityMap#NOT_FOUND} + */ + private long getTrackedItem(Object obj) { + return tracked.get(obj); + } + + private long getTrackedItemCopy(byte[] bytes) { + return trackedBytearrayCopy.get(bytes); + } + + private long getTrackedItemCopy(String string) { + return trackedStringCopy.get(string); + } + + private void track(Object obj, long obj_location) { + tracked.put(obj, obj_location - headerOffset); + } + + private void trackForCopy(byte[] bytes, long location) { + trackedBytearrayCopy.put(bytes, location - headerOffset); + } + + private void trackForCopy(String string, long location) { + trackedStringCopy.put(string, location - headerOffset); + } + + private void appendDouble(Double d) { + appendByte(SerealHeader.SRL_HDR_DOUBLE); + + long bits = Double.doubleToLongBits(d); // very convienent, thanks Java guys! :) + for (int i = 0; i < 8; i++) { + varint_buf[i] = (byte) ((bits >> (i * 8)) & 0xff); + } + appendBytes(varint_buf, 8); + } + + private void appendFloat(Float f) { + appendByte(SerealHeader.SRL_HDR_FLOAT); + + int bits = Float.floatToIntBits(f); // very convienent, thanks Java guys! :) + for (int i = 0; i < 4; i++) { + varint_buf[i] = (byte) ((bits >> (i * 8)) & 0xff); + } + appendBytes(varint_buf, 4); + } + + private void appendRefp(long location) { + setTrackBit(location); + appendByte(SerealHeader.SRL_HDR_REFP); + appendVarint(location); + } + + private boolean tryAppendRefp(Object obj) { + long location = getTrackedItem(obj); + + if (location != IdentityMap.NOT_FOUND) { + appendRefp(location); + + return true; + } else { + return false; + } + } + + private void appendAlias(long location) { + setTrackBit(location); + appendByte(SerealHeader.SRL_HDR_ALIAS); + appendVarint(location); + } + + private void appendMap(Map hash) throws SerealException { + + if (maxNumMapEntries != 0 && hash.size() > maxNumMapEntries) { + throw new SerealException("Got input hash with " + hash.size() + " entries, but the configured maximum is just " + maxNumMapEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(hash, size); + } + appendByte(SerealHeader.SRL_HDR_HASH); + appendVarint(hash.size()); + + for (Map.Entry entry : hash.entrySet()) { + encode(entry.getKey().toString()); + encode(entry.getValue()); + } + } + + private void appendRef(PerlReference ref) throws SerealException { + Object refValue = ref.getValue(); + + appendByte(SerealHeader.SRL_HDR_REFN); + track(refValue, size); + encode(refValue); + } + + private void appendArray(Object obj) throws SerealException { + // checking length without casting to Object[] since they might primitives + int count = Array.getLength(obj); + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(obj, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + // write the objects (works for both Objects and primitives) + for (int index = 0; index < count; index++) { + encode(Array.get(obj, index)); + } + } + + private void appendArray(Object[] array) throws SerealException { + int count = array.length; + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(array, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + for (Object item : array) { + encode(item); + } + } + + private void appendArray(List list) throws SerealException { + int count = list.size(); + + if (maxNumArrayEntries != 0 && count > maxNumArrayEntries) { + throw new SerealException("Got input array with " + count + " entries, but the configured maximum is just " + maxNumArrayEntries); + } + + if (!perlRefs) { + appendByte(SerealHeader.SRL_HDR_REFN); + track(list, size); + } + appendByte(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + + for (Object item : list) { + encode(item); + } + } + + private void appendStringType(byte[] bytes) throws SerealException { + if (bytes.length < SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(bytes); + } else { + appendBinary(bytes); + } + } + + private void appendStringType(Latin1String str) throws SerealException { + byte[] latin1 = str.getBytes(); + if (str.length() < SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(latin1); + } else { + appendBinary(latin1); + } + } + + private void appendStringType(String str) throws SerealException { + if (maxStringLength != 0 && str.length() > maxStringLength) { + throw new SerealException("Got input string with " + str.length() + " characters, but the configured maximum is just " + maxStringLength); + } + + // maybe we can just COPY + long copyOffset = getTrackedItemCopy(str); + if (copyOffset != StringCopyMap.NOT_FOUND) { + appendCopy(copyOffset); + return; + } + + long location = size; + + byte[] utf8 = ((String) str).getBytes(charset_utf8); + appendByte(SerealHeader.SRL_HDR_STR_UTF8); + appendVarint(utf8.length); + appendBytes(utf8); + + trackForCopy(str, location); + } + + private void appendNumber(long l) { + if (l < 0) { + if (l > -17) { + appendByte((byte) (SerealHeader.SRL_HDR_NEG_LOW | (l + 32))); + } else { + appendZigZag(l); + } + } else { + if (l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + } + + private void appendBigInteger(BigInteger bi) throws SerealException { + int compareToZero = bi.compareTo(BigInteger.ZERO); + if (compareToZero < 0 && bi.compareTo(LONG_MIN_VALUE) >= 0) { + appendNumber(bi.longValue()); + } else if (compareToZero > 0 && bi.compareTo(UNSIGNED_LONG_MAX_VALUE) <= 0) { + if (bi.compareTo(LONG_MAX_VALUE) <= 0) { + appendNumber(bi.longValue()); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(bi.longValue()); + } + } else if (compareToZero == 0) { + appendNumber(0); + } else { + throw new SerealException("BigInteger value is outside representable range"); + } + } + + private boolean isDefinitelyReference(Object value) { + if (value instanceof Map || value instanceof List) { + return true; + } else if (value instanceof PerlReference) { + return true; + } + + return false; + } + + private boolean isRefTag(byte tag) { + // the first branch is the common case, the other two branchs are unlikely + if (tag == SerealHeader.SRL_HDR_REFN || + tag == SerealHeader.SRL_HDR_REFP) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + return true; + } + + return false; + } + + /** Discard all previous tracking clear the buffers etc Call this when you reuse the encoder */ + private void reset() { + size = compressedSize = headerSize = recursionDepth = 0; + resetTracked(); + } + + private void resetTracked() { + tracked.clear(); + trackedBytearrayCopy.clear(); + trackedStringCopy.clear(); + if (perlAlias) { + aliases.clear(); + maybeAliases.clear(); + } + trackedClassnames.clear(); + } + + private void ensureAvailable(int required) { + long total = required + size; + + if (total > bytes.length) bytes = Arrays.copyOf(bytes, (int) (total * 3 / 2)); + } + + private void appendBytes(byte[] data) { + ensureAvailable(data.length); + appendBytesUnsafe(data); + } + + private void appendBytes(byte[] data, int length) { + ensureAvailable(length); + appendBytesUnsafe(data, length); + } + + private void appendBytesUnsafe(byte[] data) { + System.arraycopy(data, 0, bytes, (int) size, data.length); + size += data.length; + } + + private void appendBytesUnsafe(byte[] data, int length) { + System.arraycopy(data, 0, bytes, (int) size, length); + size += length; + } + + private void appendByte(byte data) { + ensureAvailable(1); + appendByteUnsafe(data); + } + + private void appendByteUnsafe(byte data) { + bytes[(int) size] = data; + size++; + } + + private void depthIncrement() throws SerealException { + if (recursionDepth++ > maxRecursionDepth) { + throw new SerealException("Reached recursion limit (" + maxRecursionDepth + ") during serialization"); + } + } + + private void depthDecrement() { + recursionDepth--; + } + + /** + * Close the encoder to recycle the resources, it must be called by the client at the end of the + * use, otherwise, NullPointerException will be thrown when you reuse the ZLIB encoder. + *

+ * Returns the native memory used by deflater explicitly before the garbage collector. + */ + public void close() { + if (deflater != null) { + deflater.end(); + deflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/EncoderOptions.java b/src/main/java/com/booking/sereal/EncoderOptions.java new file mode 100644 index 0000000000..285d4008ff --- /dev/null +++ b/src/main/java/com/booking/sereal/EncoderOptions.java @@ -0,0 +1,203 @@ +package com.booking.sereal; + +public class EncoderOptions { + private boolean perlRefs = false; + private boolean perlAlias = false; + private int protocolVersion = 4; + private CompressionType compressionType = CompressionType.NONE; + private long compressionThreshold = 1024; + private int zlibCompressionLevel = 6; + private int zstdCompressionLevel = 3; + + private int maxRecursionDepth = 10_000; + private int maxNumMapEntries = 0; + private int maxNumArrayEntries = 0; + private int maxStringLength = 0; + + public boolean perlReferences() { + return perlRefs; + } + + public boolean perlAliases() { + return perlAlias; + } + + public int protocolVersion() { + return protocolVersion; + } + + public CompressionType compressionType() { + return compressionType; + } + + public long compressionThreshold() { + return compressionThreshold; + } + + public int zlibCompressionLevel() { + return zlibCompressionLevel; + } + + public int zstdCompressionLevel() { + return zstdCompressionLevel; + } + + /** + * {@link Encoder} is recursive. If you pass it an Object that is deeply + * nested, it will eventually exhaust the C stack. Therefore, there is a limit on + * the depth of recursion that is accepted. It defaults to 10000 nested calls. You + * may choose to override this value with the {@link EncoderOptions#maxRecursionDepth(int)} option. + * Beware that setting it too high can cause hard crashes. + * + * Do note that the setting is somewhat approximate. Setting it to 10000 may break at + * somewhere between 9997 and 10003 nested structures depending on their types. + * + * It's also important to note that this value doesn't mirror the Perl implementation. Perl and Java + * Sereal wrappers are implemented in a different way. That means that the recursion depth marked + * as max in Perl may be different than the one needed in Java. The same applies for the Encoder + * and Decoder options. + * + * The Java stack can be controlled with different flags (for example -Xss) to change it's size. + * + * @return maximum recursion depth + */ + public int maxRecursionDepth() { + return maxRecursionDepth; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to deserialize any hash/dictionary (or hash-based object) with more than + * that number of entries. This is to be able to respond quickly to any future + * hash-collision attacks on Perl's hash function. Chances are, you don't want + * or need this. For a gentle introduction to the topic from the cryptographic + * point of view, see Collision attack. + * + * This value can be override with {@link EncoderOptions#maxNumMapEntries(int)} option + * + * @return maximum number of map entries + */ + public int maxNumMapEntries() { + return maxNumMapEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to serialize any array with more than that number of entries. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link EncoderOptions#maxNumArrayEntries(int)} option + * + * @return maximum number of array entries + */ + public int maxNumArrayEntries() { + return maxNumArrayEntries; + } + + /** + * If set to a non-zero value (default: 0), then {@link Encoder} will refuse + * to deserialize any string with more than that number of characters. + * This is to be able to respond quickly to any future memory exhaustion attacks on + * Sereal. + * + * This value can be override with {@link EncoderOptions#maxStringLength(int)} option + * + * @return maximum supported string length + */ + public int maxStringLength() { + return maxStringLength; + } + + /** + * + * @param perlReferences {@code true} if perlReferences are supported, {@code false} otherwise. + * @return EncoderOptions + */ + public EncoderOptions perlReferences(boolean perlReferences) { + this.perlRefs = perlReferences; + + return this; + } + + public EncoderOptions perlAliases(boolean perlAliases) { + this.perlAlias = perlAliases; + + return this; + } + + public EncoderOptions protocolVersion(int protocolVersion) { + if (protocolVersion < 1 || protocolVersion > 4) { + throw new IllegalArgumentException("Unknown Sereal version " + protocolVersion); + } + this.protocolVersion = protocolVersion; + + return this; + } + + public EncoderOptions compressionType(CompressionType compressionType) { + if (protocolVersion < compressionType.minProtocolVersion) { + throw new IllegalArgumentException("Compression " + compressionType + " not supported in Sereal protocol" + protocolVersion); + } + this.compressionType = compressionType; + + return this; + } + + public EncoderOptions compressionThreshold(long compressionThreshold) { + this.compressionThreshold = compressionThreshold; + + return this; + } + + public EncoderOptions zlibCompressionLevel(int zlibCompressionLevel) { + this.zlibCompressionLevel = zlibCompressionLevel; + + return this; + } + + public EncoderOptions zstdCompressionLevel(int zstdCompressionLevel) { + this.zstdCompressionLevel = zstdCompressionLevel; + return this; + } + + + public EncoderOptions maxRecursionDepth(int maxRecursionDepth) { + this.maxRecursionDepth = maxRecursionDepth; + + return this; + } + + public EncoderOptions maxNumMapEntries(int maxNumMapEntries) { + this.maxNumMapEntries = maxNumMapEntries; + + return this; + } + + public EncoderOptions maxNumArrayEntries(int maxNumArrayEntries) { + this.maxNumArrayEntries = maxNumArrayEntries; + + return this; + } + + public EncoderOptions maxStringLength(int maxStringLength) { + this.maxStringLength = maxStringLength; + + return this; + } + + public enum CompressionType { + NONE(1, SerealHeader.SRL_ENCODING_NONE), + SNAPPY(1, -1), + ZLIB(3, SerealHeader.SRL_ENCODING_ZLIB), + ZSTD(4, SerealHeader.SRL_ENCODING_ZSTD); + + final byte minProtocolVersion; + final byte encoding; + + CompressionType(int minProtocolVersion, int encoding) { + this.minProtocolVersion = (byte) minProtocolVersion; + this.encoding = (byte) encoding; + } + } +} diff --git a/src/main/java/com/booking/sereal/Latin1String.java b/src/main/java/com/booking/sereal/Latin1String.java new file mode 100644 index 0000000000..85f3e4ee00 --- /dev/null +++ b/src/main/java/com/booking/sereal/Latin1String.java @@ -0,0 +1,75 @@ +package com.booking.sereal; + +import java.nio.charset.Charset; +import java.util.Arrays; + +public class Latin1String implements CharSequence { + private static final Charset charset_latin1 = Charset.forName("ISO-8859-1"); + private final byte[] bytes; + private boolean hashCodeSet = false; + private int hashcode; + + public Latin1String(String s) { + this.bytes = s.getBytes(charset_latin1); + this.hashCodeSet = false; + } + + public Latin1String(byte[] bytes) { + this.bytes = bytes; + this.hashCodeSet = false; + } + + @Override + public String toString() { + return new String(bytes, charset_latin1); + } + + @Override + public int hashCode() { + if (!hashCodeSet) { + hashcode = Arrays.hashCode(bytes); + } + return hashcode; + } + + @Override + public char charAt(int index) { + return (char) bytes[index]; + } + + @Override + public int length() { + return bytes.length; + } + + @Override + public CharSequence subSequence(int start, int end) { + return this.toString().subSequence(start, end); + } + + public byte[] getBytes() { + return this.bytes; + } + + public String getString() { + return this.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (this.hashCode() != o.hashCode()) { + return false; + } + if (!this.getClass().equals(o.getClass())) { + return false; + } + + return Arrays.equals(this.bytes, ((Latin1String) o).bytes); + } +} diff --git a/src/main/java/com/booking/sereal/PerlAlias.java b/src/main/java/com/booking/sereal/PerlAlias.java new file mode 100644 index 0000000000..f7dd45fccb --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlAlias.java @@ -0,0 +1,18 @@ +package com.booking.sereal; + +/** + * Since the Sereal format has the notion of both reference and alias we use this class to wrap + * those values. That way we can always roundtrip between Perl and Java. + */ +public class PerlAlias { + + private final Object value; + + public PerlAlias(Object value) { + this.value = value; + } + + public Object getValue() { + return value; + } +} diff --git a/src/main/java/com/booking/sereal/PerlObject.java b/src/main/java/com/booking/sereal/PerlObject.java new file mode 100644 index 0000000000..deb9f02903 --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlObject.java @@ -0,0 +1,40 @@ +package com.booking.sereal; + +import java.util.Map; + +/* + * Perl object which is defined by a name and either an array, hash or ref + * (in Perl all of these are basically refs) + * + * Also, this is very ugly + */ +public class PerlObject { + + private final Object data; + private String name; + + public PerlObject(String className, Object obj) { + this.name = className; + this.data = obj; + } + + public String getName() { + return name; + } + + public boolean isHash() { + return getData() instanceof Map; + } + + public boolean isArray() { + return getData().getClass().isArray(); + } + + public boolean isReference() { + return !isHash() && !isArray(); + } + + public Object getData() { + return data; + } +} diff --git a/src/main/java/com/booking/sereal/PerlReference.java b/src/main/java/com/booking/sereal/PerlReference.java new file mode 100644 index 0000000000..bc167896b1 --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlReference.java @@ -0,0 +1,27 @@ +package com.booking.sereal; + +/** + * So we can encode references perl style (otherwise there is no way to distinguish Strings from + * "Stringrefs" + */ +public class PerlReference { + + private Object value; + + public PerlReference(Object value) { + this.value = value; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + @Override + public String toString() { + return "Reference to: " + (value == null ? "null" : value.toString()); + } +} diff --git a/src/main/java/com/booking/sereal/PerlUndef.java b/src/main/java/com/booking/sereal/PerlUndef.java new file mode 100644 index 0000000000..836c5a6b66 --- /dev/null +++ b/src/main/java/com/booking/sereal/PerlUndef.java @@ -0,0 +1,11 @@ +package com.booking.sereal; + +/** So we can preserve the identity of different undefs */ +public final class PerlUndef { + public static final PerlUndef CANONICAL = new PerlUndef(); + + @Override + public String toString() { + return "Undef"; + } +} diff --git a/src/main/java/com/booking/sereal/SerealException.java b/src/main/java/com/booking/sereal/SerealException.java new file mode 100644 index 0000000000..60ba10c2b9 --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealException.java @@ -0,0 +1,12 @@ +package com.booking.sereal; + +@SuppressWarnings("serial") +public class SerealException extends Exception { + public SerealException(String msg) { + super(msg); + } + + public SerealException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/booking/sereal/SerealHeader.java b/src/main/java/com/booking/sereal/SerealHeader.java new file mode 100644 index 0000000000..afecdef58b --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealHeader.java @@ -0,0 +1,80 @@ +package com.booking.sereal; + +public interface SerealHeader { + + // 0x6c72733d but little endian for some reason + int MAGIC = (0x6c) + (0x72 << 8) + (0x73 << 16) + (0x3d << 24); + + // 0x6c72f33d but little endian for some reason + int MAGIC_V3 = (0x6c) + (0x72 << 8) + (0xf3 << 16) + (0x3d << 24); + + byte SRL_MASK_SHORT_BINARY_LEN = (byte) 31; // lower 5 bits + + byte SRL_ENCODING_NONE = 0; + byte SRL_ENCODING_SNAPPY_LEGACY = 1; + byte SRL_ENCODING_SNAPPY = 2; + byte SRL_ENCODING_ZLIB = 3; + byte SRL_ENCODING_ZSTD = 4; + + /* + Note: Despite this interface already being named SerealHeader we still use SRL_HDR_ + as a prefix so grepping will show both these and the C ones. + +=for autoupdater start + +* NOTE this section is autoupdated by Sereal.git:Perl/shared/author_tools/update_from_header.pl */ + byte SRL_HDR_POS = (byte) 0; /* 0 0x00 0b00000000 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_POS_LOW = (byte) 0; /* 0 0x00 0b00000000 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_POS_HIGH = (byte) 15; /* 15 0x0f 0b00001111 small positive integer - value in low 4 bits (identity) */ + byte SRL_HDR_NEG = (byte) 16; /* 16 0x10 0b00010000 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_NEG_LOW = (byte) 16; /* 16 0x10 0b00010000 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_NEG_HIGH = (byte) 31; /* 31 0x1f 0b00011111 small negative integer - value in low 4 bits (k+32) */ + byte SRL_HDR_VARINT = (byte) 32; /* 32 0x20 0b00100000 - Varint variable length integer */ + byte SRL_HDR_ZIGZAG = (byte) 33; /* 33 0x21 0b00100001 - Zigzag variable length integer */ + byte SRL_HDR_FLOAT = (byte) 34; /* 34 0x22 0b00100010 */ + byte SRL_HDR_DOUBLE = (byte) 35; /* 35 0x23 0b00100011 */ + byte SRL_HDR_LONG_DOUBLE = (byte) 36; /* 36 0x24 0b00100100 */ + byte SRL_HDR_UNDEF = (byte) 37; /* 37 0x25 0b00100101 None - Perl undef var; eg my $var= undef; */ + byte SRL_HDR_BINARY = (byte) 38; /* 38 0x26 0b00100110 - binary/(latin1) string */ + byte SRL_HDR_STR_UTF8 = (byte) 39; /* 39 0x27 0b00100111 - utf8 string */ + byte SRL_HDR_REFN = (byte) 40; /* 40 0x28 0b00101000 - ref to next item */ + byte SRL_HDR_REFP = (byte) 41; /* 41 0x29 0b00101001 - ref to previous item stored at offset */ + byte SRL_HDR_HASH = (byte) 42; /* 42 0x2a 0b00101010 [ ...] - count followed by key/value pairs */ + byte SRL_HDR_ARRAY = (byte) 43; /* 43 0x2b 0b00101011 [ ...] - count followed by items */ + byte SRL_HDR_OBJECT = (byte) 44; /* 44 0x2c 0b00101100 - class, object-item */ + byte SRL_HDR_OBJECTV = (byte) 45; /* 45 0x2d 0b00101101 - offset of previously used classname tag - object-item */ + byte SRL_HDR_ALIAS = (byte) 46; /* 46 0x2e 0b00101110 - alias to item defined at offset */ + byte SRL_HDR_COPY = (byte) 47; /* 47 0x2f 0b00101111 - copy of item defined at offset */ + byte SRL_HDR_WEAKEN = (byte) 48; /* 48 0x30 0b00110000 - Weaken the following reference */ + byte SRL_HDR_REGEXP = (byte) 49; /* 49 0x31 0b00110001 */ + byte SRL_HDR_OBJECT_FREEZE = (byte) 50; /* 50 0x32 0b00110010 - class, object-item. Need to call "THAW" method on class after decoding */ + byte SRL_HDR_OBJECTV_FREEZE = (byte) 51; /* 51 0x33 0b00110011 - (OBJECTV_FREEZE is to OBJECT_FREEZE as OBJECTV is to OBJECT) */ + byte SRL_HDR_NO = (byte) 52; /* 52 0x34 0b00110100 SvIsBOOL() == PL_No, 5.36 and later only (json false) */ + byte SRL_HDR_YES = (byte) 53; /* 53 0x35 0b00110101 SvIsBOOL() == PL_Yes, 5.36 and later only (json true) */ + byte SRL_HDR_RESERVED = (byte) 54; /* 54 0x36 0b00110110 */ + byte SRL_HDR_RESERVED_LOW = (byte) 54; /* 54 0x36 0b00110110 */ + byte SRL_HDR_RESERVED_HIGH = (byte) 55; /* 55 0x37 0b00110111 */ + byte SRL_HDR_FLOAT_128 = (byte) 56; /* 56 0x38 0b00111000 quadmath _float128 */ + byte SRL_HDR_CANONICAL_UNDEF = (byte) 57; /* 57 0x39 0b00111001 undef (PL_sv_undef) - "the" Perl undef (see notes) */ + byte SRL_HDR_FALSE = (byte) 58; /* 58 0x3a 0b00111010 false (PL_sv_no) */ + byte SRL_HDR_TRUE = (byte) 59; /* 59 0x3b 0b00111011 true (PL_sv_yes) */ + byte SRL_HDR_MANY = (byte) 60; /* 60 0x3c 0b00111100 - repeated tag (not done yet, will be implemented in version 3) */ + byte SRL_HDR_PACKET_START = (byte) 61; /* 61 0x3d 0b00111101 (first byte of magic string in header) */ + byte SRL_HDR_EXTEND = (byte) 62; /* 62 0x3e 0b00111110 - for additional tags */ + byte SRL_HDR_PAD = (byte) 63; /* 63 0x3f 0b00111111 (ignored tag, skip to next byte) */ + byte SRL_HDR_ARRAYREF = (byte) 64; /* 64 0x40 0b01000000 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_ARRAYREF_LOW = (byte) 64; /* 64 0x40 0b01000000 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_ARRAYREF_HIGH = (byte) 79; /* 79 0x4f 0b01001111 [ ...] - count of items in low 4 bits (ARRAY must be refcnt=1) */ + byte SRL_HDR_HASHREF = (byte) 80; /* 80 0x50 0b01010000 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_HASHREF_LOW = (byte) 80; /* 80 0x50 0b01010000 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_HASHREF_HIGH = (byte) 95; /* 95 0x5f 0b01011111 [ ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1) */ + byte SRL_HDR_SHORT_BINARY = (byte) 96; /* 96 0x60 0b01100000 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_SHORT_BINARY_LOW = (byte) 96; /* 96 0x60 0b01100000 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_SHORT_BINARY_HIGH = (byte) 127; /* 127 0x7f 0b01111111 - binary/latin1 string, length encoded in low 5 bits of tag */ + byte SRL_HDR_TRACK_FLAG = (byte) 128; /* 128 0x80 0b10000000 if this bit is set track the item */ +/* +* NOTE the above section is auto-updated by Sereal.git:Perl/shared/author_tools/update_from_header.pl + +=for autoupdater stop + */ +} diff --git a/src/main/java/com/booking/sereal/SerealToken.java b/src/main/java/com/booking/sereal/SerealToken.java new file mode 100644 index 0000000000..466cd7a6de --- /dev/null +++ b/src/main/java/com/booking/sereal/SerealToken.java @@ -0,0 +1,138 @@ +package com.booking.sereal; + +/** + * Enumeration for Sereal token types, used by {@link com.booking.sereal.TokenDecoder}. + */ +public enum SerealToken { + /** Returned when there is no token. */ + NONE, + + /** + * An integer value up to 64 bits. + */ + LONG, + + /** + * A positive integer value larger than {@link java.lang.Long#MAX_VALUE}. + *

+ * Java does not have unsigned values, so the value is returned as a signed (negative) {@code long} + * with the same bit pattern as the unsigned value. + */ + UNSIGNED_LONG, + + /** + * A binary/ISO-8859-1 string. + */ + BINARY, + + /** + * An array value. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + ARRAY_START, + + /** + * Returned after the last element of an array. + */ + ARRAY_END, + + /** + * An hash value. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + HASH_START, + + /** + * Returned after the last element of an hash. + */ + HASH_END, + + /** + * Returned after the last element of the Sereal document. + */ + END, + + /** + * A 32-bit IEEE-754 floating point number. + */ + FLOAT, + + /** + * A 64-bit IEEE-754 floating point number. + */ + DOUBLE, + + /** + * Canonical {@code true} value. + */ + TRUE, + + /** + * Canonical {@code false} value. + */ + FALSE, + + /** + * An {@code undef} value. + */ + UNDEF, + + /** + * An UTF-8 encoded string. + */ + UTF8, + + /** + * A Perl reference, pointing to the object following the {@code REFN} token. + */ + REFN, + + /** + * A Perl reference, pointing to an object already encountered at a previous offset. + */ + REFP, + + /** + * A Perl object. + *

+ * This token will always by preceded by a {@code REFN} token. + */ + OBJECT_START, + + /** + * Returned after the end of a Perl object. + */ + OBJECT_END, + + /** + * Sereal copy tag, referring to a previous value. + */ + COPY, + + /** + * Sereal alias tag, referring to a previous value. + */ + ALIAS, + + /** + * Marks the following reference as a weak reference. + */ + WEAKEN, + + /** + * Regular expression value. + */ + REGEXP, + + /** + * Canonical {@code undef} value. + */ + CANONICAL_UNDEF, + + /** + * Returned after the end of a sub-decode. + */ + SUBDECODE_END; +} diff --git a/src/main/java/com/booking/sereal/TokenDecoder.java b/src/main/java/com/booking/sereal/TokenDecoder.java new file mode 100644 index 0000000000..aef8bd412e --- /dev/null +++ b/src/main/java/com/booking/sereal/TokenDecoder.java @@ -0,0 +1,923 @@ +package com.booking.sereal; + +import com.github.luben.zstd.Zstd; +import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import org.xerial.snappy.Snappy; + +/** + * A low-level stream decoder for Sereal. + *

+ * It provides a token stream, with accessor to retrieve token data (e.g. the integer value associated with a {@code LONG} + * token). + *

+ * The decoder performs some basic sanity checks, but it does not perform higher-level validation, such as checking + * that hash keys are strings, array values are scalars, or weaken tokens are followed by a reference. + *

+ * Example: + * 

+ * {@code
+ *   decoder.setData(bytes);
+ *   decoder.prepareDecodeBody();
+ *
+ *   while (decoder.nextToken() != SerealToken.END) {
+ *     if (decoder.nextToken() == SerealToken.LONG) {
+ *       System.outprintln("Long value " + decoder.longValue() + " at offset " + decoder.tokenOffset());
+ *     }
+ *   }
+ * }
+ *
+ */ +public class TokenDecoder { + private static class Context { + private final Context outer; + private final int type; + private int total, remaining; + private int originalPosition; + + Context(Context outer, int type, int remaining) { + this.outer = outer; + this.type = type; + this.remaining = this.total = remaining; + } + + Context(Context outer, int type, int remaining, int originalPosition) { + this.outer = outer; + this.type = type; + this.remaining = this.total = remaining; + this.originalPosition = originalPosition; + } + } + + private static class AllowedEncoding { + private final EncoderOptions.CompressionType compressionType; + private final int encoding, minVersion, maxVersion; + + AllowedEncoding(EncoderOptions.CompressionType compressionType, int encoding, int minVersion, int maxVersion) { + this.compressionType = compressionType; + this.encoding = encoding; + this.minVersion = minVersion; + this.maxVersion = maxVersion; + } + } + + private static final DecoderOptions DEFAULT_OPTIONS = new DecoderOptions(); + private static final AllowedEncoding[] ALLOWED_ENCODINGS = new AllowedEncoding[] { + new AllowedEncoding(EncoderOptions.CompressionType.NONE, 0, 1, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.SNAPPY, 1, 1, 1), + new AllowedEncoding(EncoderOptions.CompressionType.SNAPPY, 2, 2, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.ZLIB, 3, 3, Integer.MAX_VALUE), + new AllowedEncoding(EncoderOptions.CompressionType.ZSTD, 4, 4, Integer.MAX_VALUE), + }; + + private static final int CONTEXT_ROOT = 0; + private static final int CONTEXT_HASH = 1; + private static final int CONTEXT_ARRAY = 2; + private static final int CONTEXT_OBJECT = 3; + private static final int CONTEXT_SUBDECODE = 4; + + private byte[] data; + private int position, end; + private byte[] bodyData; + private int bodyPosition, bodySize; + private ByteArray originalData; + private int protocolVersion = -1; + private int encoding = -1; + private int baseOffset = Integer.MAX_VALUE; + private int userHeaderPosition = -1; + private int userHeaderSize = -1; + private Inflater inflater; + private byte[] bigintBuffer; + + private Context currentContext; + private SerealToken currentToken = SerealToken.NONE; + private boolean isSecondTime; + private int trackOffset, tokenOffset; + private long longValue; + private float floatValue; + private double doubleValue; + private int binarySliceStart, binarySliceEnd; + private boolean binaryIsUtf8; + private int backreferenceOffset; + + /** Create a new {@code TokenDecoder} with default options. */ + public TokenDecoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create an new {@code TokenDecoder} with the specified options. + * + * @param options {@link DecoderOptions} to use + */ + public TokenDecoder(DecoderOptions options) { + } + + private void checkHeaderSuffix() { + int suffixSize = (int) readVarint(); + + userHeaderSize = 0; + userHeaderPosition = position + suffixSize; + if (protocolVersion >= 2 && suffixSize > 0) { + byte bitfield = data[position++]; + + if ((bitfield & 0x01) == 0x01) { + userHeaderPosition = position; + userHeaderSize = suffixSize - 1; + } + } + } + + private void checkNoEOD() throws SerealException { + if ((end - position) <= 0) { + throw new SerealException("Unexpected end of data at byte " + position); + } + } + + private void checkMagicProtoAndFlags() throws SerealException { + int magic = + ((data[position] & 0xff) << 24) + + ((data[position + 1] & 0xff) << 16) + + ((data[position + 2] & 0xff) << 8) + + (data[position + 3] & 0xff); + protocolVersion = data[position + 4] & 15; // 4 bits for version + if (magic != SerealHeader.MAGIC && magic != SerealHeader.MAGIC_V3) { + throw new SerealException( + String.format("Invalid Sereal header (%08x): doesn't match magic", magic)); + } else if (protocolVersion < 1 || protocolVersion > 4) { + throw new SerealException( + String.format("Invalid Sereal header: unsupported protocol version %d", protocolVersion)); + } else if (magic == SerealHeader.MAGIC && protocolVersion > 2) { + throw new SerealException( + String.format("Invalid Sereal header: magic v1 with protocol version %d", protocolVersion)); + } else if (magic == SerealHeader.MAGIC_V3 && protocolVersion < 3) { + throw new SerealException( + String.format("Invalid Sereal header: magic v3 with protocol version %d", protocolVersion)); + } + + encoding = (data[position + 4] >> 4) & 0xf; + if (encoding > 4) { + throw new SerealException("Unsupported Sereal body encoding " + encoding); + } + for (AllowedEncoding allowedEncoding : ALLOWED_ENCODINGS) { + if (allowedEncoding.encoding == encoding) { + if (protocolVersion < allowedEncoding.minVersion || protocolVersion > allowedEncoding.maxVersion) { + throw new SerealException(String.format("Unsupported encoding %d (%s) for Sereal protocol %d", + encoding, allowedEncoding.compressionType, protocolVersion)); + } + } + } + position += 5; + } + + /** + * @return {@code true} if the document has a Sereal header. + * + * @throws SerealException header cannot be parsed. + */ + public boolean hasHeader() throws SerealException { + parseHeader(); + + return userHeaderSize > 0; + } + + /** + * @return Size of the Sereal header, if present, 0 otherwise. + * + * @throws SerealException header cannot be parsed. + */ + public int headerSize() throws SerealException { + parseHeader(); + + return userHeaderSize > 0 ? userHeaderSize : 0; + } + + /** + * Set up the decoder to iterate over the Sereal header. + *

+ * This function will fail if {@link TokenDecoder#hasHeader()} returned {@code false}. + * + * @throws SerealException Token cannot be decoded. + */ + public void prepareDecodeHeader() throws SerealException { + parseHeader(); + + if (protocolVersion == 1) { + throw new SerealException("Sereal user header not supported in protocol version 1"); + } + if (userHeaderSize <= 0) { + throw new SerealException("Sereal user header not present"); + } + + data = originalData.array; + position = userHeaderPosition; + end = userHeaderPosition + userHeaderSize; + + // because offsets start at 1 + baseOffset = position - 1; + + currentContext = new Context(null, CONTEXT_ROOT, 1); + } + + private void parseHeader() throws SerealException { + if (originalData == null) { + throw new SerealException("No data set"); + } + if (userHeaderSize >= 0) { + return; + } + if (originalData.length < 7) { + throw new SerealException("Invalid Sereal document: total size is too small"); + } + + data = originalData.array; + end = originalData.start + originalData.length; + position = originalData.start; + + checkMagicProtoAndFlags(); + checkHeaderSuffix(); + } + + /** + * Set up the decoder to iterate over the Sereal document body. + * + * @throws SerealException Token cannot be decoded. + */ + public void prepareDecodeBody() throws SerealException { + parseHeader(); + + if (bodyData == null) { + if (encoding != 0) { + position = userHeaderPosition + userHeaderSize; + end = originalData.length; + if (encoding == SerealHeader.SRL_ENCODING_SNAPPY_LEGACY || encoding == SerealHeader.SRL_ENCODING_SNAPPY) { + uncompressSnappy(); + } else if (encoding == SerealHeader.SRL_ENCODING_ZLIB) { + uncompressZlib(); + } else if (encoding == SerealHeader.SRL_ENCODING_ZSTD) { + uncompressZstd(); + } + } else { + bodyData = data; + bodyPosition = userHeaderPosition + userHeaderSize; + bodySize = originalData.start + originalData.length; + } + } + + data = bodyData; + position = bodyPosition; + end = bodySize; + + if (protocolVersion == 1) { + baseOffset = 0; + } else { + // because offsets start at 1 + baseOffset = position - 1; + } + + currentContext = new Context(null, CONTEXT_ROOT, 1); + } + + private void uncompressSnappy() throws SerealException { + int len = originalData.length - (position - originalData.start); + int pos = encoding == SerealHeader.SRL_ENCODING_SNAPPY_LEGACY ? position : originalData.start; + + if (encoding == SerealHeader.SRL_ENCODING_SNAPPY) { + len = (int) readVarint(); + } + byte[] uncompressed; + try { + if (!Snappy.isValidCompressedBuffer( + originalData.array, position, len)) + throw new SerealException("Invalid snappy data"); + uncompressed = + new byte + [pos + + Snappy.uncompressedLength( + originalData.array, position, len)]; + Snappy.uncompress( + originalData.array, position, len, uncompressed, pos); + } catch (IOException e) { + throw new SerealException(e); + } + this.bodyData = uncompressed; + this.bodyPosition = pos; + this.bodySize = uncompressed.length; + } + + private void uncompressZlib() throws SerealException { + if (inflater == null) { + inflater = new Inflater(); + } + inflater.reset(); + + long uncompressedLength = readVarint(); + long compressedLength = readVarint(); + inflater.setInput(originalData.array, position, (int) compressedLength); + byte[] uncompressed = new byte[(int) uncompressedLength]; + try { + int inflatedSize = inflater.inflate(uncompressed); + if (inflatedSize != uncompressedLength || !inflater.finished()) { + throw new SerealException("Error in zlib-compressed data"); + } + } catch (DataFormatException e) { + throw new SerealException(e); + } + this.bodyData = uncompressed; + this.bodyPosition = 0; + this.bodySize = uncompressed.length; + } + + private void uncompressZstd() throws SerealException { + int len = (int) readVarint(); + + byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len); + long decompressedSize = Zstd.decompressedSize(compressedData); + if (decompressedSize > Integer.MAX_VALUE) { + throw new SerealException("Decompressed size exceeds integer MAX_VALUE: " + decompressedSize); + } + + byte[] uncompressed = new byte[(int) decompressedSize]; + long status = Zstd.decompress(uncompressed, compressedData); + if (Zstd.isError(status)) { + throw new SerealException(Zstd.getErrorName(status)); + } + this.bodyData = uncompressed; + this.bodyPosition = 0; + this.bodySize = uncompressed.length; + } + + private void readBinary() { + int length = (int) readVarint(); + binaryIsUtf8 = false; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + // top bit set (0x80) means next byte is 7 bits more more varint + private long readVarint() { + long uv = 0; + int lshift = 0; + + byte b = data[position++]; + while ((position < end) && (b < 0)) { + uv |= ((long) b & 127) << lshift; // add 7 bits + lshift += 7; + b = data[position++]; + } + uv |= (long) b << lshift; // add final (or first if there is only 1) + + return uv; + } + + /** + * Iterate over the Sereal document returning the next token. + *

+ * Depending on the token returned, accessors can be used to retrieve additional + * information about the token. + * + * @return The next token + * + * @throws SerealException Token cannot be decoded. + */ + public SerealToken nextToken() throws SerealException { + if (currentContext.remaining == 0) { + Context popped = currentContext; + currentContext = popped.outer; + switch (popped.type) { + case CONTEXT_ROOT: + return (currentToken = SerealToken.END); + case CONTEXT_ARRAY: + return (currentToken = SerealToken.ARRAY_END); + case CONTEXT_HASH: + return (currentToken = SerealToken.HASH_END); + case CONTEXT_OBJECT: + return (currentToken = SerealToken.OBJECT_END); + case CONTEXT_SUBDECODE: + position = popped.originalPosition; + return (currentToken = SerealToken.SUBDECODE_END); + } + } + + checkNoEOD(); + + byte tag = data[position++]; + + tokenOffset = position - 1 - baseOffset; + trackOffset = 0; + if ((tag & SerealHeader.SRL_HDR_TRACK_FLAG) != 0) { + tag = (byte) (tag & ~SerealHeader.SRL_HDR_TRACK_FLAG); + trackOffset = tokenOffset; + } + + currentContext.remaining--; + + if (tag <= SerealHeader.SRL_HDR_POS_HIGH) { + longValue = (long) tag; + return (currentToken = SerealToken.LONG); + } else if (tag <= SerealHeader.SRL_HDR_NEG_HIGH) { + longValue = (long) (tag - 32); + return (currentToken = SerealToken.LONG); + } else if ((tag & SerealHeader.SRL_HDR_SHORT_BINARY_LOW) == SerealHeader.SRL_HDR_SHORT_BINARY_LOW) { + readShortBinary(tag); + return (currentToken = SerealToken.BINARY); + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + if (isSecondTime) { + isSecondTime = false; + trackOffset = 0; + int numKeys = tag & 0xf; + currentContext = new Context(currentContext, CONTEXT_HASH, numKeys * 2); + return (currentToken = SerealToken.HASH_START); + } else { + isSecondTime = true; + --position; + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + } + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + if (isSecondTime) { + isSecondTime = false; + trackOffset = 0; + int length = tag & 0xf; + currentContext = new Context(currentContext, CONTEXT_ARRAY, length); + return (currentToken = SerealToken.ARRAY_START); + } else { + isSecondTime = true; + --position; + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + } + } else { + switch (tag) { + case SerealHeader.SRL_HDR_VARINT: + longValue = readVarint(); + return (currentToken = (longValue >= 0 ? SerealToken.LONG : SerealToken.UNSIGNED_LONG)); + case SerealHeader.SRL_HDR_ZIGZAG: + longValue = readZigzag(); + return (currentToken = SerealToken.LONG); + case SerealHeader.SRL_HDR_FLOAT: + int floatBits = + ((data[position + 3] & 0xff) << 24) + + ((data[position + 2] & 0xff) << 16) + + ((data[position + 1] & 0xff) << 8) + + (data[position] & 0xff); + position += 4; + floatValue = Float.intBitsToFloat(floatBits); + return (currentToken = SerealToken.FLOAT); + case SerealHeader.SRL_HDR_DOUBLE: + long doubleBits = + ((long) (data[position + 7] & 0xff) << 56) + + ((long) (data[position + 6] & 0xff) << 48) + + ((long) (data[position + 5] & 0xff) << 40) + + ((long) (data[position + 4] & 0xff) << 32) + + ((long) (data[position + 3] & 0xff) << 24) + + ((long) (data[position + 2] & 0xff) << 16) + + ((long) (data[position + 1] & 0xff) << 8) + + ((long) (data[position] & 0xff)); + position += 8; + doubleValue = Double.longBitsToDouble(doubleBits); + return (currentToken = SerealToken.DOUBLE); + case SerealHeader.SRL_HDR_TRUE: + return (currentToken = SerealToken.TRUE); + case SerealHeader.SRL_HDR_FALSE: + return (currentToken = SerealToken.FALSE); + case SerealHeader.SRL_HDR_UNDEF: + return (currentToken = SerealToken.UNDEF); + case SerealHeader.SRL_HDR_CANONICAL_UNDEF: + return (currentToken = SerealToken.CANONICAL_UNDEF); + case SerealHeader.SRL_HDR_BINARY: + readBinary(); + return (currentToken = SerealToken.BINARY); + case SerealHeader.SRL_HDR_STR_UTF8: + readUTF8(); + return (currentToken = SerealToken.UTF8); + case SerealHeader.SRL_HDR_REFN: + currentContext.remaining++; + return (currentToken = SerealToken.REFN); + case SerealHeader.SRL_HDR_REFP: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.REFP); + case SerealHeader.SRL_HDR_OBJECT: + currentContext = new Context(currentContext, CONTEXT_OBJECT, 1); + readString(); + return (currentToken = SerealToken.OBJECT_START); + case SerealHeader.SRL_HDR_OBJECTV: + currentContext = new Context(currentContext, CONTEXT_OBJECT, 1); + readStringCopy(); + return (currentToken = SerealToken.OBJECT_START); + case SerealHeader.SRL_HDR_COPY: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.COPY); + case SerealHeader.SRL_HDR_ALIAS: + backreferenceOffset = (int) readVarint(); + return (currentToken = SerealToken.ALIAS); + case SerealHeader.SRL_HDR_WEAKEN: + currentContext.remaining++; + return (currentToken = SerealToken.WEAKEN); + case SerealHeader.SRL_HDR_HASH: + int numKeys = (int) readVarint(); + currentContext = new Context(currentContext, CONTEXT_HASH, numKeys * 2); + return (currentToken = SerealToken.HASH_START); + case SerealHeader.SRL_HDR_ARRAY: + int length = (int) readVarint(); + currentContext = new Context(currentContext, CONTEXT_ARRAY, length); + return (currentToken = SerealToken.ARRAY_START); + case SerealHeader.SRL_HDR_REGEXP: + readRegexp(); + return (currentToken = SerealToken.REGEXP); + case SerealHeader.SRL_HDR_PAD: + currentContext.remaining++; + return nextToken(); + default: + throw new SerealException("Tag not supported: " + tag); + } + } + } + + /** + * Starts a nested decoding context at the specified position. + *

+ * Position must point at the start of a Sereal tag. + *

+ * After the tag has been completely parsed (including any nested values), {@link #nextToken} + * returns {@link SerealToken#SUBDECODE_END}, and the decoder position is restored to the + * one before this call. + * + * @param offset Sereal document offset. + * + * @throws SerealException Token cannot be decoded. + */ + public void startSubDecode(int offset) throws SerealException { + currentContext = new Context(currentContext, CONTEXT_SUBDECODE, 1, position); + position = offset + baseOffset; + } + + /** + * @return After a call to {@link #nextToken}, return the current token. + */ + public SerealToken currentToken() { + return currentToken; + } + + /** + * @return The element count of the current parsing context. + *

+ * Between {@link SerealToken#ARRAY_START} and {@link SerealToken#ARRAY_END} returns the number of elements in the array. + *

+ * Between {@link SerealToken#HASH_START} and {@link SerealToken#HASH_END} returns the number of + * keys + values in the hash (always a multiple of 2). + *

+ * Between {@link SerealToken#OBJECT_START} and {@link SerealToken#OBJECT_END} returns 1. + *

+ * At the top-level of a sub-parse, returns 1. + */ + public int elementCount() { + return currentContext.total; + } + + /** + * @return Whether the offset of the current tag is the target of a reference. + *

+ * When it returns a non-zero value, it is the offset of a Sereal tag that is going to be references + * by a later {@link SerealToken#ALIAS} or {@link SerealToken#REFP} token. + */ + public int trackOffset() { + return trackOffset; + } + + /** + * The offset of the last token returned by {@link TokenDecoder#nextToken()}. + *

+ * Defined for all tokens except for {@link SerealToken#ARRAY_END}, {@link SerealToken#HASH_END} + * and {@link SerealToken#OBJECT_END}. + * + * @return a token offset that can be used for {@link #startSubDecode(int)} + */ + public int tokenOffset() { + return tokenOffset; + } + + /** + * @return Decoder position in the Sereal document. + *

+ * Defined for all tokens. + */ + public int currentOffset() { + return position - 1 - baseOffset; + } + + /** + * @return Current {@code long} value. + *

+ * Defined for {@link SerealToken#LONG} and {@link SerealToken#UNSIGNED_LONG}. + */ + public long longValue() { + return longValue; + } + + /** + * @return Current {@code long} value as a {@link java.math.BigInteger}. + *

+ * Defined for {@link SerealToken#LONG} and {@link SerealToken#UNSIGNED_LONG}. + */ + public BigInteger bigintValue() { + if (currentToken == SerealToken.UNSIGNED_LONG) { + if (bigintBuffer == null) { + bigintBuffer = new byte[8]; + } + long temp = longValue; + for (int i = 7; i >= 0; --i) { + bigintBuffer[i] = (byte) (temp & 0xff); + temp >>= 8; + } + return new BigInteger(1, bigintBuffer); + } else { + return BigInteger.valueOf(longValue); + } + } + + /** + * @return Current {@code float} value. + *

+ * Defined for {@link SerealToken#FLOAT}. + */ + public float floatValue() { + return floatValue; + } + + /** + * @return Current {@code double} value. + *

+ * Defined for {@link SerealToken#DOUBLE}. + */ + public double doubleValue() { + return doubleValue; + } + + /** + * All binary/string values are slices of this array. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + * + * @return Token binary data + */ + public byte[] decoderBuffer() { + return data; + } + + /** + * @return Start offset of the current binary/string in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceStart() { + return binarySliceStart; + } + + /** + * @return End offset of the current binary/string in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceEnd() { + return binarySliceEnd; + } + + /** + * @return Length of the current binary/string. + *

+ * Defined for {@link SerealToken#UTF8}, {@link SerealToken#BINARY}, {@link SerealToken#OBJECT_START} and {@link SerealToken#REGEXP}. + */ + public int binarySliceLength() { + return binarySliceEnd - binarySliceStart; + } + + /** + * @return Start offset of the current regexp flags in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceStart() { + return (int) longValue; + } + + /** + * @return End offset of the current regexp flags in {@link TokenDecoder#decoderBuffer()}. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceEnd() { + return backreferenceOffset; + } + + /** + * @return Length of the regexp flags string. + *

+ * Defined for {@link SerealToken#REGEXP}. + */ + public int regexpFlagsSliceLength() { + return regexpFlagsSliceEnd() - regexpFlagsSliceStart(); + } + + /** + * @return {@code true} if the byte slice between {@link TokenDecoder#binarySliceStart()}/{@link TokenDecoder#binarySliceEnd()} is UTF-8 encoded. + *

+ * Always {@code true} for {@link SerealToken#UTF8}, always {@code false} for {@link SerealToken#BINARY}. + */ + public boolean binaryIsUtf8() { + return binaryIsUtf8; + } + + /** + * @return Offset into the Sereal document. + *

+ * Defined for {@link SerealToken#REFP}, {@link SerealToken#ALIAS} and {@link SerealToken#COPY} token. + */ + public int backreferenceOffset() { + return backreferenceOffset; + } + + /** @return {@code true} if the current token is a hash key, {@code false} otherwise. */ + public boolean isHashKey() { + Context context = currentContext; + while (context.type == CONTEXT_SUBDECODE) { + context = context.outer; + } + return context.type == CONTEXT_HASH && (context.remaining & 1) == 1; + } + + private void readShortBinary(byte tag) { + int length = tag & SerealHeader.SRL_MASK_SHORT_BINARY_LEN; + binaryIsUtf8 = false; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + private void readStringCopy() throws SerealException { + int originalPosition = (int) readVarint(); + int currentPosition = position; // remember where we parked + + position = originalPosition + baseOffset; + readString(); + position = currentPosition; // go back to where we were + } + + private void readUTF8() { + int length = (int) readVarint(); + binaryIsUtf8 = true; + binarySliceStart = position; + binarySliceEnd = position + length; + position += length; + } + + private long readZigzag() { + long n = readVarint(); + + return (n >>> 1) ^ (-(n & 1)); // note the unsigned right shift + } + + private void readRegexp() throws SerealException { + readString(); + + boolean savedBinaryIfUtf8 = binaryIsUtf8; + int savedBinarySliceStart = binarySliceStart, savedBinarySliceEnd = binarySliceEnd; + + readString(); + + longValue = binarySliceStart; + backreferenceOffset = binarySliceEnd; + + binaryIsUtf8 = savedBinaryIfUtf8; + binarySliceStart = savedBinarySliceStart; + binarySliceEnd = savedBinarySliceEnd; + } + + private void readString() throws SerealException { + checkNoEOD(); + + byte tag = data[position++]; + + if ((tag & SerealHeader.SRL_HDR_SHORT_BINARY_LOW) == SerealHeader.SRL_HDR_SHORT_BINARY_LOW) { + readShortBinary(tag); + } else if (tag == SerealHeader.SRL_HDR_BINARY) { + readBinary(); + } else if (tag == SerealHeader.SRL_HDR_STR_UTF8) { + readUTF8(); + } else if (tag == SerealHeader.SRL_HDR_COPY) { + readStringCopy(); + } else { + throw new SerealException("Tag " + tag + " is not a string tag"); + } + } + + /** + * Set the Sereal data to be decoded. + *

+ * After calling this method, blob is owned by the decoder until the next call to {@code setData} or {@link #reset()}. + * + * @param blob Sereal data. + */ + public void setData(ByteArray blob) { + reset(); + originalData = blob; + } + + /** + * Set the Sereal data to be decoded. + *

+ * After calling this method, blob is owned by the decoder until the next call to {@code setData} or {@link #reset()}. + * + * @param blob Sereal data. + */ + public void setData(byte[] blob) { + setData(new ByteArray(blob)); + } + + /** Discard all internal state. */ + public void reset() { + currentToken = SerealToken.NONE; + currentContext = null; + originalData = null; + data = bodyData = null; + protocolVersion = encoding = -1; + baseOffset = Integer.MAX_VALUE; + userHeaderPosition = userHeaderSize = -1; + isSecondTime = false; + } + + /** + * Decode a string of bytes as Perl regexp modifiers, and return the corresponding {@link java.util.regex.Pattern} flags. + *

+ * It only handles {@code m, s, i, x} flags. + * + * @param buffer Buffer containing the modifier characters. + * @param start Start of the range containing the modifier characters. + * @param end Start of the range containing the modifier characters. + * @return The decoded modifiers as {@link java.util.regex.Pattern} flags + * @throws SerealException on an unrecognized modifier + */ + public static int decodeRegexpFlags(byte[] buffer, int start, int end) throws SerealException { + int flags = 0; + + for (int i = start; i < end; ++i) { + byte value = buffer[i]; + switch (value) { + case 'm': + flags = flags | Pattern.MULTILINE; + break; + case 's': + flags = flags | Pattern.DOTALL; + break; + case 'i': + flags = flags | Pattern.CASE_INSENSITIVE; + break; + case 'x': + flags = flags | Pattern.COMMENTS; + break; + case 'p': + // ignored + break; + default: + throw new SerealException(String.format("Unknown regexp modifier: '%c'", value)); + } + } + + return flags; + } + + // used for testing + SerealToken poisonNextToken() throws SerealException { + poisonFields(); + return nextToken(); + } + + private void poisonFields() { + trackOffset = (int) (Math.random() * Integer.MAX_VALUE); + tokenOffset = (int) (Math.random() * Integer.MAX_VALUE); + longValue = (long) (Math.random() * Integer.MAX_VALUE); + floatValue = (float) (Math.random() * Float.MAX_VALUE); + doubleValue = Math.random() * Double.MAX_VALUE; + binarySliceStart = (int) (Math.random() * Integer.MAX_VALUE); + binarySliceEnd = (int) (Math.random() * Integer.MAX_VALUE);; + binaryIsUtf8 = Math.random() > 0.5; + backreferenceOffset = (int) (Math.random() * Integer.MAX_VALUE);; + } + + /** + * Close the decoder to recycle the resources. + *

+ * Returns the native memory used by inflater explicitly before the garbage collector. + */ + public void close() { + if (inflater != null) { + inflater.end(); + inflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/TokenEncoder.java b/src/main/java/com/booking/sereal/TokenEncoder.java new file mode 100644 index 0000000000..f323fe861b --- /dev/null +++ b/src/main/java/com/booking/sereal/TokenEncoder.java @@ -0,0 +1,1261 @@ +package com.booking.sereal; + +import com.booking.sereal.EncoderOptions.CompressionType; +import com.github.luben.zstd.Zstd; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.regex.Pattern; +import java.util.zip.Deflater; +import org.xerial.snappy.Snappy; + +/** + * A low-level stream encoder for Sereal. + *

+ * Using this class correctly requires understanding of the Sereal specification and how the Perl encoder + * handles the produced Sereal. + *

+ * Example: + * 

+ * {@code
+ *   encoder.startDocument();
+ *
+ *   encoder.startArray(2);
+ *   encoder.appendString("Hello, Sereal");
+ *   encoder.appendLong(4);
+ *   encoder.endArray();
+ *
+ *   encoder.endDocument();
+ *
+ *   byte[] data = encoder.getData();
+ * }
+ *
+ */ +public class TokenEncoder { + private static class Context { + private final Context outer; + private final int type; + private final int expectedCount; + private final int position; + private int count; + + Context(Context outer, int type, int position, int expectedCount) { + this.outer = outer; + this.type = type; + this.position = position; + this.expectedCount = expectedCount; + } + } + + private static final int CONTEXT_ROOT = 0; + private static final int CONTEXT_HASH = 1; + private static final int CONTEXT_ARRAY = 2; + private static final int CONTEXT_OBJECT = 3; + private static final int CONTEXT_WEAKEN = 4; + private static final int CONTEXT_INITIAL = 5; + private static final int CONTEXT_HEADER = 6; + private static final int CONTEXT_FINAL = 7; + + private static final EncoderOptions DEFAULT_OPTIONS = new EncoderOptions(); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private final int MAX_VARINT_LENGTH = 10; + private final byte[] HEADER = + new byte[] { + (byte) (SerealHeader.MAGIC >> 24), + (byte) (SerealHeader.MAGIC >> 16), + (byte) (SerealHeader.MAGIC >> 8), + (byte) (SerealHeader.MAGIC), + }; + private final byte[] HEADER_V3 = + new byte[] { + (byte) (SerealHeader.MAGIC_V3 >> 24), + (byte) (SerealHeader.MAGIC_V3 >> 16), + (byte) (SerealHeader.MAGIC_V3 >> 8), + (byte) (SerealHeader.MAGIC_V3), + }; + + private final byte protocolVersion, encoding; + private final CompressionType compressionType; + private final CharsetEncoder utf8Encoder = StandardCharsets.UTF_8.newEncoder(); + private Deflater deflater; + private final int compressionThreshold; + private Context currentContext; + private byte[] bytes = new byte[1024], compressedBytes = EMPTY_BYTE_ARRAY; + private int size = 0, compressedSize = 0, trackOffset = 0; + private int headerSize, headerOffset; + private final int zstdCompressionLevel; + private boolean hasHeader; + + /** Create an new {@code TokenEncoder} with default options. */ + public TokenEncoder() { + this(DEFAULT_OPTIONS); + } + + /** + * Create an new {@code TokenEncoder} with the specified options. + * + * @param options {@link EncoderOptions} to use. + */ + public TokenEncoder(EncoderOptions options) { + protocolVersion = (byte) options.protocolVersion(); + compressionType = options.compressionType(); + zstdCompressionLevel = options.zstdCompressionLevel(); + compressionThreshold = (int) options.compressionThreshold(); + + if (compressionType == CompressionType.SNAPPY) { + encoding = protocolVersion == 1 ? SerealHeader.SRL_ENCODING_SNAPPY_LEGACY : SerealHeader.SRL_ENCODING_SNAPPY; + } else { + encoding = compressionType.encoding; + } + + if (encoding == 3) { + deflater = new Deflater(options.zlibCompressionLevel()); + } else { + deflater = null; + } + + start(); + } + + /** @return Sereal protocol version used by this encoder. */ + public int protocolVersion() { + return protocolVersion; + } + + /** @return {@code true} after the root element has been completely written. */ + public boolean isComplete() { + return currentContext.type == CONTEXT_ROOT && currentContext.count == 1; + } + + /** + * @return Sereal document offset of last written token. + *

+ * The returned value can be used for the target of {@link TokenEncoder#appendCopy(int)}, + * {@link TokenEncoder#appendRefPrevious(int)} or {@link TokenEncoder#appendAlias(int)}. + */ + public int trackOffsetLastValue() { + return trackOffset - headerOffset; + } + + /** + * @return Sereal document offset of the next token that will be written. + *

+ * The returned value can be used for the target of {@link TokenEncoder#appendCopy(int)}, + * {@link TokenEncoder#appendRefPrevious(int)} or {@link TokenEncoder#appendAlias(int)}. + */ + public int trackOffsetNextValue() { + return size - headerOffset; + } + + /** Reset internal state as it was right after construction. */ + public void reset() { + size = compressedSize = headerSize = headerOffset = trackOffset = 0; + hasHeader = false; + start(); + } + + /** + * @return Get a reference to the encoded document. + *

+ * The contents of the buffer will become invalid after calling any of the mutator methods. + */ + public ByteArray getDataReference() { + checkParts(); + if (compressedSize != 0) { + return new ByteArray(compressedBytes, compressedSize); + } else { + return new ByteArray(bytes, size); + } + } + + /** @return Get a copy of the encoded document. */ + public byte[] getData() { + checkParts(); + if (compressedSize != 0) { + return Arrays.copyOf(compressedBytes, compressedSize); + } else { + return Arrays.copyOf(bytes, size); + } + } + + private void checkParts() { + if (currentContext.outer != null) { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Missing endHash/endArray/endObject call"); + } else { + throw new IllegalStateException("Missing endHeader/endDocument call"); + } + } + if (currentContext.type == CONTEXT_INITIAL || currentContext.type == CONTEXT_HEADER) { + throw new IllegalStateException("Missing startDocument/endDocument calls"); + } else if (currentContext.type != CONTEXT_FINAL) { + throw new IllegalStateException(""); + } + } + + private static void prepareHeader( + byte[] originBytes, byte[] compressedBytes, int headerSize, int sizeLength) { + System.arraycopy(originBytes, 0, compressedBytes, 0, headerSize); + if (sizeLength > 0) { + // varint-encoded 0, filling all space + for (int i = headerSize; i < headerSize + sizeLength - 1; i++) { + compressedBytes[i] = (byte) 0x80; + } + compressedBytes[headerSize + sizeLength - 1] = 0; + } + } + + private static void finishHeader( + byte[] compressedBytes, long compressedSize, int headerSize, int sizeLength) { + int after = encodeVarint(compressedSize, compressedBytes, headerSize); + if (after != headerSize + sizeLength) { + compressedBytes[after - 1] |= (byte) 0x80; + } + } + + private void start() { + if (protocolVersion >= 3) { + appendBytesUnsafe(HEADER_V3); + } else { + appendBytesUnsafe(HEADER); + } + appendByteUnsafe((byte) ((encoding << 4) | protocolVersion)); + currentContext = new Context(null, CONTEXT_INITIAL, 0, 0); + } + + /** + * Set up the encoder to emit data to the Sereal header. + *

+ * Sereal header is optional, and if present it must be emitted before the main document. + * + * @throws SerealException header cannot be encoded. + */ + public void startHeader() throws SerealException { + if (protocolVersion == 1) { + throw new SerealException("Can't encode user header in Sereal protocol version 1"); + } else if (currentContext.outer != null) { + throw new IllegalStateException("startHeader called while already inside startHeader/startDocumentt"); + } else if (currentContext.type != CONTEXT_INITIAL) { + if (currentContext.type == CONTEXT_HEADER) { + throw new IllegalStateException("startHeader called twice"); + } else { + throw new IllegalStateException("startHeader called after emitting document body"); + } + } + + currentContext = new Context(currentContext, CONTEXT_ROOT, size, 1); + // be optimistic about encoded header size + size += 2; // one for the size, one for 8bit bitfield + // because offsets start at 1 + headerOffset = size - 1; + } + + /** + * Complete encoding of the Sereal header. + * + * @throws SerealException header cannot be encoded. + */ + public void endHeader() throws SerealException { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + hasHeader = true; + headerSize = size; + + int originalSize = currentContext.position; + int suffixSize = (size - originalSize - 1); + if (suffixSize < 128) { + bytes[originalSize] = (byte) suffixSize; + bytes[originalSize + 1] = 0x01; + } else { + // we were too optimistic + int sizeLength = varintLength(suffixSize); + + // make space + ensureAvailable(sizeLength - 1); + System.arraycopy( + bytes, + originalSize + 2, + bytes, + originalSize + sizeLength + 1, + suffixSize - 1); + size += sizeLength - 1; + headerSize = size; + + // now write size and 8bit bitfield + encodeVarint(suffixSize, bytes, originalSize); + bytes[originalSize + sizeLength] = 0x01; + } + currentContext = new Context(null, CONTEXT_HEADER, -1, 0); + } + + /** + * Set up the encoder to emit data for the Sereal body. + * + * @throws SerealException document cannot be encoded. + */ + public void startDocument() throws SerealException { + if (currentContext.outer != null) { + throw new IllegalStateException("startDocument called while already inside startHeader/startDocument"); + } + if (currentContext.type == CONTEXT_FINAL) { + throw new IllegalStateException("startDocument called twice"); + } + + currentContext = new Context(null, CONTEXT_FINAL, -1, 0); + if (!hasHeader) { + appendByteUnsafe((byte) 0x00); + headerSize = size; + } else { + if (headerSize != size) { + throw new SerealException("Can't append data between header and body"); + } + } + + currentContext = new Context(currentContext, CONTEXT_ROOT, -1, 1); + if (protocolVersion > 1) { + // because offsets start at 1 + headerOffset = headerSize - 1; + } else { + headerOffset = 0; + } + } + + /** + * Complete encoding of the Sereal body. + * + * @throws SerealException document cannot be encoded. + */ + public void endDocument() throws SerealException { + if (currentContext.type != CONTEXT_ROOT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (!compressionType.equals(CompressionType.NONE) && size - headerSize > compressionThreshold) { + if (compressionType.equals(CompressionType.SNAPPY)) { + compressSnappy(); + } else if (compressionType.equals(CompressionType.ZLIB)) { + compressZlib(); + } else if (compressionType.equals(CompressionType.ZSTD)) { + compressZstd(); + } + } else { + // we did not do compression after all + markNotCompressed(); + } + currentContext = currentContext.outer; + } + + private void checkCount(Context context) throws SerealException { + if (context.expectedCount != -1 && context.count != context.expectedCount) { + throw new SerealException("Bad value count"); + } + } + + /** + * Append a Sereal {@code REFN} tag. + * + * @throws SerealException reference cannot be encoded. + */ + public void appendRefNext() throws SerealException { + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REFN); + } + + /** + * Append a Sereal {@code WEAKEN} tag. + *

+ * The next value appended needs to be some kind of reference. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendWeaken() throws SerealException { + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_WEAKEN); + } + + /** + * Convenience method to check or force the next emitted value to be a weak reference. + *

+ * After emitting the value, {@link TokenEncoder#endWeaken()} must be called. + *

+ * If the emitted value is not a reference and {@code forceReference} is {@code false}, an exception is thrown. + *

+ * If the emitted value is not a reference and {@code forceReference} is {@code true}, the value + * is forced into a reference by using {@code REFN}. + * + * @param forceReference {@code true} to force the value as a reference, {@code false} otherwise + * + * @throws SerealException tag cannot be encoded. + */ + public void startWeaken(boolean forceReference) throws SerealException { + ensureAvailable(2); + appendByteUnsafe(SerealHeader.SRL_HDR_WEAKEN); + + if (forceReference) { + appendByteUnsafe(SerealHeader.SRL_HDR_PAD); + } + currentContext = new Context(currentContext, CONTEXT_WEAKEN, size - 1,1); + } + + /** + * Completes writing a weak reference started with {@link TokenEncoder#startWeaken(boolean)}. + * + * @throws SerealException tag cannot be encoded. + */ + public void endWeaken() throws SerealException { + if (currentContext.type != CONTEXT_WEAKEN) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (bytes[currentContext.position] == SerealHeader.SRL_HDR_PAD) { + if (!isRefTag(bytes[currentContext.position + 1])) { + bytes[currentContext.position] = SerealHeader.SRL_HDR_REFN; + } + } else { + if (!isRefTag(bytes[currentContext.position + 1])) { + throw new SerealException("Internal error while encoding weak reference"); + } + } + currentContext = currentContext.outer; + currentContext.count++; + } + + /** + * Append a Sereal {@code REFP} tag. + * + * @param offset Sereal document offset of the target of the reference. + * + * @throws SerealException reference cannot be encoded. + */ + public void appendRefPrevious(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REFP); + appendVarint(offset); + setTrackBit(offset); + } + + /** + * Append a Sereal {@code ALIAS} tag. + * + * @param offset Sereal document offset of the target of the alias. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendAlias(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_ALIAS); + appendVarint(offset); + setTrackBit(offset); + } + + /** + * Append a Sereal {@code COPY} tag. + * + * @param offset Sereal document offset of the target of the copy. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendCopy(int offset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(offset); + } + + /** + * Append an integer value. + *

+ * Depending on the value, uses one one of {@code POS_*}, {@code NEG_*}, {@code VARINT} or {@code ZIGZAG} Sereal tags. + * + * @param l Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendLong(long l) throws SerealException { + currentContext.count++; + trackOffset = size; + if (l < 0) { + if (l > -17) { + appendByte((byte) (SerealHeader.SRL_HDR_NEG_LOW | (l + 32))); + } else { + appendZigZag(l); + } + } else { + if (l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + } + + /** + * Append an unsigned integer value. Negative value are encoded as the positive 640bit value with the same bit pattern. + *

+ * Depending on the value, uses one one of {@code POS_*} or {@code VARINT} Sereal tags. + * + * @param l Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendUnsignedLong(long l) throws SerealException { + currentContext.count++; + trackOffset = size; + if (l >= 0 && l < 16) { + appendByte((byte) (SerealHeader.SRL_HDR_POS_LOW | l)); + } else { + appendByte(SerealHeader.SRL_HDR_VARINT); + appendVarint(l); + } + } + + /** + * Append a Sereal {@code FLOAT} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendFloat(float f) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(5); + appendByteUnsafe(SerealHeader.SRL_HDR_FLOAT); + int floatBits = Float.floatToIntBits(f); + for (int i = 0; i < 4; ++i) { + appendByteUnsafe((byte) (floatBits & 0xff)); + floatBits >>= 8; + } + } + + /** + * Append a Sereal {@code DOUBLE} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendDouble(double d) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(9); + appendByteUnsafe(SerealHeader.SRL_HDR_DOUBLE); + long doubleBits = Double.doubleToLongBits(d); + for (int i = 0; i < 8; ++i) { + appendByteUnsafe((byte) (doubleBits & 0xff)); + doubleBits >>= 8; + } + } + + /** + * Append a Sereal {@code TRUE} or {@code FALSE} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendBoolean(boolean b) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(b ? SerealHeader.SRL_HDR_TRUE : SerealHeader.SRL_HDR_FALSE); + } + + /** + * Append a Sereal {@code UNDEF} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendUndef() throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_UNDEF); + } + + /** + * Append a Sereal {@code CANONICAL_UNDEF} tag. + * + * @throws SerealException tag cannot be encoded. + */ + public void appendCanonicalUndef() throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_CANONICAL_UNDEF); + } + + /** + * Append an binary/ISO 8859-1 value. + *

+ * Depending on the value, uses one of {@code SHORT_BINARY_*} or {@code BINARY} Sereal tags. + * + * @param bytes Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendBinary(byte[] bytes) throws SerealException { + appendBinary(bytes, 0, bytes.length); + } + + /** + * Append an binary/ISO 8859-1 value. + *

+ * Depending on the value, uses one of {@code SHORT_BINARY_*} or {@code BINARY} Sereal tags. + * + * @param bytes Value to be appended. + * @param offset Index of the first byte to append. + * @param length Number of bytes to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendBinary(byte[] bytes, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + appendBinaryInternal(bytes, offset, length); + } + + private void appendBinaryInternal(byte[] bytes, int offset, int length) throws SerealException { + if (length <= SerealHeader.SRL_MASK_SHORT_BINARY_LEN) { + appendShortBinary(bytes, offset, length); + } else { + appendLongBinary(bytes, offset, length); + } + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string the value to be appended + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(CharSequence string) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string)); + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string Value to be appended. + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(char[] string) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string)); + } + + /** + * Append a Sereal {@code UTF8} tag. + * + * @param string Value to be appended. + * @param offset Index of the first character to append. + * @param length Number of characters to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendString(char[] string, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + appendCharBuffer(CharBuffer.wrap(string, offset, length)); + } + + private void appendCharBuffer(CharBuffer string) throws SerealException { + int maxLength = string.length() * 3; + int varintLenght = varintLength(maxLength); + ensureAvailable( maxLength + varintLenght + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_STR_UTF8); + utf8Encoder.reset(); + int stringStart = size + varintLenght; + ByteBuffer out = ByteBuffer.wrap(bytes, stringStart, bytes.length - stringStart); + CoderResult result = utf8Encoder.encode(string, out, true); + if (result.isError()) { + throw new SerealException(result.toString()); + } + int actualLength = out.position() - stringStart; + for (int varintEnd = encodeVarint(actualLength, bytes, size); varintEnd < stringStart; ++varintEnd) { + bytes[varintEnd - 1] |= (byte) 0x80; + bytes[varintEnd] = (byte) 0x00; + } + size += varintLenght + actualLength; + } + + /** + * Append a Sereal {@code UTF8} tag. + *

+ * The passed-in value is assumed to be valid UTF-8, no check is performed. + * + * @param utf8 Value to be appended. + * @param offset Index of the first byte to append. + * @param length Number of bytes to append. + * + * @throws SerealException value cannot be encoded. + */ + public void appendUTF8(byte[] utf8, int offset, int length) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable( length + MAX_VARINT_LENGTH + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_STR_UTF8); + appendVarint(length); + appendBytesUnsafe(utf8, offset, length); + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * Pattern flags other than {@link java.util.regex.Pattern#MULTILINE}, {@link java.util.regex.Pattern#DOTALL}, + * {@link java.util.regex.Pattern#CASE_INSENSITIVE} and {@link java.util.regex.Pattern#COMMENTS} are + * silently ignored. + * + * @throws SerealException value cannot be encoded. + */ + public void appendRegexp(Pattern pattern) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendCharBuffer(CharBuffer.wrap(pattern.pattern())); + ensureAvailable(5); + int nextFlag = size + 1; + int flags = pattern.flags(); + + if ((flags & Pattern.MULTILINE) != 0) { + bytes[nextFlag++] = 'm'; + } + if ((flags & Pattern.DOTALL) != 0) { + bytes[nextFlag++] = 's'; + } + if ((flags & Pattern.CASE_INSENSITIVE) != 0) { + bytes[nextFlag++] = 'i'; + } + if ((flags & Pattern.COMMENTS) != 0) { + bytes[nextFlag++] = 'x'; + } + + bytes[size] = (byte) ((nextFlag - size - 1) | SerealHeader.SRL_HDR_SHORT_BINARY); + size = nextFlag; + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * The pattern is encoded using a Sereal {@code UTF8} tag. + */ + public void appendRegexpString(String pattern, byte[] flags) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendCharBuffer(CharBuffer.wrap(pattern)); + appendBinaryInternal(flags, 0, flags.length); + } + + /** + * Append a Sereal {@code REGEXP} tag. + *

+ * The pattern is encoded using a Sereal {@code BINARY} or {@code SHORT_BINARY_*} tag. + */ + public void appendRegexpBinary(byte[] pattern, byte[] flags) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_REGEXP); + appendBinaryInternal(pattern, 0, pattern.length); + appendBinaryInternal(flags, 0, flags.length); + } + + /** + * Start writing an hash reference. + *

+ * Whenever possible, use {@link TokenEncoder#startHash(int)}. + */ + public void startHash() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + currentContext = new Context(currentContext, CONTEXT_HASH, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an hash reference. + *

+ * Depending on the key count, uses one of {@code HASHREF_*} or {@code REFN + HASH} Sereal tags. + * + * @param count Number of keys in the hash. + */ + public void startHash(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + if (count <= 15) { + appendByteUnsafe((byte) (SerealHeader.SRL_HDR_HASHREF | count)); + } else { + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + appendVarint(count); + } + currentContext = new Context(currentContext, CONTEXT_HASH, size, count); + } + + /** + * Start writing an hash value. + *

+ * Whenever possible, use {@link TokenEncoder#startHashValue(int)}. + */ + public void startHashValue() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + currentContext = new Context(currentContext, CONTEXT_HASH, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an hash value. + *

+ * Uses a Sereal {@code HASH} tag without a preceding {@code REFN}. + * + * @param count Number of keys in the hash. + */ + public void startHashValue(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_HASH); + appendVarint(count); + currentContext = new Context(currentContext, CONTEXT_HASH, size, count); + } + + /** + * Complete writing an hash value or hash reference. + */ + public void endHash() throws SerealException { + if (currentContext.type != CONTEXT_HASH) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + if ((currentContext.count & 0x1) != 0) { + throw new SerealException("Odd value count in hash"); + } + currentContext.count >>= 1; + checkCount(currentContext); + if (currentContext.expectedCount == -1) { + if (currentContext.count > 127) { + int lenght = varintLength(currentContext.count); + ensureAvailable(lenght); + System.arraycopy(bytes, currentContext.position + 1, + bytes, currentContext.position + lenght, + size - currentContext.position - 1); + size += lenght - 1; + } + encodeVarint(currentContext.count, bytes, currentContext.position); + } + currentContext = currentContext.outer; + } + + /** + * Start writing an array reference. + *

+ * Whenever possible, use {@link TokenEncoder#startArray(int)}. + */ + public void startArray() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an array reference. + *

+ * Depending on the value count, uses one of {@code ARRAYREF_*} or {@code REFN + ARRAY} Sereal tags. + * + * @param count Number of elements in the array. + */ + public void startArray(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + if (count <= 15) { + appendByteUnsafe((byte) (SerealHeader.SRL_HDR_ARRAYREF | count)); + } else { + appendByteUnsafe(SerealHeader.SRL_HDR_REFN); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + appendVarint(count); + } + currentContext = new Context(currentContext, CONTEXT_ARRAY, size, count); + } + + /** + * Start writing an array value. + *

+ * Whenever possible, use {@link TokenEncoder#startArrayValue(int)}. + */ + public void startArrayValue() throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size,-1); + appendByteUnsafe((byte) 0x0); + } + + /** + * Start writing an array value. + *

+ * Uses a Sereal {@code ARRAY} tag without a preceding {@code REFN}. + */ + public void startArrayValue(int count) throws SerealException { + currentContext.count++; + trackOffset = size; + ensureAvailable(3); + appendByteUnsafe(SerealHeader.SRL_HDR_ARRAY); + currentContext = new Context(currentContext, CONTEXT_ARRAY, size, count); + appendVarint(count); + } + + /** + * Complete writing an array value or array reference. + */ + public void endArray() throws SerealException { + if (currentContext.type != CONTEXT_ARRAY) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + if (currentContext.expectedCount == -1) { + if (currentContext.count > 127) { + int lenght = varintLength(currentContext.count); + ensureAvailable(lenght); + System.arraycopy(bytes, currentContext.position + 1, + bytes, currentContext.position + lenght, + size - currentContext.position - 1); + size += lenght - 1; + } + encodeVarint(currentContext.count, bytes, currentContext.position); + } + currentContext = currentContext.outer; + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as a {@code SHORT_BINARY_*} or {@code BINARY} tag. + */ + public void startObject(byte[] className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendBinaryInternal(className, 0, className.length); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as an {@code UTF8} tag. + */ + public void startObject(CharSequence className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendCharBuffer(CharBuffer.wrap(className)); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as an {@code UTF8} tag. + */ + public void startObject(char[] className) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendCharBuffer(CharBuffer.wrap(className)); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECT} tag, followed by the class as a {@code COPY} tag. + */ + public void startObject(int classnameOffset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECT); + appendByte(SerealHeader.SRL_HDR_COPY); + appendVarint(classnameOffset); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Start writing an object value. + *

+ * Uses a Sereal {@code OBJECTV} tag. + */ + public void startObjectV(int classnameOffset) throws SerealException { + currentContext.count++; + trackOffset = size; + appendByte(SerealHeader.SRL_HDR_OBJECTV); + appendVarint(classnameOffset); + currentContext = new Context(currentContext, CONTEXT_OBJECT, size,1); + } + + /** + * Complete writing an object value. + */ + public void endObject() throws SerealException { + if (currentContext.type != CONTEXT_OBJECT) { + throw new IllegalStateException("Mismatched begin/end calls"); + } + checkCount(currentContext); + currentContext = currentContext.outer; + } + + private void markNotCompressed() { + compressedSize = 0; + bytes[4] &= (byte) 0xf; + } + + private void compressSnappy() throws SerealException { + int maxSize = Snappy.maxCompressedLength(size - headerSize); + int sizeLength = encoding == SerealHeader.SRL_ENCODING_SNAPPY ? varintLength(maxSize) : 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + maxSize) > compressedBytes.length) { + compressedBytes = new byte[headerSize + sizeLength + maxSize]; + } + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + int compressed; + try { + compressed = + Snappy.compress( + bytes, headerSize, size - headerSize, compressedBytes, headerSize + sizeLength); + } catch (IOException e) { + throw new SerealException(e); + } + compressedSize = headerSize + sizeLength + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + if (encoding == 2) { + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + } + + // from miniz.c + private int zlibMaxSize(int sourceLen) { + return Math.max( + 128 + (sourceLen * 110) / 100, 128 + sourceLen + ((sourceLen / (31 * 1024)) + 1) * 5); + } + + private void compressZlib() { + deflater.reset(); + + int sourceSize = size - headerSize; + int maxSize = zlibMaxSize(sourceSize); + int sizeLength = varintLength(sourceSize); + int sizeLength2 = varintLength(maxSize); + int pos = 0; + + // I don't think there is any point in overallocating here + if ((headerSize + sizeLength + sizeLength2 + maxSize) > compressedBytes.length) { + compressedBytes = new byte[headerSize + sizeLength + sizeLength2 + maxSize]; + } + + System.arraycopy(bytes, 0, compressedBytes, 0, headerSize); + pos += headerSize; + pos = encodeVarint(sourceSize, compressedBytes, pos); + + // varint-encoded 0, filling all space + int encodedSizePos = pos; + for (int max = pos + sizeLength2 - 1; pos < max; ) { + compressedBytes[pos++] = (byte) 128; + } + compressedBytes[pos++] = 0; + + deflater.setInput(bytes, headerSize, sourceSize); + deflater.finish(); + + int compressed = deflater.deflate(compressedBytes, pos, compressedBytes.length - pos); + compressedSize = headerSize + sizeLength + sizeLength2 + compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + int after = encodeVarint(compressed, compressedBytes, encodedSizePos); + if (after != headerSize + sizeLength + sizeLength2) { + compressedBytes[after - 1] |= (byte) 0x80; + } + } + + private void compressZstd() throws SerealException { + long maxSize = Zstd.compressBound(size - headerSize); + int sizeLength = varintLength(maxSize); + + if (headerSize + sizeLength + maxSize > Integer.MAX_VALUE) { + throw new SerealException( + "Compressed data size exceeds integer MAX_VALUE: " + (headerSize + maxSize)); + } + if (headerSize + sizeLength + maxSize > compressedBytes.length) { + compressedBytes = new byte[(int) (headerSize + sizeLength + maxSize)]; + } + + prepareHeader(bytes, compressedBytes, headerSize, sizeLength); + + long compressed = + Zstd.compressUsingDict( + compressedBytes, + headerSize + sizeLength, + bytes, + headerSize, + size - headerSize, + new byte[0], + zstdCompressionLevel); + if (Zstd.isError(compressed)) { + throw new SerealException(Zstd.getErrorName(compressed)); + } + compressedSize = headerSize + sizeLength + (int) compressed; + if (compressedSize > size) { + markNotCompressed(); + return; + } + + finishHeader(compressedBytes, compressed, headerSize, sizeLength); + } + + private void appendShortBinary(byte[] latin1, int offset, int length) throws SerealException { + // length of string + appendByte((byte) (length | SerealHeader.SRL_HDR_SHORT_BINARY)); + + // save it + appendBytes(latin1, offset, length); + } + + private void appendLongBinary(byte[] latin1, int offset, int lenght) { + // length of string + appendByte(SerealHeader.SRL_HDR_BINARY); + appendBytesWithLength(latin1, offset, lenght); + } + + private void appendBytesWithLength(byte[] in, int offset, int length) { + appendVarint(length); + appendBytes(in, offset, length); + } + + private void appendVarint(long n) { + ensureAvailable(MAX_VARINT_LENGTH); + size = encodeVarint(n, bytes, size); + } + + private void appendZigZag(long n) { + ensureAvailable(MAX_VARINT_LENGTH + 1); + appendByteUnsafe(SerealHeader.SRL_HDR_ZIGZAG); + size = encodeVarint((n << 1) ^ (n >> 63), bytes, size); // note the signed right shift + } + + private void setTrackBit(int offset) { + bytes[offset + headerOffset] |= (byte) 0x80; + } + + private static boolean isRefTag(byte tag) { + // the first branch is the common case, the other two branchs are unlikely + if (tag == SerealHeader.SRL_HDR_REFN || + tag == SerealHeader.SRL_HDR_REFP) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_ARRAYREF) == SerealHeader.SRL_HDR_ARRAYREF) { + return true; + } else if ((tag & SerealHeader.SRL_HDR_HASHREF) == SerealHeader.SRL_HDR_HASHREF) { + return true; + } + + return false; + } + + private static int varintLength(long n) { + int length = 0; + + while (Long.compareUnsigned(n, 127) > 0) { + n >>>= 7; + length++; + } + + return length + 1; + } + + private static int encodeVarint(long n, byte[] buffer, int pos) { + while (Long.compareUnsigned(n, 127) > 0) { + buffer[pos++] = (byte) ((n & 127) | 128); + n >>>= 7; + } + buffer[pos++] = (byte) n; + + return pos; + } + + private void ensureAvailable(int required) { + long total = required + size; + + if (total > bytes.length) { + bytes = Arrays.copyOf(bytes, (int) (total * 3 / 2)); + } + } + + private void appendBytes(byte[] data, int offset, int lenght) { + ensureAvailable(lenght); + appendBytesUnsafe(data, offset, lenght); + } + + private void appendBytesUnsafe(byte[] data) { + System.arraycopy(data, 0, bytes, size, data.length); + size += data.length; + } + + private void appendBytesUnsafe(byte[] data, int offset, int length) { + System.arraycopy(data, offset, bytes, size, length); + size += length; + } + + private void appendByte(byte data) { + ensureAvailable(1); + appendByteUnsafe(data); + } + + private void appendByteUnsafe(byte data) { + bytes[size] = data; + size++; + } + + // used for testing + void poisonBuffer() { + for (int i = size; i < bytes.length; ++i) { + bytes[i] = (byte) (Math.random() * 256); + } + } + + /** + * Close the encoder to recycle the resources, it must be called by the client at the end of the + * use, otherwise, NullPointerException will be thrown when you reuse the ZLIB encoder. + *

+ * Returns the native memory used by deflater explicitly before the garbage collector. + */ + public void close() { + if (deflater != null) { + deflater.end(); + deflater = null; + } + } +} diff --git a/src/main/java/com/booking/sereal/TypeMapper.java b/src/main/java/com/booking/sereal/TypeMapper.java new file mode 100644 index 0000000000..c07469c580 --- /dev/null +++ b/src/main/java/com/booking/sereal/TypeMapper.java @@ -0,0 +1,14 @@ +package com.booking.sereal; + +import java.util.List; +import java.util.Map; + +public interface TypeMapper { + public boolean useObjectArray(); + + public List makeArray(int size); + + public Map makeMap(int size); + + public Object makeObject(String className, Object data); +} diff --git a/src/main/java/com/booking/sereal/Utils.java b/src/main/java/com/booking/sereal/Utils.java new file mode 100644 index 0000000000..6df6848353 --- /dev/null +++ b/src/main/java/com/booking/sereal/Utils.java @@ -0,0 +1,189 @@ +package com.booking.sereal; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.lang.reflect.Array; +import java.util.*; +import java.util.regex.Pattern; + +public class Utils { + private static final char[] hexDigits = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', + }; + // so we can not overflow our stack with ciruclar refs + private static Set already_output = new HashSet(); + + public static String join(String[] parts, String seperator) { + return join(Arrays.asList(parts), seperator); + } + + public static String join(List things, String separator) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < things.size(); i++) { + sb.append(things.get(i).toString()); + if (i < things.size() - 1) { + sb.append(separator); + } + } + return sb.toString(); + } + + public static String dump(Object o) { + + String d = dump(o, 0); + already_output.clear(); + return d; + } + + @SuppressWarnings({"rawtypes", "deprecation"}) + private static String dump(Object o, int indent) { + + if (o != null && already_output.contains(System.identityHashCode(o))) { + return "@" + System.identityHashCode(o); + } else if (o != null) { + already_output.add(System.identityHashCode(o)); + } + + String ind = ""; + for (int i = 0; i < indent; i++) { + ind += "\t"; + } + if (o == null) { + return "(NULL)"; + } else if (o instanceof Map) { + StringBuilder sb = new StringBuilder(ind + "Map@" + System.identityHashCode(o) + " {\n"); + Map map = (Map) o; + Object[] array = map.keySet().toArray(); + try { + Arrays.sort(array); + } catch (ClassCastException e) { + // In case where the array can't be sorted (if the keys don't + // implement Comparable) + } + for (Object key : array) { + sb.append(ind + dump(key, indent + 1)); + sb.append(" => "); + sb.append(dump(map.get(key), indent + 1)); + sb.append("\n"); + } + sb.append(ind).append("}"); + return sb.toString(); + } else if (o instanceof List) { + List l = (List) o; + String type = l.isEmpty() ? "" : l.get(0).getClass().getName(); + StringBuilder sb = new StringBuilder(ind + "List<" + type + ">[\n"); + for (Object li : l) { + sb.append(dump(li, indent + 1)); + sb.append("\n"); + } + sb.append(ind).append("]"); + return sb.toString(); + } else if (o.getClass().isArray()) { + StringBuilder sb = new StringBuilder(ind + "Array@" + System.identityHashCode(o) + " [\n"); + int length = Array.getLength(o); + for (int i = 0; i < length; i++) { + sb.append(dump(Array.get(o, i), indent + 1)); + sb.append("\n"); + } + + sb.append(ind).append("]"); + return sb.toString(); + } else if (o instanceof Pattern) { + Pattern pat = (Pattern) o; + return "/" + + pat.pattern() + + "/" + + (((pat.flags() & Pattern.CANON_EQ) > 0) ? "c" : "") + + (((pat.flags() & Pattern.CASE_INSENSITIVE) > 0) ? "i" : "") + + (((pat.flags() & Pattern.COMMENTS) > 0) ? "x" : "") + + (((pat.flags() & Pattern.DOTALL) > 0) ? "s" : "") + + (((pat.flags() & Pattern.LITERAL) > 0) ? "q" : "") + + (((pat.flags() & Pattern.MULTILINE) > 0) ? "m" : "") + + (((pat.flags() & Pattern.UNICODE_CASE) > 0) ? "l" : "") + + (((pat.flags() & Pattern.UNIX_LINES) > 0) ? "u" : "") + + "@" + + System.identityHashCode(o); + + } else if (o instanceof PerlAlias) { + return ind + "Alias: " + dump(((PerlAlias) o).getValue(), indent); + } else if (o instanceof PerlReference) { + return ind + + "Perlref@" + + System.identityHashCode(o) + + ": " + + dump(((PerlReference) o).getValue(), indent); + } else if (o instanceof WeakReference) { + return ind + + "(weakref@" + + System.identityHashCode(o) + + ") " + + dump(((WeakReference) o).get(), 0); + } else if (o instanceof PerlObject) { + PerlObject po = (PerlObject) o; + return ind + + "Object(" + + (po.isHash() ? "hash" : (po.isArray() ? "array" : "reference")) + + "):" + + po.getName() + + "= " + + dump(po.getData(), 0); + } else { + // ad system ident hascode (which is normally memory location) so you + // can see if things point to the same + return ind + + o.getClass().getSimpleName() + + "@" + + System.identityHashCode(o) + + ": " + + o.toString(); + } + } + + public static String hexStringFromByteArray(byte[] in) { + return hexStringFromByteArray(in, -1); + } + + public static String hexStringFromByteArray(byte[] in, int group) { + StringBuilder out = + new StringBuilder(2 + in.length * 2 + (group != -1 ? (in.length / group + 1) : 0)); + out.append("0x"); + + int count = 0; + for (byte b : in) { + out.append(hexDigits[(b >> 4) & 0xf]); + out.append(hexDigits[b & 0xf]); + + if (group > 0 && (++count == group)) { + out.append(' '); + count = 0; + } + } + + return out.toString(); + } + + public static Object decodeFile(Decoder decoder, File f) throws SerealException, IOException { + if (!f.exists()) { + throw new FileNotFoundException("No such file: " + f.getCanonicalPath()); + } + + // read everything + int size = (int) f.length(); // yeah yeah truncate + byte[] buf = new byte[size]; + FileInputStream fi = new FileInputStream(f); + try { + fi.read(buf); + } finally { + fi.close(); + } + + decoder.setData(buf); + Object structure = decoder.decode(); + + return structure; + } +} diff --git a/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java b/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java new file mode 100644 index 0000000000..31f4fde773 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/BytearrayCopyMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class BytearrayCopyMap { + public static final long NOT_FOUND = -1; + + private static final byte[] NO_KEY = new byte[0]; + byte[][] keys; + private long[] values; + private int size, maxLoad, modulus; + + public BytearrayCopyMap() { + init(32); + } + + private void init(int capacity) { + keys = new byte[capacity][]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(byte[] key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(byte[] key, long value) { + int slot = findSlot(key); + + if (keys[slot] == NO_KEY) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + byte[][] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(byte[] key) { + int slot = Arrays.hashCode(key) & modulus; + while (keys[slot] != NO_KEY && !Arrays.equals(keys[slot], key)) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/IdentityMap.java b/src/main/java/com/booking/sereal/impl/IdentityMap.java new file mode 100644 index 0000000000..38578f1840 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/IdentityMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class IdentityMap { + public static final long NOT_FOUND = -1; + + private static final Object NO_KEY = new Object(); + private Object[] keys; + private long[] values; + private int size, maxLoad, modulus; + + public IdentityMap() { + init(32); + } + + private void init(int capacity) { + keys = new Object[capacity]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(Object key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(Object key, long value) { + int slot = findSlot(key); + + if (keys[slot] != key) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + Object[] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(Object key) { + int slot = System.identityHashCode(key) & modulus; + while (keys[slot] != NO_KEY && keys[slot] != key) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/RefpMap.java b/src/main/java/com/booking/sereal/impl/RefpMap.java new file mode 100644 index 0000000000..9a8310cb0d --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/RefpMap.java @@ -0,0 +1,78 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class RefpMap { + public static final Object NOT_FOUND = new Object(); + + private static final long NO_KEY = -1; + private long[] keys; + private Object[] values; + private int size, maxLoad, modulus; + + public RefpMap() { + init(32); + } + + private void init(int capacity) { + keys = new long[capacity]; + values = new Object[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final Object get(long key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(long key, Object value) { + int slot = findSlot(key); + + if (keys[slot] != key) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + long[] oldKeys = keys; + Object[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(long key) { + int slot = hash6432Shift(key) & modulus; + while (keys[slot] != NO_KEY && keys[slot] != key) slot = (slot + 1) & modulus; + return slot; + } + + // http://burtleburtle.net/bob/hash/integer.html + // not tested for the actual distribution of offsets in Sereal + private int hash6432Shift(long key) { + key = (~key) + (key << 18); // key = (key << 18) - key - 1; + key = key ^ (key >>> 31); + key = key * 21; // key = (key + (key << 2)) + (key << 4); + key = key ^ (key >>> 11); + key = key + (key << 6); + key = key ^ (key >>> 22); + return (int) key; + } +} diff --git a/src/main/java/com/booking/sereal/impl/StringCopyMap.java b/src/main/java/com/booking/sereal/impl/StringCopyMap.java new file mode 100644 index 0000000000..263f630271 --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/StringCopyMap.java @@ -0,0 +1,66 @@ +package com.booking.sereal.impl; + +import java.util.Arrays; + +public class StringCopyMap { + public static final long NOT_FOUND = -1; + + private static final String NO_KEY = null; + String[] keys; + private long[] values; + private int size, maxLoad, modulus; + + public StringCopyMap() { + init(32); + } + + private void init(int capacity) { + keys = new String[capacity]; + values = new long[capacity]; + Arrays.fill(keys, NO_KEY); + modulus = capacity - 1; + maxLoad = (int) (capacity * 0.80); + size = 0; + } + + public void clear() { + Arrays.fill(keys, NO_KEY); + size = 0; + } + + public final long get(String key) { + int slot = findSlot(key); + + return keys[slot] == NO_KEY ? NOT_FOUND : values[slot]; + } + + public final void put(String key, long value) { + int slot = findSlot(key); + + if (keys[slot] == NO_KEY) { + if (size == maxLoad) { + rehash(); + slot = findSlot(key); + } + + keys[slot] = key; + values[slot] = value; + size++; + } else values[slot] = value; + } + + private void rehash() { + String[] oldKeys = keys; + long[] oldValues = values; + + init(keys.length * 2); + + for (int i = 0, max = oldKeys.length; i < max; ++i) put(oldKeys[i], oldValues[i]); + } + + private int findSlot(String key) { + int slot = key == null ? 0 : key.hashCode() & modulus; + while (keys[slot] != NO_KEY && !keys[slot].equals(key)) slot = (slot + 1) & modulus; + return slot; + } +} diff --git a/src/main/java/com/booking/sereal/impl/package-info.java b/src/main/java/com/booking/sereal/impl/package-info.java new file mode 100644 index 0000000000..d8d7eb8abe --- /dev/null +++ b/src/main/java/com/booking/sereal/impl/package-info.java @@ -0,0 +1,4 @@ +/** + * Internal helper classes used by Sereal implementation. + */ +package com.booking.sereal.impl; \ No newline at end of file diff --git a/src/main/java/com/booking/sereal/package-info.java b/src/main/java/com/booking/sereal/package-info.java new file mode 100644 index 0000000000..887c3d7ee4 --- /dev/null +++ b/src/main/java/com/booking/sereal/package-info.java @@ -0,0 +1,14 @@ +/** + * Encoder/Decoder interface for Sereal + *

+ * See {@code com.booking.sereal.jackson.SerealObjectMapper} for a higher-level, easier to use interface. + *

+ * The main entry points are {@link com.booking.sereal.Encoder} and {@link com.booking.sereal.Decoder}, + * which allow simple encoding and decoding of nested hash/array data structures, and mirror + * the functionality available in the Perl implementation. + *

+ * {@link com.booking.sereal.TokenEncoder} and {@link com.booking.sereal.TokenDecoder} offer a low-level + * interface that can be used to build custom encoders/decoders, but is generally inconvenient and + * error-prone when used directly. + */ +package com.booking.sereal; \ No newline at end of file diff --git a/src/main/java/org/perlonjava/runtime/mro/DFS.java b/src/main/java/org/perlonjava/runtime/mro/DFS.java index 2579e1cc27..00d5c241a5 100644 --- a/src/main/java/org/perlonjava/runtime/mro/DFS.java +++ b/src/main/java/org/perlonjava/runtime/mro/DFS.java @@ -96,7 +96,7 @@ private static void populateIsaMapWithCycleDetection(String className, // Get current @ISA array - FORCE fresh read RuntimeArray isaArray = InheritanceResolver.getIsaArrayForClass(className); List parents = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : InheritanceResolver.visibleArrayElements(isaArray)) { String parentName = entity.toString(); // FIXED: Skip empty or null parent names if (parentName != null && !parentName.isEmpty()) { diff --git a/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java b/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java index d5a153852b..602be25d58 100644 --- a/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java +++ b/src/main/java/org/perlonjava/runtime/mro/InheritanceResolver.java @@ -157,7 +157,7 @@ private static boolean hasIsaChanged(String className, MroRuntimeState state) { // Build current ISA list List currentIsa = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : visibleArrayElements(isaArray)) { String parentName = entity.toString(); if (parentName != null && !parentName.isEmpty()) { currentIsa.add(parentName); @@ -342,7 +342,7 @@ private static void populateIsaMapHelper(String className, // Retrieve @ISA array for the given class RuntimeArray isaArray = getIsaArrayForClass(className); List parents = new ArrayList<>(); - for (RuntimeBase entity : isaArray.elements) { + for (RuntimeBase entity : visibleArrayElements(isaArray)) { String parentName = entity.toString(); // Handle undef elements as "main" for Perl compatibility if (parentName == null || parentName.equals("")) { @@ -369,6 +369,26 @@ private static void populateIsaMapHelper(String className, currentPath.remove(className); } + /** + * Return the Perl-visible contents of an array. + * + *

Most package {@code @ISA} arrays are ordinary arrays, but Perl permits + * them to be tied. Method lookup must honor the tie's {@code FETCHSIZE} and + * {@code FETCH} methods; reading {@link RuntimeArray#elements} directly sees + * only the empty backing list used by a tied array.

+ */ + static List visibleArrayElements(RuntimeArray array) { + if (array.type != RuntimeArray.TIED_ARRAY) { + return array.elements; + } + List visible = new ArrayList<>(); + int size = TieArray.tiedFetchSize(array).getInt(); + for (int i = 0; i < size; i++) { + visible.add(array.get(i)); + } + return visible; + } + /** * Searches for a method in the class hierarchy starting from a specific index. * Uses method caching to improve performance for both found and not-found methods. diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java b/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java new file mode 100644 index 0000000000..2f33626c59 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CacheFastMmap.java @@ -0,0 +1,247 @@ +package org.perlonjava.runtime.perlmodule; + +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Cache::FastMmap's XS primitive API implemented with a JVM shared map. + * + *

The Perl layer retains expiry, serialization, callbacks, and atomic-operation + * semantics. This backend replaces only the native mmap page store, which is not + * loadable on the JVM.

+ */ +public class CacheFastMmap extends PerlModuleBase { + private static final Map> SHARED = new ConcurrentHashMap<>(); + private static volatile long timeOverride; + + private static final class Entry { + RuntimeScalar value; + long expireOn; + int flags; + Long modseq; + long lastAccess; + } + + private static final class State { + final Map params = new ConcurrentHashMap<>(); + ConcurrentHashMap entries = new ConcurrentHashMap<>(); + boolean locked; + long reads; + long hits; + } + + public CacheFastMmap() { super("Cache::FastMmap", false); } + + public static void initialize() { + CacheFastMmap module = new CacheFastMmap(); + try { + for (String name : new String[]{ + "fc_new", "fc_set_param", "fc_init", "fc_hash", "fc_lock", "fc_unlock", + "fc_is_locked", "fc_read", "fc_write", "fc_delete", "fc_tombstone", + "fc_get_keys", "fc_get_page_details", "fc_reset_page_details", "fc_expunge", + "fc_set_time_override", "fc_close"}) { + module.registerMethod(name, null); + } + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList fc_new(RuntimeArray args, int ctx) { + return new RuntimeScalar(new State()).getList(); + } + + public static RuntimeList fc_set_param(RuntimeArray args, int ctx) { + state(args.get(0)).params.put(args.get(1).toString(), new RuntimeScalar(args.get(2))); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_init(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + RuntimeScalar share = state.params.get("share_file"); + if (share != null && share.defined().getBoolean()) { + String file = share.toString(); + boolean init = state.params.getOrDefault("init_file", new RuntimeScalar(0)).getBoolean(); + state.entries = SHARED.computeIfAbsent(file, ignored -> new ConcurrentHashMap<>()); + if (init) state.entries.clear(); + try { + Path path = Path.of(file); + if (path.getParent() != null) Files.createDirectories(path.getParent()); + Files.write(path, new byte[0], StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (Exception e) { + throw new IllegalStateException("Unable to initialize cache share file " + file, e); + } + } + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_hash(RuntimeArray args, int ctx) { + String key = args.get(1).toString(); + int hash = key.hashCode() & 0x7fffffff; + int pages = state(args.get(0)).params.getOrDefault("num_pages", new RuntimeScalar(89)).getInt(); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(hash % Math.max(1, pages))); + out.add(new RuntimeScalar(hash)); + return out; + } + + public static RuntimeList fc_lock(RuntimeArray args, int ctx) { + state(args.get(0)).locked = true; + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_unlock(RuntimeArray args, int ctx) { + state(args.get(0)).locked = false; + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_is_locked(RuntimeArray args, int ctx) { + return new RuntimeScalar(state(args.get(0)).locked).getList(); + } + + public static RuntimeList fc_read(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + state.reads++; + Entry entry = state.entries.get(args.get(2).toString()); + if (entry != null && expired(entry)) { + state.entries.remove(args.get(2).toString(), entry); + entry = null; + } + RuntimeList out = new RuntimeList(); + if (entry == null) { + out.add(new RuntimeScalar()); + out.add(new RuntimeScalar(0)); + out.add(new RuntimeScalar(0)); + out.add(new RuntimeScalar()); + out.add(new RuntimeScalar()); + } else { + state.hits++; + entry.lastAccess = now(); + out.add(new RuntimeScalar(entry.value)); + out.add(new RuntimeScalar(entry.flags)); + out.add(new RuntimeScalar(1)); + out.add(entry.expireOn < 0 ? new RuntimeScalar() : new RuntimeScalar(entry.expireOn)); + out.add(entry.modseq == null ? new RuntimeScalar() : new RuntimeScalar(entry.modseq)); + } + return out; + } + + public static RuntimeList fc_write(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + String key = args.get(2).toString(); + long expireOn = args.get(4).getLong(); + int flags = args.get(5).getInt(); + Long modseq = args.size() > 6 && args.get(6).defined().getBoolean() + ? args.get(6).getLong() : null; + Entry old = state.entries.get(key); + if (old != null && old.modseq != null && (modseq == null || modseq < old.modseq)) { + return new RuntimeScalar(-1).getList(); + } + Entry entry = new Entry(); + entry.value = new RuntimeScalar(args.get(3)); + entry.expireOn = expireOn; + entry.flags = flags; + entry.modseq = modseq; + entry.lastAccess = now(); + state.entries.put(key, entry); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_delete(RuntimeArray args, int ctx) { + Entry removed = state(args.get(0)).entries.remove(args.get(2).toString()); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(removed != null)); + out.add(new RuntimeScalar(removed == null ? 0 : removed.flags)); + return out; + } + + public static RuntimeList fc_tombstone(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + String key = args.get(2).toString(); + long modseq = args.get(4).getLong(); + Entry old = state.entries.get(key); + if (old != null && old.modseq != null && old.modseq > modseq) return new RuntimeScalar(0).getList(); + Entry tombstone = new Entry(); + tombstone.value = new RuntimeScalar(); + tombstone.expireOn = args.get(3).getLong(); + tombstone.modseq = modseq; + tombstone.lastAccess = now(); + state.entries.put(key, tombstone); + return new RuntimeScalar(1).getList(); + } + + public static RuntimeList fc_get_keys(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + int mode = args.size() > 1 ? args.get(1).getInt() : 0; + RuntimeList out = new RuntimeList(); + state.entries.forEach((key, entry) -> { + if (expired(entry)) return; + if (mode == 0) { + out.add(new RuntimeScalar(key)); + } else { + RuntimeHash detail = new RuntimeHash(); + detail.put("key", new RuntimeScalar(key)); + detail.put("last_access", new RuntimeScalar(entry.lastAccess)); + detail.put("expire_on", entry.expireOn < 0 ? new RuntimeScalar() : new RuntimeScalar(entry.expireOn)); + detail.put("flags", new RuntimeScalar(entry.flags)); + if (mode >= 2) detail.put("value", new RuntimeScalar(entry.value)); + out.add(detail.createAnonymousReference()); + } + }); + return out; + } + + public static RuntimeList fc_get_page_details(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + RuntimeList out = new RuntimeList(); + out.add(new RuntimeScalar(state.reads)); + out.add(new RuntimeScalar(state.hits)); + return out; + } + + public static RuntimeList fc_reset_page_details(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + state.reads = state.hits = 0; + return new RuntimeList(); + } + + public static RuntimeList fc_expunge(RuntimeArray args, int ctx) { + State state = state(args.get(0)); + int mode = args.get(1).getInt(); + RuntimeList removed = new RuntimeList(); + state.entries.entrySet().removeIf(item -> { + Entry entry = item.getValue(); + boolean remove = mode == 1 || expired(entry); + if (remove && args.get(2).getBoolean()) { + RuntimeHash detail = new RuntimeHash(); + detail.put("key", new RuntimeScalar(item.getKey())); + detail.put("value", new RuntimeScalar(entry.value)); + detail.put("expire_on", new RuntimeScalar(entry.expireOn)); + detail.put("flags", new RuntimeScalar(entry.flags)); + removed.add(detail.createAnonymousReference()); + } + return remove; + }); + return removed; + } + + public static RuntimeList fc_set_time_override(RuntimeArray args, int ctx) { + timeOverride = args.isEmpty() ? 0 : args.get(0).getLong(); + return new RuntimeList(); + } + + public static RuntimeList fc_close(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static State state(RuntimeScalar scalar) { + if (scalar.type == RuntimeScalarType.JAVAOBJECT && scalar.value instanceof State state) return state; + throw new IllegalArgumentException("Invalid Cache::FastMmap native cache handle"); + } + + private static long now() { return timeOverride != 0 ? timeOverride : System.currentTimeMillis() / 1000L; } + private static boolean expired(Entry entry) { return entry.expireOn > 0 && entry.expireOn <= now(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java new file mode 100644 index 0000000000..ab0f6c7d4c --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLVerify.java @@ -0,0 +1,85 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.openssl.PEMParser; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.runtimetypes.*; + +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; + +/** Crypt::OpenSSL::Verify implemented with the existing Bouncy Castle/JCA stack. */ +public class CryptOpenSSLVerify extends PerlModuleBase { + private static final String CLASS_NAME = "Crypt::OpenSSL::Verify"; + private static final String STATE_KEY = "_verify_certificates"; + + public CryptOpenSSLVerify() { super(CLASS_NAME, false); } + + public static void initialize() { + CryptOpenSSLVerify module = new CryptOpenSSLVerify(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("verify", null); + module.registerMethod("register_verify_cb", "noop", null); + module.registerMethod("ctx_error_code", "zero", null); + module.registerMethod("__X509_cleanup", "noop", null); + module.registerMethod("DESTROY", "noop", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + try { + List certificates = new ArrayList<>(); + if (args.size() > 1 && args.get(1).defined().getBoolean()) { + String pem = Files.readString(Path.of(args.get(1).toString())); + try (PEMParser parser = new PEMParser(new StringReader(pem))) { + Object value; + while ((value = parser.readObject()) != null) { + if (value instanceof X509CertificateHolder holder) { + certificates.add(new JcaX509CertificateConverter().setProvider("BC") + .getCertificate(holder)); + } + } + } + } + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(certificates)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } catch (Exception e) { + return CryptOpenSSLX509.fail("Unable to load CA certificates: " + e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private static List certificates(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.value instanceof List) return (List) state.value; + throw new IllegalArgumentException("Invalid Crypt::OpenSSL::Verify object"); + } + + public static RuntimeList verify(RuntimeArray args, int ctx) { + X509Certificate target = CryptOpenSSLX509.certificate(args.get(1)); + for (X509Certificate issuer : certificates(args.get(0))) { + if (!target.getIssuerX500Principal().equals(issuer.getSubjectX500Principal())) continue; + try { + target.verify(issuer.getPublicKey()); + return new RuntimeScalar(1).getList(); + } catch (Exception ignored) { + // Try the remaining candidate issuers. + } + } + return CryptOpenSSLX509.fail("certificate verify failed"); + } + + public static RuntimeList noop(RuntimeArray args, int ctx) { return new RuntimeList(); } + public static RuntimeList zero(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java new file mode 100644 index 0000000000..940a76550e --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509.java @@ -0,0 +1,161 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.Extensions; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.openssl.PEMParser; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +import java.io.ByteArrayInputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.Locale; + +/** Crypt::OpenSSL::X509 compatibility backed by Bouncy Castle and JCA. */ +public class CryptOpenSSLX509 extends PerlModuleBase { + static final String CLASS_NAME = "Crypt::OpenSSL::X509"; + static final String STATE_KEY = "_x509_certificate"; + private static final DateTimeFormatter OPENSSL_TIME = + DateTimeFormatter.ofPattern("MMM dd HH:mm:ss yyyy 'GMT'", Locale.US).withZone(ZoneOffset.UTC); + + static { + if (java.security.Security.getProvider("BC") == null) { + java.security.Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); + } + } + + public CryptOpenSSLX509() { super(CLASS_NAME, false); } + + public static void initialize() { + CryptOpenSSLX509 module = new CryptOpenSSLX509(); + try { + module.registerMethod("new_from_string", null); + module.registerMethod("new_from_file", null); + module.registerMethod("subject", null); + module.registerMethod("issuer", null); + module.registerMethod("notBefore", null); + module.registerMethod("notAfter", null); + module.registerMethod("checkend", null); + module.registerMethod("extensions_by_oid", null); + module.registerMethod("as_string", null); + module.registerMethod("DESTROY", "destroy", null); + module.registerMethod("__X509_cleanup", "destroy", null); + CryptOpenSSLX509Extension.initialize(); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_from_string(RuntimeArray args, int ctx) { + if (args.size() < 2) return fail("Usage: Crypt::OpenSSL::X509->new_from_string(string)"); + try { + return object(parse(args.get(1).toString())).getList(); + } catch (Exception e) { + return fail("Unable to parse X509 certificate: " + e.getMessage()); + } + } + + public static RuntimeList new_from_file(RuntimeArray args, int ctx) { + if (args.size() < 2) return fail("Usage: Crypt::OpenSSL::X509->new_from_file(file)"); + try { + return object(parse(java.nio.file.Files.readString(java.nio.file.Path.of(args.get(1).toString())))).getList(); + } catch (Exception e) { + return fail("Unable to parse X509 certificate file: " + e.getMessage()); + } + } + + static X509Certificate parse(String text) throws Exception { + try (PEMParser parser = new PEMParser(new StringReader(text))) { + Object value = parser.readObject(); + if (value instanceof X509CertificateHolder holder) { + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + } + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + return (X509Certificate) factory.generateCertificate( + new ByteArrayInputStream(text.getBytes(StandardCharsets.ISO_8859_1))); + } + + static RuntimeScalar object(X509Certificate certificate) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(certificate)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref; + } + + static X509Certificate certificate(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.type == RuntimeScalarType.JAVAOBJECT + && state.value instanceof X509Certificate certificate) return certificate; + throw new IllegalArgumentException("Invalid Crypt::OpenSSL::X509 object"); + } + + public static RuntimeList subject(RuntimeArray args, int ctx) { + return new RuntimeScalar(certificate(args.get(0)).getSubjectX500Principal().getName()).getList(); + } + + public static RuntimeList issuer(RuntimeArray args, int ctx) { + return new RuntimeScalar(certificate(args.get(0)).getIssuerX500Principal().getName()).getList(); + } + + public static RuntimeList notBefore(RuntimeArray args, int ctx) { + return new RuntimeScalar(OPENSSL_TIME.format(certificate(args.get(0)).getNotBefore().toInstant())).getList(); + } + + public static RuntimeList notAfter(RuntimeArray args, int ctx) { + return new RuntimeScalar(OPENSSL_TIME.format(certificate(args.get(0)).getNotAfter().toInstant())).getList(); + } + + public static RuntimeList checkend(RuntimeArray args, int ctx) { + long seconds = args.size() > 1 ? args.get(1).getLong() : 0; + Date when = new Date(System.currentTimeMillis() + seconds * 1000L); + X509Certificate cert = certificate(args.get(0)); + boolean invalid = when.before(cert.getNotBefore()) || when.after(cert.getNotAfter()); + return new RuntimeScalar(invalid).getList(); + } + + public static RuntimeList extensions_by_oid(RuntimeArray args, int ctx) { + try { + X509CertificateHolder holder = new X509CertificateHolder(certificate(args.get(0)).getEncoded()); + RuntimeHash hash = new RuntimeHash(); + Extensions extensions = holder.getExtensions(); + if (extensions != null) { + for (ASN1ObjectIdentifier oid : extensions.getExtensionOIDs()) { + hash.put(oid.getId(), CryptOpenSSLX509Extension.object(extensions.getExtension(oid))); + } + } + return hash.createAnonymousReference().getList(); + } catch (Exception e) { + return fail("Unable to read X509 extensions: " + e.getMessage()); + } + } + + public static RuntimeList as_string(RuntimeArray args, int ctx) { + try { + String base64 = java.util.Base64.getMimeEncoder(64, new byte[]{'\n'}) + .encodeToString(certificate(args.get(0)).getEncoded()); + return new RuntimeScalar("-----BEGIN CERTIFICATE-----\n" + base64 + + "\n-----END CERTIFICATE-----\n").getList(); + } catch (Exception e) { + return fail("Unable to encode X509 certificate: " + e.getMessage()); + } + } + + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + static RuntimeList fail(String message) { + return WarnDie.die(new RuntimeScalar(message), new RuntimeScalar("\n")).getList(); + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java new file mode 100644 index 0000000000..6752749f21 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CryptOpenSSLX509Extension.java @@ -0,0 +1,63 @@ +package org.perlonjava.runtime.perlmodule; + +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.runtimetypes.*; + +/** Methods on Crypt::OpenSSL::X509::Extension objects. */ +public class CryptOpenSSLX509Extension extends PerlModuleBase { + private static final String CLASS_NAME = "Crypt::OpenSSL::X509::Extension"; + private static final String STATE_KEY = "_x509_extension"; + + public CryptOpenSSLX509Extension() { super(CLASS_NAME, false); } + + public static void initialize() throws NoSuchMethodException { + CryptOpenSSLX509Extension module = new CryptOpenSSLX509Extension(); + module.registerMethod("value", null); + module.registerMethod("to_string", null); + module.registerMethod("critical", null); + module.registerMethod("extendedKeyUsage", null); + module.registerMethod("DESTROY", "destroy", null); + } + + static RuntimeScalar object(Extension extension) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(extension)); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref; + } + + private static Extension extension(RuntimeScalar self) { + RuntimeScalar state = self.hashDeref().get(STATE_KEY); + if (state != null && state.value instanceof Extension extension) return extension; + throw new IllegalArgumentException("Invalid X509 extension object"); + } + + public static RuntimeList value(RuntimeArray args, int ctx) { + return new RuntimeScalar("#" + java.util.HexFormat.of().withUpperCase() + .formatHex(extension(args.get(0)).getExtnValue().getOctets())).getList(); + } + + public static RuntimeList to_string(RuntimeArray args, int ctx) { + Extension extension = extension(args.get(0)); + if (Extension.extendedKeyUsage.equals(extension.getExtnId())) { + KeyPurposeId[] usages = ExtendedKeyUsage.getInstance(extension.getParsedValue()).getUsages(); + StringBuilder out = new StringBuilder(); + for (KeyPurposeId usage : usages) { + if (!out.isEmpty()) out.append(", "); + out.append(usage.getId()); + } + return new RuntimeScalar(out.toString()).getList(); + } + return value(args, ctx); + } + + public static RuntimeList extendedKeyUsage(RuntimeArray args, int ctx) { return to_string(args, ctx); } + public static RuntimeList critical(RuntimeArray args, int ctx) { + return new RuntimeScalar(extension(args.get(0)).isCritical()).getList(); + } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java b/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java index 49c18c7f23..a535aee820 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/DynaLoader.java @@ -28,6 +28,7 @@ public static void initialize() { // and the modules fall through to their pure-Perl implementations. dynaLoader.registerMethod("dl_findfile", "dl_empty", null); dynaLoader.registerMethod("dl_load_file", "dl_empty", null); + dynaLoader.registerMethod("dl_load_flags", "dl_load_flags", null); dynaLoader.registerMethod("dl_find_symbol", "dl_empty", null); dynaLoader.registerMethod("dl_find_symbol_anywhere", "dl_empty", null); dynaLoader.registerMethod("dl_install_xsub", "dl_empty", null); @@ -74,6 +75,10 @@ public static RuntimeList dl_empty(RuntimeArray args, int ctx) { return new RuntimeList(); } + public static RuntimeList dl_load_flags(RuntimeArray args, int ctx) { + return new RuntimeScalar(0).getList(); + } + public static RuntimeList dl_error(RuntimeArray args, int ctx) { return new RuntimeScalar("DynaLoader is not supported in PerlOnJava").getList(); } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java b/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java index ddb6a5ca1d..3daeb0d741 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/HttpTiny.java @@ -102,11 +102,27 @@ public static RuntimeList request(RuntimeArray args, int ctx) throws Exception { responseMap.put("headers", responseHeaders.createReference()); return responseMap.createReference().getList(); - } catch (IOException | InterruptedException e) { - throw new RuntimeException("HTTP request failed", e); + } catch (IOException e) { + return transportFailure(url, e).getList(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return transportFailure(url, e).getList(); } } + private static RuntimeScalar transportFailure(String url, Exception error) { + RuntimeHash response = new RuntimeHash(); + response.put("success", new RuntimeScalar(false)); + response.put("status", new RuntimeScalar(599)); + response.put("reason", new RuntimeScalar("Internal Exception")); + String message = error.getMessage(); + response.put("content", new RuntimeScalar( + message == null || message.isEmpty() ? error.getClass().getSimpleName() : message)); + response.put("url", new RuntimeScalar(url)); + response.put("headers", new RuntimeHash().createReference()); + return response.createReference(); + } + private static String getStatusReason(int statusCode) { return switch (statusCode) { case 100 -> "Continue"; @@ -329,4 +345,4 @@ public static RuntimeList mirror(RuntimeArray args, int ctx) throws Exception { return responseMap.createReference().getList(); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java b/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java index 8fbb1eda80..41b2199c20 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Internals.java @@ -72,6 +72,7 @@ public static void initialize() { internals.registerMethod("jperl_cv_start_location", "jperlCvStartLocation", "$"); internals.registerMethod("jperl_cv_deparse_info", "jperlCvDeparseInfo", "$"); internals.registerMethod("jperl_cv_is_constant", "jperlCvIsConstant", "$"); + internals.registerMethod("jperl_mark_pseudo_constant", "jperlMarkPseudoConstant", "$$"); internals.registerMethod("jperl_end_av_ref", "jperlEndAvRef", ""); internals.registerMethod("jperl_b_object_2svref", "jperlBObject2svref", "$"); internals.registerMethod("jperl_set_closed_over", "jperlSetClosedOver", null); @@ -982,4 +983,21 @@ public static RuntimeList jperlCvIsConstant(RuntimeArray args, int ctx) { boolean isConst = code.constantValue != null || code.isConstantCv; return new RuntimeScalar(isConst ? 1 : 0).getList(); } + + /** Preserve constant.pm's scalar-reference proxy in the stash hash view. */ + public static RuntimeList jperlMarkPseudoConstant(RuntimeArray args, int ctx) { + if (args.size() >= 2) { + RuntimeScalar proxy = args.get(1); + if (proxy.type == RuntimeScalarType.REFERENCE + && proxy.value instanceof RuntimeScalar target) { + proxy = target; + } + if (proxy.type == RuntimeScalarType.READONLY_SCALAR + && proxy.value instanceof RuntimeScalar target) { + proxy = target; + } + GlobalVariable.setGlobalPseudoConstant(args.get(0).toString(), proxy); + } + return new RuntimeList(); + } } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java new file mode 100644 index 0000000000..ff8063ce7b --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealDecoder.java @@ -0,0 +1,101 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Decoder; +import com.booking.sereal.DecoderOptions; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.charset.StandardCharsets; + +/** Sereal::Decoder XS compatibility layer backed by Sereal's official Java codec. */ +public class SerealDecoder extends PerlModuleBase { + private static final String CLASS_NAME = "Sereal::Decoder"; + private static final String STATE_KEY = "_sereal_decoder"; + + public SerealDecoder() { super(CLASS_NAME, false); } + + public static void initialize() { + SerealDecoder module = new SerealDecoder(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("decode", "decode_sereal", null); + module.registerMethod("decode_sereal", null); + module.registerMethod("sereal_decode_with_object", null); + module.registerMethod("looks_like_sereal", null); + module.registerMethod("scalar_looks_like_sereal", null); + module.registerMethod("bytes_consumed", null); + module.registerMethod("flags", null); + module.registerMethod("DESTROY", "destroy", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + private static Decoder newDecoder() { + return new Decoder(new DecoderOptions().perlReferences(true).preserveUndef(true)); + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(newDecoder())); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } + + public static RuntimeList decode_sereal(RuntimeArray args, int ctx) { + int blobIndex = isObject(args) ? 1 : 0; + Decoder decoder = isObject(args) ? state(args.get(0)) : newDecoder(); + return decode(decoder, args, blobIndex); + } + + public static RuntimeList sereal_decode_with_object(RuntimeArray args, int ctx) { + return decode(state(args.get(0)), args, 1); + } + + private static RuntimeList decode(Decoder decoder, RuntimeArray args, int blobIndex) { + if (args.size() <= blobIndex) return WarnDie.die( + new RuntimeScalar("Usage: decode_sereal(blob)"), new RuntimeScalar("\n")).getList(); + try { + byte[] bytes = args.get(blobIndex).toString().getBytes(StandardCharsets.ISO_8859_1); + decoder.setData(bytes); + return SerealRuntimeConverter.fromJava(decoder.decode()).getList(); + } catch (Exception e) { + return WarnDie.die(new RuntimeScalar("Sereal decode failed: " + e.getMessage()), + new RuntimeScalar("\n")).getList(); + } + } + + public static RuntimeList looks_like_sereal(RuntimeArray args, int ctx) { + int index = isObject(args) ? 1 : 0; + return likely(args, index); + } + + public static RuntimeList scalar_looks_like_sereal(RuntimeArray args, int ctx) { return likely(args, 0); } + + private static RuntimeList likely(RuntimeArray args, int index) { + if (args.size() <= index) return new RuntimeScalar(0).getList(); + String value = args.get(index).toString(); + boolean match = value.length() >= 4 && value.charAt(0) == '=' + && (value.substring(1, 4).equals("srl") || value.charAt(1) == (char) 0xF3); + return new RuntimeScalar(match).getList(); + } + + public static RuntimeList bytes_consumed(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList flags(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static boolean isObject(RuntimeArray args) { + return !args.isEmpty() && RuntimeScalarType.blessedId(args.get(0)) != 0; + } + + private static Decoder state(RuntimeScalar self) { + RuntimeScalar stored = self.hashDeref().get(STATE_KEY); + if (stored != null && stored.type == RuntimeScalarType.JAVAOBJECT && stored.value instanceof Decoder decoder) { + return decoder; + } + WarnDie.die(new RuntimeScalar("Invalid Sereal::Decoder object"), new RuntimeScalar("\n")); + return null; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java new file mode 100644 index 0000000000..0b0179ea39 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealEncoder.java @@ -0,0 +1,89 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Encoder; +import com.booking.sereal.EncoderOptions; +import org.perlonjava.runtime.operators.ReferenceOperators; +import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.runtimetypes.*; + +/** Sereal::Encoder XS compatibility layer backed by Sereal's official Java codec. */ +public class SerealEncoder extends PerlModuleBase { + private static final String CLASS_NAME = "Sereal::Encoder"; + private static final String STATE_KEY = "_sereal_encoder"; + + public SerealEncoder() { super(CLASS_NAME, false); } + + public static void initialize() { + SerealEncoder module = new SerealEncoder(); + try { + module.registerMethod("new", "new_", null); + module.registerMethod("encode", "encode_sereal", null); + module.registerMethod("encode_sereal", null); + module.registerMethod("sereal_encode_with_object", null); + module.registerMethod("encode_sereal_with_header_data", null); + module.registerMethod("flags", null); + module.registerMethod("DESTROY", "destroy", null); + } catch (NoSuchMethodException e) { + throw new IllegalStateException(e); + } + } + + public static RuntimeList new_(RuntimeArray args, int ctx) { + RuntimeHash hash = new RuntimeHash(); + hash.put(STATE_KEY, new RuntimeScalar(new Encoder(options()))); + RuntimeScalar ref = hash.createAnonymousReference(); + ReferenceOperators.bless(ref, new RuntimeScalar(CLASS_NAME)); + return ref.getList(); + } + + public static RuntimeList encode_sereal(RuntimeArray args, int ctx) { + int valueIndex = isObject(args) ? 1 : 0; + Encoder encoder = isObject(args) ? state(args.get(0)) : new Encoder(options()); + return encode(encoder, args, valueIndex, -1); + } + + public static RuntimeList sereal_encode_with_object(RuntimeArray args, int ctx) { + return encode(state(args.get(0)), args, 1, -1); + } + + public static RuntimeList encode_sereal_with_header_data(RuntimeArray args, int ctx) { + return encode(new Encoder(options()), args, 0, 1); + } + + private static RuntimeList encode(Encoder encoder, RuntimeArray args, int valueIndex, int headerIndex) { + if (args.size() <= valueIndex) return WarnDie.die( + new RuntimeScalar("Usage: encode_sereal(value)"), new RuntimeScalar("\n")).getList(); + try { + Object value = SerealRuntimeConverter.toJava(args.get(valueIndex)); + if (headerIndex >= 0 && args.size() > headerIndex) { + encoder.write(value, SerealRuntimeConverter.toJava(args.get(headerIndex))); + } else { + encoder.write(value); + } + return SerealRuntimeConverter.byteScalar(encoder.getData()).getList(); + } catch (Exception e) { + return WarnDie.die(new RuntimeScalar("Sereal encode failed: " + e.getMessage()), + new RuntimeScalar("\n")).getList(); + } + } + + public static RuntimeList flags(RuntimeArray args, int ctx) { return new RuntimeScalar(0).getList(); } + public static RuntimeList destroy(RuntimeArray args, int ctx) { return new RuntimeList(); } + + private static boolean isObject(RuntimeArray args) { + return !args.isEmpty() && RuntimeScalarType.blessedId(args.get(0)) != 0; + } + + private static EncoderOptions options() { + return new EncoderOptions().protocolVersion(4).perlReferences(true); + } + + private static Encoder state(RuntimeScalar self) { + RuntimeScalar stored = self.hashDeref().get(STATE_KEY); + if (stored != null && stored.type == RuntimeScalarType.JAVAOBJECT && stored.value instanceof Encoder encoder) { + return encoder; + } + WarnDie.die(new RuntimeScalar("Invalid Sereal::Encoder object"), new RuntimeScalar("\n")); + return null; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java b/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java new file mode 100644 index 0000000000..9765a6e97d --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/perlmodule/SerealRuntimeConverter.java @@ -0,0 +1,146 @@ +package org.perlonjava.runtime.perlmodule; + +import com.booking.sereal.Latin1String; +import com.booking.sereal.PerlObject; +import com.booking.sereal.PerlReference; +import com.booking.sereal.PerlUndef; +import org.perlonjava.runtime.runtimetypes.*; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.perlonjava.runtime.runtimetypes.RuntimeScalarType.*; + +/** Converts PerlOnJava runtime values to and from the official Sereal Java model. */ +final class SerealRuntimeConverter { + private SerealRuntimeConverter() {} + + static Object toJava(RuntimeScalar scalar) { + return toJavaScalar(scalar, new IdentityHashMap<>()); + } + + private static Object toJavaScalar(RuntimeScalar scalar, IdentityHashMap seen) { + while (scalar.type == READONLY_SCALAR && scalar.value instanceof RuntimeScalar inner) { + scalar = inner; + } + return switch (scalar.type) { + case UNDEF -> PerlUndef.CANONICAL; + case INTEGER -> scalar.getLong(); + case DOUBLE -> scalar.getDouble(); + case BOOLEAN -> scalar.getBoolean(); + case BYTE_STRING -> new Latin1String(scalar.toString()); + case STRING, VSTRING, DUALVAR -> scalar.toString(); + case ARRAYREFERENCE -> referenceValue(scalar, scalar.arrayDeref(), seen); + case HASHREFERENCE -> referenceValue(scalar, scalar.hashDeref(), seen); + case REFERENCE -> referenceValue(scalar, scalar.scalarDeref(), seen); + default -> scalar.toString(); + }; + } + + private static Object referenceValue(RuntimeScalar scalar, RuntimeBase referent, + IdentityHashMap seen) { + Object converted = seen.get(referent); + if (converted == null) { + if (referent instanceof RuntimeArray array) { + List list = new ArrayList<>(array.size()); + PerlReference reference = new PerlReference(list); + seen.put(referent, reference); + for (RuntimeScalar value : array) list.add(toJavaScalar(value, seen)); + converted = reference; + } else if (referent instanceof RuntimeHash hash) { + Map map = new LinkedHashMap<>(); + PerlReference reference = new PerlReference(map); + seen.put(referent, reference); + for (Map.Entry entry : hash.elements.entrySet()) { + map.put(entry.getKey(), toJavaScalar(entry.getValue(), seen)); + } + converted = reference; + } else if (referent instanceof RuntimeScalar target) { + PerlReference placeholder = new PerlReference(null); + seen.put(referent, placeholder); + placeholder.setValue(toJavaScalar(target, seen)); + converted = placeholder; + } else { + converted = new PerlReference(referent.toString()); + seen.put(referent, converted); + } + } + int blessId = RuntimeScalarType.blessedId(scalar); + if (blessId != 0) { + String className = NameNormalizer.getBlessStr(blessId); + converted = new PerlObject(className == null ? "__ANON__" : className, converted); + } + return converted; + } + + static RuntimeScalar fromJava(Object value) { + return fromJava(value, new IdentityHashMap<>()); + } + + private static RuntimeScalar fromJava(Object value, IdentityHashMap seen) { + if (value == null || value instanceof PerlUndef) return new RuntimeScalar(); + RuntimeScalar prior = seen.get(value); + if (prior != null) return prior; + if (value instanceof PerlObject object) { + RuntimeScalar result = fromJava(object.getData(), seen); + if (RuntimeScalarType.isReference(result) && result.value instanceof RuntimeBase base) { + base.blessId = NameNormalizer.getBlessId(object.getName()); + } + return result; + } + if (value instanceof PerlReference reference) { + Object referent = reference.getValue(); + RuntimeScalar target = fromJava(referent, seen); + // Java's PerlReference wraps both aggregate references and scalar + // references. Aggregate conversion already returns a Perl reference; + // only scalar referents need one more level of indirection. + RuntimeScalar result = !(referent instanceof PerlReference) + && (target.type == ARRAYREFERENCE || target.type == HASHREFERENCE) + ? target : target.createReference(); + seen.put(value, result); + return result; + } + if (value instanceof Map map) { + RuntimeHash hash = new RuntimeHash(); + RuntimeScalar result = hash.createAnonymousReference(); + seen.put(value, result); + for (Map.Entry entry : map.entrySet()) { + hash.put(entry.getKey().toString(), fromJava(entry.getValue(), seen)); + } + return result; + } + if (value instanceof List list) { + RuntimeArray array = new RuntimeArray(); + RuntimeScalar result = array.createAnonymousReference(); + seen.put(value, result); + for (Object item : list) RuntimeArray.push(array, fromJava(item, seen)); + return result; + } + if (value instanceof Object[] arrayValue) { + RuntimeArray array = new RuntimeArray(); + RuntimeScalar result = array.createAnonymousReference(); + seen.put(value, result); + for (Object item : arrayValue) RuntimeArray.push(array, fromJava(item, seen)); + return result; + } + if (value instanceof byte[] bytes) return byteScalar(bytes); + if (value instanceof Latin1String latin1) return byteScalar(latin1.getBytes()); + if (value instanceof Boolean bool) return new RuntimeScalar(bool); + if (value instanceof Byte || value instanceof Short || value instanceof Integer) { + return new RuntimeScalar(((Number) value).intValue()); + } + if (value instanceof Long number) return new RuntimeScalar(number); + if (value instanceof Number number) return new RuntimeScalar(number.doubleValue()); + return new RuntimeScalar(value.toString()); + } + + static RuntimeScalar byteScalar(byte[] bytes) { + RuntimeScalar scalar = new RuntimeScalar(new String(bytes, StandardCharsets.ISO_8859_1)); + scalar.type = BYTE_STRING; + return scalar; + } +} diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java b/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java index 88e5008ca7..7e8b34c2aa 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/TimeHiRes.java @@ -35,6 +35,7 @@ public static void initialize() { module.registerMethod("time", ""); module.registerMethod("sleep", null); module.registerMethod("alarm", null); + module.registerMethod("ualarm", null); } catch (NoSuchMethodException e) { System.err.println("Warning: Missing Time::HiRes method: " + e.getMessage()); } @@ -88,4 +89,11 @@ public static RuntimeList alarm(RuntimeArray args, int ctx) { // Implement alarm functionality if needed return new RuntimeScalar(0).getList(); } + + public static RuntimeList ualarm(RuntimeArray args, int ctx) { + // Match the existing alarm compatibility behavior. Registering the + // function is important even where JVM signal delivery is unavailable: + // callers commonly use ualarm(0) to cancel an optional timeout. + return new RuntimeScalar(0).getList(); + } } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java b/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java index 065ea3a8d4..4792024604 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/XMLLibXML.java @@ -14,6 +14,8 @@ import javax.xml.transform.*; import javax.xml.transform.dom.*; import javax.xml.transform.stream.*; +import javax.xml.validation.Schema; +import javax.xml.validation.Validator; import javax.xml.xpath.*; import org.w3c.dom.*; import org.xml.sax.*; @@ -40,6 +42,7 @@ public class XMLLibXML extends PerlModuleBase { private static final String OPTS_KEY = "_parser_opts"; private static final String XPC_KEY = "_xpc_state"; private static final String READER_KEY = "_reader_state"; + private static final String RELAXNG_KEY = "_relaxng_schema"; /** Pseudo-namespace for functions registered without namespace ("{}name"). */ private static final String NONS_NS = "http://perlonjava.org/xpc-nons"; @@ -422,6 +425,15 @@ public static void initialize() { module.registerMethodInPackage(xpePkg, "new", "xpeNew"); module.registerMethodInPackage(xpePkg, "expression", "xpeExpression"); + // XML::LibXML::RelaxNG. Jing supplies the standard JAXP RELAX NG + // provider; the compiled schema is retained in the blessed Perl + // object just as DOM nodes retain their backing Java object. + String rngPkg = "XML::LibXML::RelaxNG"; + module.registerMethodInPackage(rngPkg, "parse_buffer", "relaxNGParseBuffer"); + module.registerMethodInPackage(rngPkg, "parse_location", "relaxNGParseLocation"); + module.registerMethodInPackage(rngPkg, "parse_document", "relaxNGParseDocument"); + module.registerMethodInPackage(rngPkg, "validate", "relaxNGValidate"); + setupISA(); } catch (NoSuchMethodException e) { @@ -506,6 +518,70 @@ public static Node getNode(RuntimeScalar self) { throw new RuntimeException("Not a valid XML::LibXML node (missing " + NODE_KEY + " key)"); } + private static RuntimeScalar wrapRelaxNG(Schema schema) { + RuntimeHash hash = new RuntimeHash(); + hash.put(RELAXNG_KEY, new RuntimeScalar(schema)); + RuntimeScalar ref = hash.createReferenceWithTrackedElements(); + return ReferenceOperators.bless(ref, new RuntimeScalar("XML::LibXML::RelaxNG")); + } + + private static Schema getRelaxNG(RuntimeScalar self) { + RuntimeScalar state = self.hashDerefRaw().get(RELAXNG_KEY); + if (state != null && state.type == RuntimeScalarType.JAVAOBJECT + && state.value instanceof Schema schema) return schema; + throw new PerlDieException(new RuntimeScalar("Invalid XML::LibXML::RelaxNG object\n")); + } + + private static Schema compileRelaxNG(Source source) throws SAXException { + // Jing does not publish a META-INF JAXP provider entry, so instantiate + // its standard XML-syntax factory explicitly. + return new com.thaiopensource.relaxng.jaxp.XMLSyntaxSchemaFactory().newSchema(source); + } + + private static RuntimeList relaxNGError(Exception error) { + Throwable cause = error; + while (cause.getCause() != null && cause.getCause() != cause) cause = cause.getCause(); + String message = cause.getMessage(); + if (message == null || message.isEmpty()) message = cause.toString(); + throw new PerlDieException(new RuntimeScalar(message + "\n")); + } + + public static RuntimeList relaxNGParseBuffer(RuntimeArray args, int ctx) { + try { + return wrapRelaxNG(compileRelaxNG(new StreamSource(new StringReader(args.get(1).toString())))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGParseLocation(RuntimeArray args, int ctx) { + try { + return wrapRelaxNG(compileRelaxNG(new StreamSource(args.get(1).toString()))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGParseDocument(RuntimeArray args, int ctx) { + try { + String schemaText = serializeNode(getNode(args.get(1)), false, true); + return wrapRelaxNG(compileRelaxNG(new StreamSource(new StringReader(schemaText)))).getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + + public static RuntimeList relaxNGValidate(RuntimeArray args, int ctx) { + try { + Validator validator = getRelaxNG(args.get(0)).newValidator(); + String documentText = serializeNode(getNode(args.get(1)), false, true); + validator.validate(new StreamSource(new StringReader(documentText))); + return scalarZero.getList(); + } catch (Exception e) { + return relaxNGError(e); + } + } + /** * Update the Java Node stored in a Perl XML::LibXML node object. * Needed after Document.renameNode() which may return a new Node instance. diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java index f73cc97a31..cb6863bd44 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalVariable.java @@ -1217,6 +1217,18 @@ public static boolean hasGlobalPseudoConstant(String key) { return resolvedKey != key && globalPseudoConstants().containsKey(resolvedKey); } + public static RuntimeScalar getGlobalPseudoConstant(String key) { + if (key == null) { + return null; + } + RuntimeScalar scalar = globalPseudoConstants().get(key); + if (scalar != null) { + return scalar; + } + String resolvedKey = resolveAliasedFqn(key); + return resolvedKey != key ? globalPseudoConstants().get(resolvedKey) : null; + } + /** * Retrieves a global array by its key, initializing it if necessary. * diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java index 804b526538..afd80c117c 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java @@ -9,6 +9,7 @@ Handling pipes (e.g., |- or -| modes). */ import org.perlonjava.runtime.io.*; +import org.perlonjava.runtime.HintHashRegistry; import org.perlonjava.runtime.operators.IOOperator; import org.perlonjava.runtime.operators.WarnDie; import org.perlonjava.runtime.perlmodule.Warnings; @@ -720,7 +721,7 @@ public static RuntimeIO open(String fileName, String mode) { // Use SeekableJarHandle to support seek operations (needed by Module::Metadata) fh.ioHandle = new SeekableJarHandle(is); addHandle(fh.ioHandle); - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } return fh; @@ -761,7 +762,7 @@ public static RuntimeIO open(String fileName, String mode) { } // Apply any I/O layers - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } @@ -772,7 +773,16 @@ public static RuntimeIO open(String fileName, String mode) { return fh; } - private boolean applyOpenLayers(String ioLayers) { + private boolean applyOpenLayers(String ioLayers, String mode) { + if (ioLayers == null || ioLayers.isEmpty()) { + Map hints = HintHashRegistry.getCurrentCallSiteHintHash(); + String key = mode != null && mode.contains(">") ? "open>" : "open<"; + String lexicalLayer = hints == null ? null : hints.get(key); + // open.pm defaults are lexical compiler hints. An unrelated + // package's use open must not leak through the process-global + // ${^OPEN} compatibility scalar into this call site. + ioLayers = lexicalLayer == null || lexicalLayer.isEmpty() ? ":raw" : lexicalLayer; + } RuntimeScalar status = binmode(ioLayers); if (status.getBoolean()) { return true; @@ -879,7 +889,7 @@ public static RuntimeIO open(RuntimeScalar scalarRef, String mode) { // PerlIO::scalar is not a real OS file descriptor, so a plain scalar // open should not inherit the platform text layer such as Windows :crlf. if (!ioLayers.isEmpty()) { - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } } @@ -972,7 +982,7 @@ public static RuntimeIO openPipe(RuntimeList runtimeList) { // Apply any I/O layers (excluding the already-processed :noshell) if (!ioLayers.isEmpty()) { - if (!fh.applyOpenLayers(ioLayers)) { + if (!fh.applyOpenLayers(ioLayers, mode)) { return null; } } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java index ebd702499b..a3104b3190 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeStashEntry.java @@ -27,7 +27,7 @@ public RuntimeStashEntry(String globName, boolean isDefined) { // assigned reference even though globDeref() still provides the // symbol's complete typeglob. type = REFERENCE; - value = GlobalVariable.getGlobalVariable(globName); + value = GlobalVariable.getGlobalPseudoConstant(globName); } // System.out.println("Stash Entry create: " + globName + " " + isDefined); } @@ -43,6 +43,23 @@ public RuntimeGlob createDetachedCopy() { return this; } + /** + * A constant.pm proxy is a scalar-reference value stored directly in the + * stash hash, not a typeglob value. Taking a reference to that hash value + * must therefore preserve the scalar shape instead of forcing the + * GLOBREFERENCE representation used by ordinary stash entries. + */ + @Override + public RuntimeScalar createReference() { + if (GlobalVariable.hasGlobalPseudoConstant(this.globName)) { + RuntimeScalar reference = new RuntimeScalar(); + reference.type = REFERENCE; + reference.value = this; + return reference; + } + return super.createReference(); + } + /** * Override globDeref for stash entries to return a plain RuntimeGlob. * This ensures that *{$stash{name}} = \$value goes through RuntimeGlob.set() @@ -128,6 +145,15 @@ public RuntimeScalar set(RuntimeScalar value) { code.constantValue = value.scalarDeref().getList(); GlobalVariable.defineGlobalCodeRef(this.globName).set( new RuntimeScalar(code)); + // Perl 5 exposes a constant.pm entry through the stash hash + // as the assigned scalar-reference proxy, even though the + // same symbol also has a callable CODE slot. Keep the proxy + // metadata alongside our concrete constant CV so stash + // introspection can distinguish constants from ordinary + // subroutines. + GlobalVariable.setGlobalPseudoConstant(this.globName, targetScalar); + this.type = REFERENCE; + this.value = targetScalar; notifyCodeSlotChanged(); } else { // Ordinary scalar references alias the stash scalar slot, diff --git a/src/main/perl/lib/DynaLoader.pm b/src/main/perl/lib/DynaLoader.pm index 2fb1f6f10e..0ab83947d3 100644 --- a/src/main/perl/lib/DynaLoader.pm +++ b/src/main/perl/lib/DynaLoader.pm @@ -41,6 +41,9 @@ BEGIN { unless (defined &dl_load_file) { *dl_load_file = sub { return }; } + unless (defined &dl_load_flags) { + *dl_load_flags = sub { return 0 }; + } unless (defined &dl_find_symbol) { *dl_find_symbol = sub { return }; } diff --git a/src/main/perl/lib/HTTP/Tiny.pm b/src/main/perl/lib/HTTP/Tiny.pm index e105ae7dcc..a01404ea89 100644 --- a/src/main/perl/lib/HTTP/Tiny.pm +++ b/src/main/perl/lib/HTTP/Tiny.pm @@ -227,15 +227,13 @@ sub _set_proxies { for my $sub_name ( qw/get head put post patch delete/ ) { my $req_method = uc $sub_name; - for my $accessor ( @attributes ) { - my $sym_ref = qualify_to_ref($sub_name, __PACKAGE__); - *{$sym_ref} = sub { - my ($self, $url, $args) = @_; - @_ == 2 || (@_ == 3 && ref $args eq 'HASH') - or _croak("Usage: \$http->$sub_name(URL, [HASHREF])\n"); - return $self->request($req_method, $url, $args || {}); - }; - } + my $sym_ref = qualify_to_ref($sub_name, __PACKAGE__); + *{$sym_ref} = sub { + my ($self, $url, $args) = @_; + @_ == 2 || (@_ == 3 && ref $args eq 'HASH') + or _croak("Usage: \$http->$sub_name(URL, [HASHREF])\n"); + return $self->request($req_method, $url, $args || {}); + }; } #pod =method post_form diff --git a/src/main/perl/lib/constant.pm b/src/main/perl/lib/constant.pm index f8d58e3996..b5aec754f6 100644 --- a/src/main/perl/lib/constant.pm +++ b/src/main/perl/lib/constant.pm @@ -165,6 +165,10 @@ sub import { Internals::SvREADONLY($scalar, 1); if (!exists $symtab->{$name}) { $symtab->{$name} = \$scalar; + # PerlOnJava stores a concrete CV for callable constant + # lookup; retain the scalar-reference proxy separately so + # stash introspection still sees Perl's PCS representation. + Internals::jperl_mark_pseudo_constant($full_name, \$scalar); ++$flush_mro->{$pkg}; } else { diff --git a/src/main/perl/lib/open.pm b/src/main/perl/lib/open.pm index 4a9d62c4da..8c21108eff 100644 --- a/src/main/perl/lib/open.pm +++ b/src/main/perl/lib/open.pm @@ -60,7 +60,6 @@ sub import { } } - ${^OPEN} = join( "\0", defined $in ? $in : '', defined $out ? $out : '' ); $^H{'open<'} = $in if defined $in; $^H{'open>'} = $out if defined $out; diff --git a/src/test/resources/unit/constant_stash_proxy.t b/src/test/resources/unit/constant_stash_proxy.t new file mode 100644 index 0000000000..e8c4a02008 --- /dev/null +++ b/src/test/resources/unit/constant_stash_proxy.t @@ -0,0 +1,27 @@ +use strict; +use warnings; +use Scalar::Util qw(reftype); +use Test::More tests => 6; + +{ + package ConstantStashProxy; + use constant { + NUMBER => 3, + ARRAY => [qw(a b c)], + }; +} + +for my $name (qw(NUMBER ARRAY)) { + no strict 'refs'; + my $stash = \%ConstantStashProxy::; + my $entry = \$stash->{$name}; + + ok(reftype(${$entry}) eq ($name eq 'ARRAY' ? 'REF' : 'SCALAR'), + "$name is exposed as a proxy constant in the stash"); + ok(defined &{"ConstantStashProxy::$name"}, + "$name remains available through the CODE slot"); +} + +is(ConstantStashProxy::NUMBER(), 3, 'scalar proxy constant remains callable'); +is_deeply(ConstantStashProxy::ARRAY(), [qw(a b c)], + 'reference proxy constant remains callable'); diff --git a/src/test/resources/unit/dynaloader_load_flags.t b/src/test/resources/unit/dynaloader_load_flags.t new file mode 100644 index 0000000000..540ee5b8e0 --- /dev/null +++ b/src/test/resources/unit/dynaloader_load_flags.t @@ -0,0 +1,10 @@ +use strict; +use warnings FATAL => 'all'; +use Test::More; + +require DynaLoader; + +ok(DynaLoader->can('dl_load_flags'), 'DynaLoader exposes dl_load_flags'); +is(DynaLoader->dl_load_flags, 0, 'default dynamic loader flags are zero'); + +done_testing; diff --git a/src/test/resources/unit/http_tiny_method_installation.t b/src/test/resources/unit/http_tiny_method_installation.t new file mode 100644 index 0000000000..ea8024387a --- /dev/null +++ b/src/test/resources/unit/http_tiny_method_installation.t @@ -0,0 +1,19 @@ +use strict; +use warnings FATAL => 'all'; +use Test::More; + +require HTTP::Tiny; + +{ + package Local::HTTP::Tiny; + our @ISA = ('HTTP::Tiny'); + sub request { return $_[1] } +} + +my $client = bless {}, 'Local::HTTP::Tiny'; +for my $method (qw(get head put post patch delete)) { + ok(HTTP::Tiny->can($method), "HTTP::Tiny installs $method once without warnings"); + is($client->$method('http://example.test/'), uc($method), "$method dispatches the correct HTTP verb"); +} + +done_testing; diff --git a/src/test/resources/unit/http_tiny_transport_failure.t b/src/test/resources/unit/http_tiny_transport_failure.t new file mode 100644 index 0000000000..770a8c4bd3 --- /dev/null +++ b/src/test/resources/unit/http_tiny_transport_failure.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 4; +use HTTP::Tiny; + +my $response = HTTP::Tiny->new(timeout => 1)->get('http://127.0.0.1:1/'); +ok(!$response->{success}, 'transport failure is not successful'); +is($response->{status}, 599, 'transport failure uses HTTP::Tiny status 599'); +is($response->{reason}, 'Internal Exception', 'transport failure has standard reason'); +ok(exists $response->{content}, 'transport failure includes diagnostic content'); diff --git a/src/test/resources/unit/open_pragma_lexical_io.t b/src/test/resources/unit/open_pragma_lexical_io.t new file mode 100644 index 0000000000..dc4dcabb72 --- /dev/null +++ b/src/test/resources/unit/open_pragma_lexical_io.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use Test::More tests => 2; +use Fcntl qw(O_WRONLY O_CREAT O_TRUNC); + +my $encoded_file = "/tmp/perlonjava-open-encoded-$$"; +my $raw_file = "/tmp/perlonjava-open-raw-$$"; + +{ + use open ':encoding(UTF-8)'; + sysopen(my $handle, $encoded_file, O_WRONLY | O_CREAT | O_TRUNC) or die $!; + print {$handle} chr(0xF3); + close $handle; +} + +{ + no open; + sysopen(my $handle, $raw_file, O_WRONLY | O_CREAT | O_TRUNC) or die $!; + print {$handle} chr(0xF3); + close $handle; +} + +sub file_hex { + my ($file) = @_; + open my $handle, '<:raw', $file or die $!; + local $/; + return unpack('H*', <$handle>); +} + +is(file_hex($encoded_file), 'c3b3', 'use open applies to its lexical scope'); +is(file_hex($raw_file), 'f3', 'open defaults do not leak into another scope'); +unlink $encoded_file, $raw_file; diff --git a/src/test/resources/unit/tied_isa_method_lookup.t b/src/test/resources/unit/tied_isa_method_lookup.t new file mode 100644 index 0000000000..9dd4b7eb9f --- /dev/null +++ b/src/test/resources/unit/tied_isa_method_lookup.t @@ -0,0 +1,33 @@ +use strict; +use warnings; +use Test::More tests => 4; + +{ + package TiedISA::Parent; + sub inherited { 42 } +} + +{ + package TiedISA::Array; + + sub TIEARRAY { + my ($class, $values) = @_; + return bless [ @{$values} ], $class; + } + + sub FETCHSIZE { scalar @{$_[0]} } + sub FETCH { $_[0]->[$_[1]] } +} + +{ + package TiedISA::Child; + our @ISA = ('TiedISA::Parent'); + my @parents = @ISA; + tie @ISA, 'TiedISA::Array', \@parents; +} + +is_deeply([ @TiedISA::Child::ISA ], ['TiedISA::Parent'], 'tied @ISA exposes its parent'); +ok(TiedISA::Child->isa('TiedISA::Parent'), 'isa follows tied @ISA'); +my $method = TiedISA::Child->can('inherited'); +ok($method, 'can finds a method through tied @ISA'); +is($method->(), 42, 'inherited method remains callable'); diff --git a/src/test/resources/unit/time_hires_ualarm.t b/src/test/resources/unit/time_hires_ualarm.t new file mode 100644 index 0000000000..5801277fd0 --- /dev/null +++ b/src/test/resources/unit/time_hires_ualarm.t @@ -0,0 +1,8 @@ +use strict; +use warnings; +use Test::More tests => 2; + +use Time::HiRes qw(ualarm); + +ok(defined &ualarm, 'ualarm is exported on request'); +cmp_ok(ualarm(0), '>=', 0, 'ualarm can cancel a timer'); diff --git a/src/test/resources/unit/xml_libxml_relaxng.t b/src/test/resources/unit/xml_libxml_relaxng.t new file mode 100644 index 0000000000..76e13f1aff --- /dev/null +++ b/src/test/resources/unit/xml_libxml_relaxng.t @@ -0,0 +1,22 @@ +use strict; +use warnings; +use Test::More; + +eval { require XML::LibXML; 1 } + or plan skip_all => 'XML::LibXML is not installed'; + +my $schema = XML::LibXML::RelaxNG->new(string => <<'RNG'); + + + +RNG + +ok($schema, 'compiled a RELAX NG schema'); +my $parser = XML::LibXML->new; +is($schema->validate($parser->parse_string('ok')), 0, + 'valid document returns zero'); +my $valid = eval { $schema->validate($parser->parse_string('')); 1 }; +ok(!$valid, 'invalid document throws'); +like($@, qr/(?:expect|allow|valid|element)/i, 'validation error is meaningful'); + +done_testing; From 60a0cc70b5f08c64c9b99960c9047398dbabc9d7 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 17:07:49 +0200 Subject: [PATCH 2/3] fix: preserve Windows default text I/O layers Keep lexical open pragma lookup scoped to the call site while retaining the platform default layer when no pragma applies. This restores CRLF translation for ordinary Windows text handles without reintroducing process-global ${^OPEN} leakage. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/jcpan-compiler-tooling-followup.md | 3 +++ .../java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/design/jcpan-compiler-tooling-followup.md b/dev/design/jcpan-compiler-tooling-followup.md index c687828a81..97929aa3a1 100644 --- a/dev/design/jcpan-compiler-tooling-followup.md +++ b/dev/design/jcpan-compiler-tooling-followup.md @@ -52,6 +52,9 @@ The following are not treated as PerlOnJava regressions because their current di - [x] Phase 5: final verification (2026-08-15) - Full `make` passed. - Mail::BIMI passed all 31 test programs and 82 assertions; network- and author-only tests skipped as expected. + - Preserved the platform-default I/O layer when no lexical `open` pragma is + active, including Windows `:crlf`, without consulting the leaked + process-global `${^OPEN}` value. ### Next Steps diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java index afd80c117c..8ae8e0c1d6 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeIO.java @@ -781,7 +781,10 @@ private boolean applyOpenLayers(String ioLayers, String mode) { // open.pm defaults are lexical compiler hints. An unrelated // package's use open must not leak through the process-global // ${^OPEN} compatibility scalar into this call site. - ioLayers = lexicalLayer == null || lexicalLayer.isEmpty() ? ":raw" : lexicalLayer; + // ":" asks binmode() for the platform default (:crlf on Windows, + // :raw elsewhere) without consulting the legacy process-global + // ${^OPEN} value. + ioLayers = lexicalLayer == null || lexicalLayer.isEmpty() ? ":" : lexicalLayer; } RuntimeScalar status = binmode(ioLayers); if (status.getBoolean()) { From 6229de2b9c784820ffaed8319563a68b891025a5 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 17:23:30 +0200 Subject: [PATCH 3/3] docs: record PR and CI completion Mark the jcpan compiler/tooling follow-up as ready for review after successful Ubuntu and Windows CI runs. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/jcpan-compiler-tooling-followup.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dev/design/jcpan-compiler-tooling-followup.md b/dev/design/jcpan-compiler-tooling-followup.md index 97929aa3a1..9ff51562ba 100644 --- a/dev/design/jcpan-compiler-tooling-followup.md +++ b/dev/design/jcpan-compiler-tooling-followup.md @@ -37,7 +37,7 @@ The following are not treated as PerlOnJava regressions because their current di ## Progress Tracking -### Current Status: implementation and local verification complete +### Current Status: implementation complete; PR ready for review ### Completed Phases @@ -55,11 +55,15 @@ The following are not treated as PerlOnJava regressions because their current di - Preserved the platform-default I/O layer when no lexical `open` pragma is active, including Windows `:crlf`, without consulting the leaked process-global `${^OPEN}` value. +- [x] Phase 6: pull request and CI (2026-08-15) + - Opened PR #962 from `fix/jcpan-compiler-tooling-followup`. + - GitHub Actions passed on Ubuntu and Windows; the Windows run specifically + confirmed the platform-default newline fix in `pipe_jperl_shebang.t`. ### Next Steps -1. Open the pull request. -2. Verify CI. +1. Review PR #962. +2. Merge after approval. ### Open Questions