Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ dependencies {
implementation libs.jing // RELAX NG validation for XML::LibXML
implementation libs.snakeyaml.engine // YAML processing
implementation libs.tomlj // TOML processing
implementation libs.zxing.core // QR encoding for Text::QRCode
implementation libs.commons.csv // CSV processing
implementation libs.commonmark // CommonMark rendering
implementation libs.commonmark.autolink // GFM autolinks
Expand Down
23 changes: 20 additions & 3 deletions dev/design/jcpan-compiler-tooling-followup.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ The following are not treated as PerlOnJava regressions because their current di

## Progress Tracking

### Current Status: implementation complete; PR ready for review
### Current Status: implementation complete; PR #963 CI green

### Completed Phases

Expand All @@ -59,11 +59,28 @@ The following are not treated as PerlOnJava regressions because their current di
- Opened PR #962 from `fix/jcpan-compiler-tooling-followup`.
- GitHub Actions passed on Ubuntu and Windows; the Windows run specifically
confirmed the platform-default newline fix in `pipe_jperl_shebang.t`.
- [x] Phase 7: post-merge warning cleanup (2026-08-15)
- Replaced deprecated `Zstd.decompressedSize` calls with
`Zstd.getFrameContentSize` and reject unknown or invalid frame sizes before
allocating decoder buffers.
- [x] Phase 8: core-suite regression audit (2026-08-15)
- Compared the reported core files against isolated current-master and
historical baseline builds instead of treating aggregate TAP counts as
deterministic.
- Fixed PVLV filehandle handling so `-t` follows the glob's IO slot without
stringification and `close` warnings retain ASCII and Unicode glob names.
- Restored `op/gv.t` from 253/304 to 255/304 and `uni/gv.t` from 176/206 to
178/206. `re/pat_advanced.t`, `re/pat_advanced_thr.t`, and
`test_pl/examples.t` reproduce their higher reported counts. The remaining
`japh/abigail.t` 109/130 result matches both current master and the
documented historical baseline, so it is not a PR #963 regression.
- Added a four-case system-Perl oracle and verified it with both PerlOnJava
backends; the full `make` suite passes.
- PR #963 CI passed on Ubuntu and Windows at commit `b48f504d8`.

### Next Steps

1. Review PR #962.
2. Merge after approval.
1. Hand the unified PR #963 back for user testing and review.

### Open Questions

Expand Down
2 changes: 2 additions & 0 deletions docs/reference/xs-compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ These modules have optimized Java implementations built into PerlOnJava:
| Exporter::Lexical | ExporterLexical.java | 0.02 | Installs lexical subs into the enclosing compile-time scope |
| HTML::Content::Extractor | HTMLContentExtractor.java | 0.17 | Uses jsoup's HTML5 parser with a small legacy tree-compatibility layer |
| Crypt::Twofish2 | CryptTwofish2.java | 1.03 | Uses BouncyCastle Twofish; ECB, stateful zero-IV CBC, and CFB1 |
| Math::Cephes | MathCephes.java | 0.5308 | Implements the normal and chi-square distribution functions used by CPAN statistics modules |
| Text::QRCode | TextQRCode.java | 0.05 | Uses ZXing in forced byte mode with libqrencode-compatible mask selection |

## Modules with PP Fallbacks or Shims

Expand Down
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ snappy-java = "1.1.10.8"
sqlite-jdbc = "3.53.2.1"
tomlj = "1.1.1"
zstd-jni = "1.5.7-8"
zxing = "3.5.4"

[libraries]
asm = { module = "org.ow2.asm:asm", version.ref = "asm" }
Expand All @@ -37,6 +38,7 @@ snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy-
sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" }
tomlj = { module = "org.tomlj:tomlj", version.ref = "tomlj" }
zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstd-jni" }
zxing-core = { module = "com.google.zxing:core", version.ref = "zxing" }

[plugins]
cyclonedx = "org.cyclonedx.bom:2.3.0"
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/com/booking/sereal/Decoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,13 @@ private void uncompressZstd() throws SerealException {
int len = (int) read_varint();

byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len);
long decompressedSize = Zstd.decompressedSize(compressedData);
long decompressedSize = Zstd.getFrameContentSize(compressedData);
if (Zstd.isError(decompressedSize)) {
String message = decompressedSize == -1
? "Zstd frame content size is unknown"
: Zstd.getErrorName(decompressedSize);
throw new SerealException(message);
}

if (decompressedSize > this.maxSize) {
throw new SerealException("The expected uncompressed size is larger than the allowed maximum size");
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/com/booking/sereal/TokenDecoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,13 @@ private void uncompressZstd() throws SerealException {
int len = (int) readVarint();

byte[] compressedData = Arrays.copyOfRange(originalData.array, position, position + len);
long decompressedSize = Zstd.decompressedSize(compressedData);
long decompressedSize = Zstd.getFrameContentSize(compressedData);
if (Zstd.isError(decompressedSize)) {
String message = decompressedSize == -1
? "Zstd frame content size is unknown"
: Zstd.getErrorName(decompressedSize);
throw new SerealException(message);
}
if (decompressedSize > Integer.MAX_VALUE) {
throw new SerealException("Decompressed size exceeds integer MAX_VALUE: " + decompressedSize);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.google.zxing.qrcode.encoder;

import com.google.zxing.WriterException;
import com.google.zxing.common.BitArray;
import com.google.zxing.qrcode.decoder.ErrorCorrectionLevel;
import com.google.zxing.qrcode.decoder.Mode;
import com.google.zxing.qrcode.decoder.Version;

import java.nio.charset.Charset;

/** Accesses ZXing's package-level encoder primitives to force QR byte mode. */
public final class PerlOnJavaByteModeEncoder {
private PerlOnJavaByteModeEncoder() {
}

public static QRCode encode(String text, Charset charset, ErrorCorrectionLevel level,
int requestedVersion, int mask) throws WriterException {
BitArray data = new BitArray();
Encoder.append8BitBytes(text, data, charset);

Version version = requestedVersion > 0
? Version.getVersionForNumber(requestedVersion)
: smallestVersion(data, level);
BitArray headerAndData = new BitArray();
Encoder.appendModeInfo(Mode.BYTE, headerAndData);
Encoder.appendLengthInfo(data.getSizeInBytes(), version, Mode.BYTE, headerAndData);
headerAndData.appendBitArray(data);

Version.ECBlocks ecBlocks = version.getECBlocksForLevel(level);
int totalCodewords = version.getTotalCodewords();
int dataCodewords = totalCodewords - ecBlocks.getTotalECCodewords();
if (!Encoder.willFit(headerAndData.getSize(), version, level)) {
throw new WriterException("Data too big for requested version");
}
Encoder.terminateBits(dataCodewords, headerAndData);
BitArray finalBits = Encoder.interleaveWithECBytes(
headerAndData, totalCodewords, dataCodewords, ecBlocks.getNumBlocks());

ByteMatrix matrix = new ByteMatrix(
version.getDimensionForVersion(), version.getDimensionForVersion());
MatrixUtil.buildMatrix(finalBits, level, version, mask, matrix);
QRCode result = new QRCode();
result.setMode(Mode.BYTE);
result.setECLevel(level);
result.setVersion(version);
result.setMaskPattern(mask);
result.setMatrix(matrix);
return result;
}

private static Version smallestVersion(BitArray data, ErrorCorrectionLevel level)
throws WriterException {
for (int number = 1; number <= 40; number++) {
Version version = Version.getVersionForNumber(number);
int bits = 4 + Mode.BYTE.getCharacterCountBits(version) + data.getSize();
if (Encoder.willFit(bits, version, level)) return version;
}
throw new WriterException("Data too big");
}
}
45 changes: 45 additions & 0 deletions src/main/java/org/perlonjava/frontend/parser/Variable.java
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ static Node parseArrayHashAccessInBraces(Parser parser, Node operand, boolean is
}
case "{" -> {
// Hash access
preprocessBackslashQuotesInInterpolatedHashAccess(parser, parser.tokenIndex);
operand = ParseInfix.parseInfixOperation(parser, operand, 0);
if (operand == null) {
throw new PerlCompilerException(parser.tokenIndex, "syntax error: Missing closing brace", parser.ctx.errorUtil);
Expand All @@ -534,6 +535,10 @@ static Node parseArrayHashAccessInBraces(Parser parser, Node operand, boolean is
switch (text) {
case "[", "{", "@*", "$*", "%*", "&*", "$#", "@", "%" -> {
// Dereference followed by access: $var->[0] or $var->{key}
if (text.equals("{")) {
preprocessBackslashQuotesInInterpolatedHashAccess(
parser, parser.tokenIndex);
}
parser.tokenIndex = previousIndex; // Re-parse "->"
operand = ParseInfix.parseInfixOperation(parser, operand, 0);
if (operand == null) {
Expand Down Expand Up @@ -565,6 +570,41 @@ static Node parseArrayHashAccessInBraces(Parser parser, Node operand, boolean is
return operand;
}

/**
* Inside a double-quoted string, quotes delimiting an interpolated hash key
* are escaped for the outer string: {@code "$ref->{\"key\"}"}. The string
* lexer preserves those backslashes, but the embedded expression parser must
* see an ordinary quoted key.
*/
private static void preprocessBackslashQuotesInInterpolatedHashAccess(
Parser parser, int openingBraceIndex) {
if (!parser.preprocessBracedBackslashQuotesInInterpolation) {
return;
}

int scan = openingBraceIndex;
int braceLevel = 0;
while (scan < parser.tokens.size()) {
String text = parser.tokens.get(scan).text;
if ("{".equals(text)) {
braceLevel++;
scan++;
} else if ("}".equals(text)) {
braceLevel--;
if (braceLevel == 0) {
return;
}
scan++;
} else if ("\\".equals(text)
&& scan + 1 < parser.tokens.size()
&& "\"".equals(parser.tokens.get(scan + 1).text)) {
parser.tokens.remove(scan);
} else {
scan++;
}
}
}

/**
* Parses array and hash access operations following a variable.
*
Expand Down Expand Up @@ -640,6 +680,7 @@ static Node parseArrayHashAccess(Parser parser, Node operand, boolean isRegex) {
}
case "{" -> {
// Hash access
preprocessBackslashQuotesInInterpolatedHashAccess(parser, parser.tokenIndex);
int savedIndex = parser.tokenIndex;
Node result = null;
try {
Expand All @@ -664,6 +705,10 @@ static Node parseArrayHashAccess(Parser parser, Node operand, boolean isRegex) {
switch (text) {
case "[", "{", "@*", "$*", "%*", "&*", "$#", "@", "%" -> {
// Dereference followed by access: $var->[0] or $var->{key}
if (text.equals("{")) {
preprocessBackslashQuotesInInterpolatedHashAccess(
parser, parser.tokenIndex);
}
parser.tokenIndex = previousIndex; // Re-parse "->"
Node result = ParseInfix.parseInfixOperation(parser, operand, 0);
if (result == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
* 1. -R, -W, -X, -O (for real uid/gid) are not implemented due to lack of
* straightforward Java equivalents.
* <p>
* 2. -t (tty check) is not implemented as it's specific to file handles
* rather than file paths.
* 2. -t (tty check) is implemented for open file handles through their
* native or synthetic descriptor; path operands still return undef.
* <p>
* 3. -p, -S, -b, and -c are approximated using file names or paths, as Java
* doesn't provide direct equivalents.
Expand Down Expand Up @@ -349,37 +349,27 @@ public static RuntimeScalar fileTest(String operator, RuntimeScalar fileHandle)
return fileTest(operator, new RuntimeScalar(dirPath.toString()));
}
}
// Special handling for -t on standard streams (STDIN, STDOUT, STDERR)
// -t operates on the handle's IO slot, not on the name of the glob
// which happens to contain it. This matters for PVLVs and detached
// globs such as `$_ = *name; *$_ = *STDOUT{IO}`: stringifying or
// resolving the outer glob loses the real descriptor.
if (operator.equals("-t")) {
String globName = null;
if (fileHandle.value instanceof RuntimeGlob rg) {
globName = rg.globName;
} else if (fileHandle.value instanceof RuntimeIO rio) {
globName = rio.globName;
}
if (globName != null) {
int fd = -1;
if (globName.endsWith("::STDIN") || globName.equals("STDIN")) {
fd = 0;
} else if (globName.endsWith("::STDOUT") || globName.equals("STDOUT")) {
fd = 1;
} else if (globName.endsWith("::STDERR") || globName.equals("STDERR")) {
fd = 2;
}
if (fd >= 0) {
try {
boolean isTty = FFMPosix.get().isatty(fd) != 0;
getGlobalVariable("main::!").set(0);
return getScalarBoolean(isTty);
} catch (Exception e) {
// Fall back to System.console() check for fd 0
if (fd == 0) {
boolean isTty = System.console() != null;
getGlobalVariable("main::!").set(0);
return getScalarBoolean(isTty);
}
}
}
RuntimeScalar descriptor = fh.fileno();
if (!descriptor.getDefinedBoolean()) {
getGlobalVariable("main::!").set(9);
updateLastStat(fileHandle, false, 9);
return scalarUndef;
}
int fd = descriptor.getInt();
try {
boolean isTty = FFMPosix.get().isatty(fd) != 0;
getGlobalVariable("main::!").set(0);
return getScalarBoolean(isTty);
} catch (Exception e) {
// Preserve a defined false result for an open, non-terminal
// handle when the platform cannot perform isatty().
getGlobalVariable("main::!").set(0);
return scalarFalse;
}
}
// Fallback for non-file handles (pipes, sockets, etc.)
Expand Down
21 changes: 21 additions & 0 deletions src/main/java/org/perlonjava/runtime/operators/IOOperator.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.perlonjava.runtime.nativ.NativeUtils;
import org.perlonjava.runtime.nativ.ffm.FFMPosix;
import org.perlonjava.runtime.perlmodule.Socket;
import org.perlonjava.runtime.perlmodule.Warnings;
import org.perlonjava.runtime.runtimetypes.*;

import java.io.File;
Expand Down Expand Up @@ -922,6 +923,12 @@ public static RuntimeScalar close(int ctx, RuntimeBase... args) {

// Handle case where the filehandle is invalid/corrupted
if (fh == null) {
if (unopenedWarningsEnabled()) {
String name = filehandleShortName(handle);
String message = "close() on unopened filehandle"
+ (name == null || name.isEmpty() ? "" : " " + name);
WarnDie.warn(new RuntimeScalar(message), new RuntimeScalar(""));
}
// Return false (undef in boolean context) for invalid filehandle
return new RuntimeScalar();
}
Expand All @@ -933,6 +940,20 @@ public static RuntimeScalar close(int ctx, RuntimeBase... args) {
return fh.close();
}

private static boolean unopenedWarningsEnabled() {
return getGlobalVariable("main::" + Character.toString('W' - 'A' + 1)).getBoolean()
|| Warnings.warningManager.isWarningEnabled("unopened")
|| Warnings.warningManager.isWarningEnabled("all");
}

private static String filehandleShortName(RuntimeScalar handle) {
if (!(handle.value instanceof RuntimeGlob glob) || glob.globName == null) {
return null;
}
int separator = glob.globName.lastIndexOf("::");
return separator >= 0 ? glob.globName.substring(separator + 2) : glob.globName;
}

/**
* Prints the elements to the specified file handle according to the format string.
*
Expand Down
Loading
Loading