diff --git a/dev/modules/cpan_compiler_tooling_batch_20260814.md b/dev/modules/cpan_compiler_tooling_batch_20260814.md new file mode 100644 index 000000000..8b629bd46 --- /dev/null +++ b/dev/modules/cpan_compiler_tooling_batch_20260814.md @@ -0,0 +1,102 @@ +# CPAN compiler/tooling batch (2026-08-14) + +## Scope + +Investigate `Test::XML::Compare`, `Heap::Simple::Any`, +`Lingua::StanfordCoreNLP`, `CPAN::Index::API::Object::Author`, +`Chart::Sequence`, `Padre::Plugin::FormBuilder`, and +`Lingua::EN::Keywords`, preferring reusable compiler and tooling fixes over +distribution-specific preferences. + +## Progress Tracking + +### Current Status: PR #955 rebased and validated locally; CI pending + +### Completed Phases + +- [x] Phase 1: baseline and system-Perl comparison (2026-08-14) + - Confirmed portable PerlOnJava failures in `Test::XML::Compare`, + `CPAN::Index::API`, and the `Lingua::EN::Keywords` dependency chain. + - Excluded `Heap::Simple::Perl` and `Chart::Sequence` because their own test + suites or module load fail with system Perl. + - Classified `Lingua::StanfordCoreNLP` (Inline::Java) and + `Padre::Plugin::FormBuilder` (Wx/native XS) as unavailable native stacks. +- [x] Phase 2: reduce initial root causes (2026-08-14) + - Reduced `Lingua::EN::Tagger` to unverifiable bytecode generated for the + closure installed by `Memoize`. + - Reduced `CPAN::Index::API` to zero-based virtual DATA positions preventing + `File::Slurp` from restoring a DATA handle after reading it. + - Preserved absolute DATA offsets both when updating the early placeholder + handle and when a module declares DATA under a later package name. +- [x] Phase 3: compiler/runtime implementation (2026-08-14) + - Made JVM list construction use the common `RuntimeBase` overload, avoiding + invalid bytecode when context conversion changes a statically inferred type. + - Made scalar-backed DATA handles expose source-absolute `tell`/`seek` + positions, including module package handles without an early placeholder. + - Matched Perl's caller line for a standalone first call in an `if`/`unless` + body in both backends. +- [x] Phase 4: CPAN verification (2026-08-14) + - `Test::XML::Compare`: PASS, 13 files / 67 tests. + - `CPAN::Index::API`: PASS, 7 files / 74 tests. + - `Lingua::EN::Keywords`: PASS, 2 tests; its Tagger, Memoize, and stemming + dependency chain built successfully. +- [x] Phase 5: regression validation (2026-08-14) + - New focused tests pass with system Perl and with both PerlOnJava backends. + - Full `make` suite passes. +- [x] Phase 6: pull request and CI (2026-08-14) + - Opened PR #955 from `fix/cpan-compiler-tooling-20260814`. + - Ubuntu and Windows CI jobs passed. +- [x] Phase 7: post-PR regression repair (2026-08-14) + - Rebased PR #955 onto `origin/master` at `4de2934e4`. + - Replaced the broad `RuntimeBase` list-add fallback with a verifier-safe + `RuntimeList.addList(RuntimeBase)` entry point for statically list-valued + expressions, retaining specialized overloads and existing control-flow-list + flattening semantics. + - Limited conditional caller-line overrides to conditions containing lexical + declarations, matching Perl without changing ordinary `if`/`elsif` calls. + - Restored the rebased-master counts for `op/do.t` (94/99), `op/caller.t` + (92/115 locally), and `op/lexsub.t` (110/160); the additional caller failure + reported by the comparison also reproduces on current master. + - Confirmed the keys benchmark is load-sensitive (branch results varied from + 5/6 to 6/6; paired current master scored 4/6 while the branch scored 5/6). + - Restored watchdog-limited regex progress to 26/59 for `re/speed.t` and + 20/59 for `re/speed_thr.t`, meeting or exceeding the reported baselines. + - Full `make` suite passes; focused caller-line behavior passes with system + Perl and with both PerlOnJava backends. + - Revalidated the affected CPAN paths: `Test::XML::Compare` (13 files / 67 + tests), `Lingua::EN::Keywords` (2 tests), and `CPAN::Index::API` (7 files / + 74 tests) all pass. +- [x] Phase 8: rebase onto current master (2026-08-15) + - Rebased PR #955 onto `origin/master` at `1be14c00b`. + - Kept master's newer full-source DATA handle implementation and dropped the + superseded scalar-handle base-offset shim while retaining the absolute + DATA-position regression test. + - Full `make` suite passes after the rebase. + - Rechecked the reported regression files: `op/do.t` 94/99, + `op/caller.t` 93/115, `op/lexsub.t` 110/160, keys benchmark 5/6, + `re/speed.t` 26/59, and `re/speed_thr.t` 26/59. +- [x] Phase 9: latest-master regex regression repair (2026-08-15) + - Rebased PR #955 onto newly fetched `origin/master` at `d22f54d24`. + - Moved the recursive lexical-declaration scan behind the cheap branch-shape + checks, avoiding compiler work for conditionals that cannot need a caller + line override. + - Restored `re/pat_advanced.t` and `re/pat_advanced_thr.t` to 1376/1687, + matching a clean latest-master build. + - Confirmed `japh/abigail.t` is 109/130 on both PR #955 and latest master, so + the reported one-test difference is not caused by this PR. + - Full `make` suite passes after the latest rebase and fast-path change. + +### Next Steps + +1. Push the current-master rebase and wait for PR #955 CI. +2. Review and merge PR #955 after CI passes. + +### Open Questions + +- None. No additional portable failure appeared after the Tagger verifier + boundary was fixed. + +## Related References + +- `.agents/skills/debug-perlonjava/SKILL.md` +- `dev/modules/cpan_compiler_tooling_suite.md` diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java b/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java index 387ec885e..fb46b12a9 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java @@ -701,44 +701,40 @@ public static void emitIdentifier(EmitterVisitor visitor, EmitterContext ctx, Id } /** - * Adds an element to a RuntimeList with type-specific optimizations. + * Adds an element to a RuntimeList with verifier-safe specialization. * - *
This method uses compile-time type information to generate optimized - * bytecode that calls the specific addToList method for known types, avoiding - * the overhead of interface dispatch when possible.
+ *List-valued expressions can cross control-flow/context boundaries that + * widen their verifier type to RuntimeBase. Other statically known values + * retain the specialized overloads used by hot list-construction paths.
* *Stack transformation: [RuntimeList] [element] → [RuntimeList]
* * @param mv The method visitor for bytecode generation * @param element The AST node representing the element being added - * @param contextType The context type (used for scalar context optimization) + * @param contextType The context in which the list element was emitted */ private static void addElementToList(MethodVisitor mv, Node element, int contextType) { - String returnType; // Stack: [RuntimeList] [element] - // Determine the element's return type for optimization - if (contextType == RuntimeContextType.SCALAR) { - // In scalar context, all elements are treated as scalars - returnType = RuntimeDescriptorConstants.SCALAR_TYPE; + String returnType = ReturnTypeVisitor.getReturnType(element); + String methodName = "add"; + // Memoize::_wrap exposed a list-valued expression whose verifier type had + // been widened while control-flow branches were merged. The addList base + // entry point keeps the bytecode verifier-safe while preserving the old + // list-flattening semantics inside RuntimeList. + if (RuntimeDescriptorConstants.LIST_TYPE.equals(returnType) + && contextType != RuntimeContextType.RUNTIME) { + methodName = "addList"; + returnType = RuntimeDescriptorConstants.BASE_TYPE; } else if (contextType == RuntimeContextType.RUNTIME) { - // In RUNTIME context, array/hash elements may have been converted to RuntimeBase - // via emitRuntimeContextConversion(), so we must use the generic add(RuntimeBase) returnType = RuntimeDescriptorConstants.BASE_TYPE; - } else { - // Use static analysis to determine the element's return type - returnType = ReturnTypeVisitor.getReturnType(element); } - // Generate type-specific method call for better performance - if (RuntimeDescriptorConstants.isKnownRuntimeType(returnType)) { - mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, RuntimeDescriptorConstants.LIST_CLASS, - "add", "(" + returnType + ")V", false); - } else { - // Fall back for unknown types - mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, RuntimeDescriptorConstants.LIST_CLASS, - "add", "(" + RuntimeDescriptorConstants.BASE_TYPE + ")V", false); + if (!RuntimeDescriptorConstants.isKnownRuntimeType(returnType)) { + returnType = RuntimeDescriptorConstants.BASE_TYPE; } + mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, RuntimeDescriptorConstants.LIST_CLASS, + methodName, "(" + returnType + ")V", false); } /** diff --git a/src/main/java/org/perlonjava/frontend/parser/StatementParser.java b/src/main/java/org/perlonjava/frontend/parser/StatementParser.java index a035456f7..e5be5e072 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StatementParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StatementParser.java @@ -338,6 +338,7 @@ public static Node parseIfStatement(Parser parser) { * @return An IfNode representing the if/unless/elsif statement */ private static Node parseIfStatementInternal(Parser parser, boolean enterNewScope) { + int statementStartIndex = parser.tokenIndex; LexerToken operator = TokenUtils.consume(parser, LexerTokenType.IDENTIFIER); // "if", "unless", "elsif" // Enter a new scope for 'if' and 'unless' (but not for 'elsif' which is part of the same chain) @@ -375,12 +376,42 @@ private static Node parseIfStatementInternal(Parser parser, boolean enterNewScop TestMoreHelper.handleSkipTest(parser, thenBranch); IfNode result = new IfNode(operator.text, condition, thenBranch, elseBranch, parser.tokenIndex); + // A lexical declaration in the condition owns the next COP in Perl, so + // the first standalone call reports the conditional's source line. + // Ordinary conditions keep the call's own line (see op/caller.t). + if (!thenBranch.elements.isEmpty() + && thenBranch.elements.get(0) instanceof BinaryOperatorNode first + && "(".equals(first.operator) + && conditionContainsLexicalDeclaration(condition)) { + first.setAnnotation("callerLineTokenOverride", statementStartIndex); + } if (enterNewScope) { result.setAnnotation("postBlockHintHashId", HintHashRegistry.snapshotCurrentHintHash()); } return result; } + private static boolean conditionContainsLexicalDeclaration(Node node) { + if (node instanceof OperatorNode operatorNode) { + if ("my".equals(operatorNode.operator) || "state".equals(operatorNode.operator)) { + return true; + } + return conditionContainsLexicalDeclaration(operatorNode.operand); + } + if (node instanceof BinaryOperatorNode binaryNode) { + return conditionContainsLexicalDeclaration(binaryNode.left) + || conditionContainsLexicalDeclaration(binaryNode.right); + } + if (node instanceof ListNode listNode) { + for (Node element : listNode.elements) { + if (conditionContainsLexicalDeclaration(element)) { + return true; + } + } + } + return false; + } + /** * Parses a try-catch-finally statement. * diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeList.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeList.java index 98d95ce69..a5f7d76a5 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeList.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeList.java @@ -198,6 +198,15 @@ public void add(RuntimeList value) { } } + /** + * Verifier-safe entry point for values statically known to be RuntimeList. + * Control-flow merges may widen the JVM stack type to RuntimeBase even though + * the Perl expression contract still guarantees a list result. + */ + public void addList(RuntimeBase value) { + add((RuntimeList) value); + } + /** * Gets the size of the list. * diff --git a/src/test/resources/unit/caller_exported_first_statement_in_if.t b/src/test/resources/unit/caller_exported_first_statement_in_if.t new file mode 100644 index 000000000..050f9e8b0 --- /dev/null +++ b/src/test/resources/unit/caller_exported_first_statement_in_if.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::Builder::Tester tests => 1; + +sub failing_test($$;$) { + return Test::Builder->new->ok(0, $_[2]); +} + +test_out('not ok 1 - captured failure'); +if (my $fail = 1) { + test_fail(6); +} +if (my $error = 0) { + test_err('unused'); +} +failing_test('left', 'right', 'captured failure'); +test_test('exported first statement retains Perl caller line'); diff --git a/src/test/resources/unit/caller_if_condition_line.t b/src/test/resources/unit/caller_if_condition_line.t new file mode 100644 index 000000000..c807f5b29 --- /dev/null +++ b/src/test/resources/unit/caller_if_condition_line.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +no warnings 'syntax'; +use Test::More tests => 2; + +my @caller_lines; +sub capture_caller_line { push @caller_lines, (caller)[2] } + +my $ordinary_call_line = __LINE__ + 2; +if (1) { + capture_caller_line(); +} +is($caller_lines[0], $ordinary_call_line, + 'ordinary if body call keeps its own source line'); + +my $declaration_condition_line = __LINE__ + 1; +if (my $enabled = 1) { + capture_caller_line(); +} +is($caller_lines[1], $declaration_condition_line, + 'lexical declaration condition owns the first call source line'); diff --git a/src/test/resources/unit/data_handle_absolute_position.t b/src/test/resources/unit/data_handle_absolute_position.t new file mode 100644 index 000000000..705d2a9e2 --- /dev/null +++ b/src/test/resources/unit/data_handle_absolute_position.t @@ -0,0 +1,19 @@ +package DataHandleAbsolutePosition; +use strict; +use warnings; +use Test::More; +use Fcntl qw(SEEK_SET); + +my $start = tell(DATA); +ok($start > 0, 'DATA tell is the absolute source-file position'); + +my $first = do { local $/; }; +ok(seek(DATA, $start, SEEK_SET), 'DATA seeks back to its absolute start'); +my $second = do { local $/; }; + +is($second, $first, 'DATA content can be read again after restoring tell'); + +done_testing; + +__DATA__ +payload diff --git a/src/test/resources/unit/memoize_runtime_list_bytecode.t b/src/test/resources/unit/memoize_runtime_list_bytecode.t new file mode 100644 index 000000000..56c429597 --- /dev/null +++ b/src/test/resources/unit/memoize_runtime_list_bytecode.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More; +use Memoize; + +my $calls = 0; +sub doubled { + $calls++; + return $_[0] * 2; +} + +my $memoized = memoize(\&doubled, INSTALL => undef); + +is($memoized->(21), 42, 'memoized closure returns its value'); +is($memoized->(21), 42, 'memoized closure returns its cached value'); +is($calls, 1, 'memoized wrapper called the original once'); + +done_testing;