From 5d5e10857e03ee7a11a96a121e509eda40ea0512 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 15:27:15 +0200 Subject: [PATCH 1/2] feat(regex): vendor Joni with match-time callouts Import the official Joni 2.2.7 source snapshot from tag joni-2.2.7 at 57fd57b4f977813a7b4b35e0179943b1f06f51d7. Retain its source packages, tests, MIT license, copyright headers, and authorship notices unchanged except for the generic callout extension maintained here. Replace the binary Joni dependency with the vendored source and retain JCodings 1.0.64 as the upstream binary dependency. Its license was copied from tag jcodings-1.0.64 at 996ae0f72c5cc6bb28ef92f29bbd5ef0f63d5250. Add matcher-local callout handlers, provisional capture views, a callout opcode, and matcher-stack unwind tokens. Cleanup now covers backtracking, success, failure, handler exceptions, interruption, and timeout in reverse execution order. Matcher-local handlers keep shared compiled regexes safe for runtime-local callback state. Relocate Joni and JCodings into org.perlonjava.internal in the standalone JAR, package all upstream license and modification notices, and record the vendored component and JCodings relationship in the combined SBOM. Add the history-free design contract in docs/design/joni-callout-fork.md, the imported upstream Joni suite, focused callout tests, and an automated packaging verifier. Verification: full make passes, including the upstream Joni suite, focused callout tests, five PerlOnJava unit shards, standalone JAR relocation and notice checks, and SBOM validation. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- build.gradle | 63 +- dev/tools/merge-sbom.pl | 44 + dev/tools/verify-joni-packaging.pl | 49 + docs/README.md | 1 + docs/design/joni-callout-fork.md | 138 + third_party/joni/.github/workflows/maven.yml | 54 + third_party/joni/.gitignore | 2 + .../.mvn/wrapper/maven-wrapper.properties | 19 + third_party/joni/.travis.yml | 13 + third_party/joni/Jenkinsfile | 71 + third_party/joni/LICENSE | 21 + third_party/joni/PERLONJAVA-NOTICE.md | 11 + third_party/joni/README.md | 67 + third_party/joni/mvnw | 259 ++ third_party/joni/mvnw.cmd | 149 ++ third_party/joni/pom.xml | 287 ++ third_party/joni/src/module-info.java | 7 + third_party/joni/src/org/joni/Analyser.java | 2374 +++++++++++++++++ .../joni/src/org/joni/ApplyCaseFold.java | 109 + .../joni/src/org/joni/ApplyCaseFoldArg.java | 36 + .../joni/src/org/joni/ArrayCompiler.java | 1317 +++++++++ third_party/joni/src/org/joni/BitSet.java | 123 + third_party/joni/src/org/joni/BitStatus.java | 55 + .../joni/src/org/joni/ByteCodeMachine.java | 2028 ++++++++++++++ .../joni/src/org/joni/ByteCodePrinter.java | 392 +++ .../joni/src/org/joni/CalloutAction.java | 26 + .../joni/src/org/joni/CalloutHandler.java | 27 + .../joni/src/org/joni/CalloutResult.java | 53 + .../joni/src/org/joni/CaptureTreeNode.java | 71 + .../joni/src/org/joni/CodeRangeBuffer.java | 403 +++ third_party/joni/src/org/joni/Compiler.java | 196 ++ third_party/joni/src/org/joni/Config.java | 89 + .../joni/src/org/joni/ConfigSupport.java | 13 + third_party/joni/src/org/joni/Lexer.java | 1336 ++++++++++ third_party/joni/src/org/joni/MatchView.java | 31 + third_party/joni/src/org/joni/Matcher.java | 672 +++++ .../joni/src/org/joni/MatcherFactory.java | 37 + third_party/joni/src/org/joni/MinMaxLen.java | 129 + .../joni/src/org/joni/MultiRegion.java | 72 + third_party/joni/src/org/joni/NameEntry.java | 98 + .../joni/src/org/joni/NativeMachine.java | 27 + .../joni/src/org/joni/NodeOptInfo.java | 127 + .../joni/src/org/joni/OptAnchorInfo.java | 96 + .../joni/src/org/joni/OptEnvironment.java | 39 + .../joni/src/org/joni/OptExactInfo.java | 170 ++ third_party/joni/src/org/joni/OptMapInfo.java | 126 + third_party/joni/src/org/joni/Option.java | 149 ++ third_party/joni/src/org/joni/Parser.java | 1433 ++++++++++ third_party/joni/src/org/joni/Regex.java | 521 ++++ third_party/joni/src/org/joni/Region.java | 66 + .../joni/src/org/joni/ScanEnvironment.java | 164 ++ .../joni/src/org/joni/ScannerSupport.java | 182 ++ third_party/joni/src/org/joni/Search.java | 633 +++++ .../joni/src/org/joni/SingleRegion.java | 75 + third_party/joni/src/org/joni/StackEntry.java | 202 ++ .../joni/src/org/joni/StackMachine.java | 635 +++++ third_party/joni/src/org/joni/Syntax.java | 753 ++++++ third_party/joni/src/org/joni/Token.java | 180 ++ .../joni/src/org/joni/UnsetAddrList.java | 68 + .../joni/src/org/joni/WarnCallback.java | 40 + .../joni/src/org/joni/ast/AnchorNode.java | 95 + .../joni/src/org/joni/ast/AnyCharNode.java | 37 + .../joni/src/org/joni/ast/BackRefNode.java | 86 + .../joni/src/org/joni/ast/CClassNode.java | 535 ++++ .../joni/src/org/joni/ast/CTypeNode.java | 48 + .../joni/src/org/joni/ast/CallNode.java | 71 + .../joni/src/org/joni/ast/CalloutNode.java | 42 + .../joni/src/org/joni/ast/EncloseNode.java | 130 + .../joni/src/org/joni/ast/ListNode.java | 96 + third_party/joni/src/org/joni/ast/Node.java | 120 + .../joni/src/org/joni/ast/QuantifierNode.java | 273 ++ .../joni/src/org/joni/ast/StateNode.java | 235 ++ .../joni/src/org/joni/ast/StringNode.java | 216 ++ .../src/org/joni/bench/AbstractBench.java | 50 + .../org/joni/bench/BenchGreedyBacktrack.java | 7 + .../src/org/joni/bench/BenchRailsRegs.java | 31 + .../org/joni/bench/BenchSeveralRegexps.java | 17 + .../joni/src/org/joni/constants/MetaChar.java | 31 + .../org/joni/constants/SyntaxProperties.java | 141 + .../joni/constants/internal/AnchorType.java | 71 + .../joni/constants/internal/Arguments.java | 31 + .../joni/constants/internal/EncloseType.java | 31 + .../joni/constants/internal/NodeStatus.java | 39 + .../org/joni/constants/internal/NodeType.java | 66 + .../org/joni/constants/internal/OPCode.java | 355 +++ .../org/joni/constants/internal/OPSize.java | 81 + .../constants/internal/StackPopLevel.java | 27 + .../joni/constants/internal/StackType.java | 54 + .../joni/constants/internal/StringType.java | 27 + .../joni/constants/internal/TargetInfo.java | 27 + .../joni/constants/internal/TokenType.java | 51 + .../org/joni/constants/internal/Traverse.java | 26 + .../src/org/joni/exception/ErrorMessages.java | 89 + .../org/joni/exception/InternalException.java | 28 + .../src/org/joni/exception/JOniException.java | 28 + .../org/joni/exception/SyntaxException.java | 28 + .../org/joni/exception/TimeoutException.java | 9 + .../org/joni/exception/ValueException.java | 37 + third_party/joni/test/org/joni/test/Test.java | 303 +++ .../joni/test/org/joni/test/TestA.java | 557 ++++ .../joni/test/org/joni/test/TestC.java | 732 +++++ .../joni/test/org/joni/test/TestCallout.java | 245 ++ .../org/joni/test/TestCaseInsensitive.java | 32 + .../test/org/joni/test/TestCornerCases.java | 59 + .../joni/test/org/joni/test/TestCrnl.java | 84 + .../joni/test/org/joni/test/TestError.java | 106 + .../test/org/joni/test/TestInterrupt.java | 110 + .../joni/test/org/joni/test/TestNSU8.java | 61 + .../joni/test/org/joni/test/TestPerl.java | 47 + .../joni/test/org/joni/test/TestU.java | 802 ++++++ .../joni/test/org/joni/test/TestU8.java | 359 +++ .../TestUtf8CaseFoldingLatin1Supplement.java | 56 + third_party/licenses/jcodings-LICENSE.txt | 20 + 113 files changed, 23566 insertions(+), 1 deletion(-) create mode 100644 dev/tools/verify-joni-packaging.pl create mode 100644 docs/design/joni-callout-fork.md create mode 100644 third_party/joni/.github/workflows/maven.yml create mode 100644 third_party/joni/.gitignore create mode 100755 third_party/joni/.mvn/wrapper/maven-wrapper.properties create mode 100644 third_party/joni/.travis.yml create mode 100644 third_party/joni/Jenkinsfile create mode 100644 third_party/joni/LICENSE create mode 100644 third_party/joni/PERLONJAVA-NOTICE.md create mode 100644 third_party/joni/README.md create mode 100755 third_party/joni/mvnw create mode 100755 third_party/joni/mvnw.cmd create mode 100644 third_party/joni/pom.xml create mode 100644 third_party/joni/src/module-info.java create mode 100644 third_party/joni/src/org/joni/Analyser.java create mode 100644 third_party/joni/src/org/joni/ApplyCaseFold.java create mode 100644 third_party/joni/src/org/joni/ApplyCaseFoldArg.java create mode 100644 third_party/joni/src/org/joni/ArrayCompiler.java create mode 100644 third_party/joni/src/org/joni/BitSet.java create mode 100644 third_party/joni/src/org/joni/BitStatus.java create mode 100644 third_party/joni/src/org/joni/ByteCodeMachine.java create mode 100644 third_party/joni/src/org/joni/ByteCodePrinter.java create mode 100644 third_party/joni/src/org/joni/CalloutAction.java create mode 100644 third_party/joni/src/org/joni/CalloutHandler.java create mode 100644 third_party/joni/src/org/joni/CalloutResult.java create mode 100644 third_party/joni/src/org/joni/CaptureTreeNode.java create mode 100644 third_party/joni/src/org/joni/CodeRangeBuffer.java create mode 100644 third_party/joni/src/org/joni/Compiler.java create mode 100644 third_party/joni/src/org/joni/Config.java create mode 100644 third_party/joni/src/org/joni/ConfigSupport.java create mode 100644 third_party/joni/src/org/joni/Lexer.java create mode 100644 third_party/joni/src/org/joni/MatchView.java create mode 100644 third_party/joni/src/org/joni/Matcher.java create mode 100644 third_party/joni/src/org/joni/MatcherFactory.java create mode 100644 third_party/joni/src/org/joni/MinMaxLen.java create mode 100644 third_party/joni/src/org/joni/MultiRegion.java create mode 100644 third_party/joni/src/org/joni/NameEntry.java create mode 100644 third_party/joni/src/org/joni/NativeMachine.java create mode 100644 third_party/joni/src/org/joni/NodeOptInfo.java create mode 100644 third_party/joni/src/org/joni/OptAnchorInfo.java create mode 100644 third_party/joni/src/org/joni/OptEnvironment.java create mode 100644 third_party/joni/src/org/joni/OptExactInfo.java create mode 100644 third_party/joni/src/org/joni/OptMapInfo.java create mode 100644 third_party/joni/src/org/joni/Option.java create mode 100644 third_party/joni/src/org/joni/Parser.java create mode 100644 third_party/joni/src/org/joni/Regex.java create mode 100644 third_party/joni/src/org/joni/Region.java create mode 100644 third_party/joni/src/org/joni/ScanEnvironment.java create mode 100644 third_party/joni/src/org/joni/ScannerSupport.java create mode 100644 third_party/joni/src/org/joni/Search.java create mode 100644 third_party/joni/src/org/joni/SingleRegion.java create mode 100644 third_party/joni/src/org/joni/StackEntry.java create mode 100644 third_party/joni/src/org/joni/StackMachine.java create mode 100644 third_party/joni/src/org/joni/Syntax.java create mode 100644 third_party/joni/src/org/joni/Token.java create mode 100644 third_party/joni/src/org/joni/UnsetAddrList.java create mode 100644 third_party/joni/src/org/joni/WarnCallback.java create mode 100644 third_party/joni/src/org/joni/ast/AnchorNode.java create mode 100644 third_party/joni/src/org/joni/ast/AnyCharNode.java create mode 100644 third_party/joni/src/org/joni/ast/BackRefNode.java create mode 100644 third_party/joni/src/org/joni/ast/CClassNode.java create mode 100644 third_party/joni/src/org/joni/ast/CTypeNode.java create mode 100644 third_party/joni/src/org/joni/ast/CallNode.java create mode 100644 third_party/joni/src/org/joni/ast/CalloutNode.java create mode 100644 third_party/joni/src/org/joni/ast/EncloseNode.java create mode 100644 third_party/joni/src/org/joni/ast/ListNode.java create mode 100644 third_party/joni/src/org/joni/ast/Node.java create mode 100644 third_party/joni/src/org/joni/ast/QuantifierNode.java create mode 100644 third_party/joni/src/org/joni/ast/StateNode.java create mode 100644 third_party/joni/src/org/joni/ast/StringNode.java create mode 100644 third_party/joni/src/org/joni/bench/AbstractBench.java create mode 100644 third_party/joni/src/org/joni/bench/BenchGreedyBacktrack.java create mode 100644 third_party/joni/src/org/joni/bench/BenchRailsRegs.java create mode 100644 third_party/joni/src/org/joni/bench/BenchSeveralRegexps.java create mode 100644 third_party/joni/src/org/joni/constants/MetaChar.java create mode 100644 third_party/joni/src/org/joni/constants/SyntaxProperties.java create mode 100644 third_party/joni/src/org/joni/constants/internal/AnchorType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/Arguments.java create mode 100644 third_party/joni/src/org/joni/constants/internal/EncloseType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/NodeStatus.java create mode 100644 third_party/joni/src/org/joni/constants/internal/NodeType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/OPCode.java create mode 100644 third_party/joni/src/org/joni/constants/internal/OPSize.java create mode 100644 third_party/joni/src/org/joni/constants/internal/StackPopLevel.java create mode 100644 third_party/joni/src/org/joni/constants/internal/StackType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/StringType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/TargetInfo.java create mode 100644 third_party/joni/src/org/joni/constants/internal/TokenType.java create mode 100644 third_party/joni/src/org/joni/constants/internal/Traverse.java create mode 100644 third_party/joni/src/org/joni/exception/ErrorMessages.java create mode 100644 third_party/joni/src/org/joni/exception/InternalException.java create mode 100644 third_party/joni/src/org/joni/exception/JOniException.java create mode 100644 third_party/joni/src/org/joni/exception/SyntaxException.java create mode 100644 third_party/joni/src/org/joni/exception/TimeoutException.java create mode 100644 third_party/joni/src/org/joni/exception/ValueException.java create mode 100644 third_party/joni/test/org/joni/test/Test.java create mode 100644 third_party/joni/test/org/joni/test/TestA.java create mode 100755 third_party/joni/test/org/joni/test/TestC.java create mode 100644 third_party/joni/test/org/joni/test/TestCallout.java create mode 100644 third_party/joni/test/org/joni/test/TestCaseInsensitive.java create mode 100755 third_party/joni/test/org/joni/test/TestCornerCases.java create mode 100755 third_party/joni/test/org/joni/test/TestCrnl.java create mode 100755 third_party/joni/test/org/joni/test/TestError.java create mode 100755 third_party/joni/test/org/joni/test/TestInterrupt.java create mode 100755 third_party/joni/test/org/joni/test/TestNSU8.java create mode 100755 third_party/joni/test/org/joni/test/TestPerl.java create mode 100755 third_party/joni/test/org/joni/test/TestU.java create mode 100755 third_party/joni/test/org/joni/test/TestU8.java create mode 100644 third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java create mode 100644 third_party/licenses/jcodings-LICENSE.txt diff --git a/build.gradle b/build.gradle index f065399dc..bee88bff1 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,9 @@ version = '5.44.0' // CycloneDX SBOM generation configuration cyclonedxBom { + // Production runtime dependencies are the contents of the standalone JAR; + // test and annotation configurations do not belong in the shipped SBOM. + includeConfigs = ["runtimeClasspath"] projectType = "application" schemaVersion = "1.5" includeLicenseText = false @@ -229,7 +232,7 @@ dependencies { implementation libs.bcpkix // Bouncy Castle PEM/PKCS parsing implementation libs.snappy.java // Official Sereal Java codec compression implementation libs.zstd.jni // Official Sereal Java codec compression - implementation 'org.jruby.joni:joni:2.2.7' // Stack-safe recursive regex backend + implementation 'org.jruby.jcodings:jcodings:1.0.64' // Encoding support for vendored Joni implementation 'io.netty:netty-codec-http:4.1.115.Final' // Netty HTTP codec for PSGI server // Testing dependencies @@ -343,6 +346,23 @@ shadowJar { } exclude 'module-info.class' exclude 'META-INF/MANIFEST.MF' + + // Keep the embedded regex engine private when PerlOnJava shares a + // classpath with JRuby or another Joni/JCodings version. + relocate 'org.joni', 'org.perlonjava.internal.joni' + relocate 'org.jcodings', 'org.perlonjava.internal.jcodings' + + from('third_party/joni/LICENSE') { + into 'META-INF/licenses' + rename { 'joni-LICENSE.txt' } + } + from('third_party/licenses/jcodings-LICENSE.txt') { + into 'META-INF/licenses' + } + from('third_party/joni/PERLONJAVA-NOTICE.md') { + into 'META-INF/licenses' + rename { 'joni-PERLONJAVA-NOTICE.md' } + } // Include combined SBOM in JAR's META-INF/sbom/ directory from("$buildDir/reports/sbom.json") { @@ -350,6 +370,17 @@ shadowJar { } } +tasks.register('verifyJoniPackaging', Exec) { + description = 'Verifies Joni namespace isolation, notices, and SBOM metadata' + group = 'verification' + dependsOn shadowJar + inputs.file("dev/tools/verify-joni-packaging.pl") + inputs.file("target/perlonjava-${project.version}.jar") + inputs.file("build/reports/sbom.json") + commandLine 'perl', 'dev/tools/verify-joni-packaging.pl', + "target/perlonjava-${project.version}.jar", 'build/reports/sbom.json' +} + // Some Perl unit tests spawn a child interpreter via $^X. Under Gradle those // children go through the repository launcher, so the target jar must be built // before tests run; otherwise a stale target/perlonjava-*.jar can be executed. @@ -392,6 +423,7 @@ tasks.register('mergeSbom', Exec) { // Declare inputs so Gradle knows dependencies inputs.file("build/reports/bom.json") inputs.file("build/reports/perl-bom.json") + inputs.file("dev/tools/merge-sbom.pl") workingDir = projectDir commandLine 'perl', 'dev/tools/merge-sbom.pl', @@ -420,6 +452,11 @@ tasks.named('build') { // Source sets configuration for including Perl resources sourceSets { main { + java { + // Start below Joni's optional Java 9 module descriptor so the + // otherwise classpath-based PerlOnJava build remains unnamed. + srcDir 'third_party/joni/src/org' + } resources { srcDir 'src/main/perl' srcDir 'src/main/resources' @@ -437,6 +474,11 @@ sourceSets { include 'META-INF/services/**' } } + joniTest { + java.srcDir 'third_party/joni/test' + compileClasspath += sourceSets.main.output + configurations.testRuntimeClasspath + runtimeClasspath += output + compileClasspath + } test { resources { srcDir 'src/test/resources' @@ -444,6 +486,23 @@ sourceSets { } } +configurations { + joniTestImplementation.extendsFrom testImplementation + joniTestRuntimeOnly.extendsFrom testRuntimeOnly +} + +dependencies { + joniTestImplementation 'junit:junit:4.13.2' +} + +tasks.register('testJoni', Test) { + description = 'Runs the imported upstream Joni test suite' + group = 'verification' + testClassesDirs = sourceSets.joniTest.output.classesDirs + classpath = sourceSets.joniTest.runtimeClasspath + useJUnit() +} + // Resource processing configuration tasks.named('processResources', Copy) { from(sourceSets.main.resources.srcDirs) { @@ -498,6 +557,8 @@ tasks.register('testUnitParallel') { description = 'Runs unit tests in parallel across multiple JVMs. Usage: gradle testUnitParallel --parallel' def shardTasks = (0..{components}) { for my $comp (@{$java_bom->{components}}) { @@ -97,6 +100,43 @@ } } +# Joni is compiled from the pinned source snapshot in third_party/joni rather +# than resolved as a binary dependency, so CycloneDX cannot discover it from +# runtimeClasspath. Record the vendored component explicitly. +push @all_components, { + type => 'library', + 'bom-ref' => $joni_ref, + group => 'org.jruby.joni', + name => 'joni', + version => '2.2.7', + description => 'Vendored Java port of Oniguruma with PerlOnJava callout extensions', + licenses => [ + { + license => { + id => 'MIT' + } + } + ], + purl => $joni_ref, + externalReferences => [ + { + type => 'website', + url => 'https://github.com/jruby/joni' + }, + { + type => 'vcs', + url => 'https://github.com/jruby/joni.git' + } + ], + properties => [ + { + name => 'perlonjava:vendored', + value => 'true' + } + ] +}; +push @root_deps, $joni_ref; + # Add Perl components if ($perl_bom->{components}) { for my $comp (@{$perl_bom->{components}}) { @@ -112,6 +152,10 @@ { ref => 'perlonjava', dependsOn => \@root_deps + }, + { + ref => $joni_ref, + dependsOn => [$jcodings_ref] } ]; diff --git a/dev/tools/verify-joni-packaging.pl b/dev/tools/verify-joni-packaging.pl new file mode 100644 index 000000000..27e6760c9 --- /dev/null +++ b/dev/tools/verify-joni-packaging.pl @@ -0,0 +1,49 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use JSON::PP; + +die "Usage: $0 \n" unless @ARGV == 2; +my ($jar_file, $sbom_file) = @ARGV; + +my %entries; +open my $jar_fh, '-|', 'jar', 'tf', $jar_file + or die "Cannot list $jar_file: $!\n"; +while (my $entry = <$jar_fh>) { + chomp $entry; + $entries{$entry} = 1; +} +close $jar_fh or die "Cannot list $jar_file: jar exited with status $?\n"; + +die "Standalone JAR contains unrelocated Joni classes\n" + if grep { m{^org/joni/} } keys %entries; +die "Standalone JAR contains unrelocated JCodings classes\n" + if grep { m{^org/jcodings/} } keys %entries; +die "Standalone JAR does not contain relocated Joni classes\n" + unless grep { m{^org/perlonjava/internal/joni/} } keys %entries; +die "Standalone JAR does not contain relocated JCodings classes\n" + unless grep { m{^org/perlonjava/internal/jcodings/} } keys %entries; + +for my $notice (qw( + joni-LICENSE.txt + jcodings-LICENSE.txt + joni-PERLONJAVA-NOTICE.md +)) { + die "Standalone JAR is missing $notice\n" + unless $entries{"META-INF/licenses/$notice"}; +} + +open my $sbom_fh, '<', $sbom_file or die "Cannot open $sbom_file: $!\n"; +local $/; +my $sbom = JSON::PP->new->utf8->decode(<$sbom_fh>); +close $sbom_fh; + +my ($joni) = grep { + ($_->{group} // '') eq 'org.jruby.joni' + && ($_->{name} // '') eq 'joni' + && ($_->{version} // '') eq '2.2.7' +} @{$sbom->{components} // []}; +die "Combined SBOM is missing vendored Joni 2.2.7\n" unless $joni; + +print "Joni packaging verification passed\n"; diff --git a/docs/README.md b/docs/README.md index 90946cc1e..7db719c4e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,7 @@ Looking to contribute? See: - **[CONTRIBUTING.md](../CONTRIBUTING.md)** - Contribution guidelines - **[dev/](../dev/)** - Developer documentation and internal architecture +- **[Joni Callout Fork](design/joni-callout-fork.md)** - Vendored regex engine design and implementation contract ## Finding What You Need diff --git a/docs/design/joni-callout-fork.md b/docs/design/joni-callout-fork.md new file mode 100644 index 000000000..a36c38425 --- /dev/null +++ b/docs/design/joni-callout-fork.md @@ -0,0 +1,138 @@ +# Vendored Joni Callout Engine + +## Purpose + +Perl executable regex constructs must run while the matcher owns provisional +captures and its backtracking stack. Stock Joni provides the required +stack-based matcher but does not expose match-time callbacks or unwind +notifications. PerlOnJava therefore vendors a minimally extended Joni engine. + +This document defines the current architecture and acceptance contract. Change +history, imported revision details, and completed-work records belong in Git +commit messages, not here. + +## Repository and dependency model + +- Joni is a pinned source snapshot in `third_party/joni`. Its production sources + and upstream tests are dedicated source sets in the root Gradle project. This + keeps the code boundary explicit without exposing the vendored engine as a + separately resolved project dependency. +- The source retains the upstream `org.joni` packages to keep changes reviewable + and suitable for upstream submission. +- The vendored project is the only Joni implementation on PerlOnJava's compile + and runtime classpaths. +- JCodings remains an upstream binary dependency because PerlOnJava does not + modify its encoding implementation. +- Ordinary Java-regex patterns keep their existing fast path. Declarative + recursive patterns and executable patterns use the vendored Joni engine. + +## Internal callout syntax + +The structured Perl regex frontend emits `(?{=CALL:})` only in an +engine-facing skeleton. `` is a non-negative decimal index into the callback +table owned by that regex value. The fork parses this representation into a +dedicated callout node; it never parses Perl source. + +Runtime-interpolated pattern text must not be promoted to trusted skeleton +syntax. The PerlOnJava frontend remains responsible for preserving source +origin and rejecting injected internal tokens. + +## Engine API + +The fork exposes a runtime-neutral API in `org.joni`: + +```java +interface CalloutHandler { + CalloutResult execute(int calloutId, MatchView match); + void unwind(Object backtrackToken); +} + +interface MatchView { + int currentBytePosition(); + int captureCount(); + int captureBegin(int capture); + int captureEnd(int capture); +} +``` + +The handler is installed on each `Matcher`, not on the compiled `Regex`, so a +shared compiled pattern can execute concurrently with runtime-local callback +state. + +`CalloutResult` selects `CONTINUE` or `FAIL` and may carry an opaque unwind +token. The engine neither interprets that token nor depends on PerlOnJava +runtime classes. + +## Execution and unwind contract + +1. The callout opcode invokes the handler with the callout ID and a read-only + view of the current byte position and provisional capture boundaries. +2. A non-null token is stored in a matcher stack frame before execution + continues or fails. +3. Backtracking across that frame calls `unwind(token)` exactly once. +4. Successful completion, failed completion, interruption, timeout, and handler + exceptions unwind every still-active token in reverse execution order. +5. A `FAIL` result enters normal matcher backtracking after installing the frame, + so cleanup follows the same path as later-pattern failure. +6. Patterns without callout nodes allocate no handler state or callout frames. + +The first engine slice supports plain callouts. Conditional callbacks and +dynamic nested patterns extend the action set only after their Perl semantics +and outer-backtracking behavior are proven independently. + +## Capture view + +Capture offsets are byte offsets relative to the matcher's input buffer, matching +Joni's native representation. Capture zero describes the provisional overall +match from the current match start to the current instruction position. Unset +captures return `Region.REGION_NOTPOS`. The bridge converts offsets to Perl +character positions when publishing match variables. + +The view is valid only during `execute`; handlers must copy any data they retain. +The engine must not allocate a `Region` snapshot for each callout. + +## Packaging and namespace isolation + +Source code remains in `org.joni`, but the standalone JAR relocates it to +`org.perlonjava.internal.joni`. JCodings is relocated to +`org.perlonjava.internal.jcodings` because Joni's API and implementation refer +to its classes. This lets an embedding application load PerlOnJava beside JRuby +or stock Joni without linkage collisions. + +The standalone JAR contains the unmodified Joni and JCodings license texts and a +separate PerlOnJava modification notice under `META-INF/licenses`. The SBOM must +identify the vendored Joni component, its upstream version, MIT license, and +JCodings dependency. + +## Implementation stages + +1. Import and build the pinned upstream snapshot with its original tests through + the dedicated root source sets. +2. Replace the external dependency and prove declarative regex parity. +3. Add the callout parser node, opcode, handler API, provisional match view, and + forward execution. +4. Add matcher-owned callout frames and exact-once cleanup for every exit path. +5. Connect structured Perl callback templates without enabling unsupported + callback forms. +6. Add conditional and nested-program actions only after focused differential + tests establish their contracts. + +## Verification + +- The complete imported Joni suite passes through the root build. +- Existing recursive-regex tests match stock Joni behavior. +- Focused tests cover provisional captures, repeated execution after + backtracking, `FAIL`, nested frames, handler exceptions, interruption, and + exact-once reverse-order unwind. +- The standalone JAR contains no `org/joni` or `org/jcodings` classes, contains + both relocated trees, and includes all required notices. +- An embedding smoke test can load stock Joni and PerlOnJava together. +- Full `make` and the focused direct/thread regex matrix pass. + +## Stop conditions + +- Do not approximate callbacks as post-match hooks. +- Do not enable a callback form if provisional state or unwind is incomplete. +- Do not implement dynamic patterns as atomic nested matches; their alternatives + must participate in outer backtracking. +- Keep unsupported syntax fatal until its complete semantic gate passes. diff --git a/third_party/joni/.github/workflows/maven.yml b/third_party/joni/.github/workflows/maven.yml new file mode 100644 index 000000000..8e2ae53d9 --- /dev/null +++ b/third_party/joni/.github/workflows/maven.yml @@ -0,0 +1,54 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: +# - https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-java-with-maven +# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions +name: Java CI with Maven + +on: + push: + branches-ignore: # build all branches except: + - 'dependabot/**' # prevent GHA triggered twice (once for commit to the branch and once for opening/syncing the PR) + tags-ignore: # don't build tags + - '**' + paths-ignore: + - 'Jenkinsfile' + - 'LICENSE' + - '**/*.md' + - '.git*' + - '.github/*.yml' + pull_request: + paths-ignore: + - 'Jenkinsfile' + - 'LICENSE' + - '**/*.md' + - '.git*' + - '.github/*.yml' + workflow_dispatch: + # https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/ + + +defaults: + run: + shell: bash + + +jobs: + build: + runs-on: ubuntu-latest + + env: + JAVA_VERSION: 11 + + steps: + - name: Git Checkout + uses: actions/checkout@v4 # https://github.com/actions/checkout + + - name: Set up JDK ${{ env.JAVA_VERSION }} ☕ + uses: actions/setup-java@v4 # https://github.com/actions/setup-java + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + cache: maven + + - name: Build with Maven 🔨 + run: mvn -ntp -B verify diff --git a/third_party/joni/.gitignore b/third_party/joni/.gitignore new file mode 100644 index 000000000..0bf6ba832 --- /dev/null +++ b/third_party/joni/.gitignore @@ -0,0 +1,2 @@ +target +.tool-versions diff --git a/third_party/joni/.mvn/wrapper/maven-wrapper.properties b/third_party/joni/.mvn/wrapper/maven-wrapper.properties new file mode 100755 index 000000000..d58dfb70b --- /dev/null +++ b/third_party/joni/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/third_party/joni/.travis.yml b/third_party/joni/.travis.yml new file mode 100644 index 000000000..ff3b50095 --- /dev/null +++ b/third_party/joni/.travis.yml @@ -0,0 +1,13 @@ +language: java + +jdk: +- openjdk9 + +arch: + - amd64 + - ppc64le + + +cache: + directories: + - $HOME/.m2 diff --git a/third_party/joni/Jenkinsfile b/third_party/joni/Jenkinsfile new file mode 100644 index 000000000..dc7d1d478 --- /dev/null +++ b/third_party/joni/Jenkinsfile @@ -0,0 +1,71 @@ +#!/usr/bin/env groovy + +pipeline { + agent none + options { + buildDiscarder(logRotator(numToKeepStr: '10')) + timeout(time: 1, unit: 'HOURS') + } + stages { + stage('OpenJDK 8') { + agent { docker 'openjdk:8-jdk' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + + stage('Alternative Platforms') { + parallel { + stage('OpenJDK 9') { + agent { docker 'openjdk:9-jdk' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + stage('Alpine Linux') { + agent { docker 'openjdk:8-jdk-alpine' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + stage('FreeBSD 11') { + agent { label 'freebsd' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + /* awaiting platform support in Code Valet */ + stage('Windows 2016') { + when { branch 'windows-support' } + steps { + echo 'Not yet available' + } + } + } + } + } +} diff --git a/third_party/joni/LICENSE b/third_party/joni/LICENSE new file mode 100644 index 000000000..9341f05ee --- /dev/null +++ b/third_party/joni/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 JRuby Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/joni/PERLONJAVA-NOTICE.md b/third_party/joni/PERLONJAVA-NOTICE.md new file mode 100644 index 000000000..a708cc8ef --- /dev/null +++ b/third_party/joni/PERLONJAVA-NOTICE.md @@ -0,0 +1,11 @@ +# Modified Joni distribution + +This directory contains Joni, the Java port of Oniguruma maintained by the +JRuby project, with generic match-time callout extensions maintained by the +PerlOnJava project. + +Joni remains licensed under its original MIT License. All upstream copyright, +license, and authorship notices are retained. PerlOnJava's modifications do not +replace or narrow those notices. + +Upstream project: https://github.com/jruby/joni diff --git a/third_party/joni/README.md b/third_party/joni/README.md new file mode 100644 index 000000000..b8e784bea --- /dev/null +++ b/third_party/joni/README.md @@ -0,0 +1,67 @@ +joni +==== + +[![Maven Central](https://img.shields.io/maven-central/v/org.jruby.joni/joni)](https://central.sonatype.com/artifact/org.jruby.joni/joni) +[![Build Status](https://github.com/jruby/joni/actions/workflows/maven.yml/badge.svg)](https://github.com/jruby/joni/actions/workflows/maven.yml) + +Java port of [Oniguruma](https://github.com/kkos/oniguruma) regexp library + +## Usage + +### Imports + +```java +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +``` + +### Matching + +```java +byte[] pattern = "a*".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +``` + +### Using captures + +```java +byte[] pattern = "(a*)".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +if (result != -1) { + Region region = matcher.getEagerRegion(); +} +``` + +### Using named captures + +```java +byte[] pattern = "(?a*)".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +if (result != -1) { + Region region = matcher.getEagerRegion(); + for (Iterator entry = regex.namedBackrefIterator(); entry.hasNext();) { + NameEntry e = entry.next(); + int number = e.getBackRefs()[0]; // can have many refs per name + // int begin = region.beg[number]; + // int end = region.end[number]; + } +} +``` + +## License + +Joni is released under the [MIT License](http://www.opensource.org/licenses/MIT). diff --git a/third_party/joni/mvnw b/third_party/joni/mvnw new file mode 100755 index 000000000..19529ddf8 --- /dev/null +++ b/third_party/joni/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/third_party/joni/mvnw.cmd b/third_party/joni/mvnw.cmd new file mode 100755 index 000000000..8dfb09e8d --- /dev/null +++ b/third_party/joni/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/third_party/joni/pom.xml b/third_party/joni/pom.xml new file mode 100644 index 000000000..e89cafa3d --- /dev/null +++ b/third_party/joni/pom.xml @@ -0,0 +1,287 @@ + + + 4.0.0 + org.jruby.joni + joni + jar + 2.2.7 + Joni + + Java port of Oniguruma: http://www.geocities.jp/kosako3/oniguruma + that uses byte arrays directly instead of java Strings and chars + + https://github.com/jruby/joni + + + UTF-8 + + + + GitHub + https://github.com/jruby/joni/issues + + + + scm:git:https://github.com/jruby/joni.git + scm:git:git@github.com:jruby/joni.git + https://github.com/jruby/joni + + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + + lopex + Marcin Mielzynski + lopx@gazeta.pl + + + + + + org.jruby.jcodings + jcodings + 1.0.64 + + + junit + junit + 4.13.2 + test + + + + + src + test + joni + + + org.apache.maven.wagon + wagon-webdav + + + + + maven-compiler-plugin + 3.13.0 + + 1.8 + 1.8 + + + + default-compile + + + module-info.java + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + maven-jar-plugin + 3.4.2 + + + + true + true + + + Joni (Java port of Oniguruma) + Joni (Java port of Oniguruma) + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.7.0 + true + + central + + + + + + + jacoco + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + + + + + default-prepare-agent + + prepare-agent + + + + pre-unit-test + + prepare-agent + + + ${project.build.directory}/coverage-reports/jacoco.exec + surefireArgLine + + + + post-unit-test + test + + report + + + ${project.build.directory}/coverage-reports/jacoco.exec + ${project.reporting.outputDirectory}/jacoco + + + + default-report + prepare-package + + report + + + + + + + + + + release-on-9 + + [9,) + + + + + maven-compiler-plugin + + + compile9 + + compile + + + 9 + + module-info.java + + + + + + + maven-source-plugin + 3.3.1 + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + 3.8.0 + + + attach-javadocs + + jar + + + + + -Xdoclint:none + -html5 + true + false + + + + + + + release + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + maven-gpg-plugin + 3.2.4 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + + true + + + + diff --git a/third_party/joni/src/module-info.java b/third_party/joni/src/module-info.java new file mode 100644 index 000000000..a8b9e8683 --- /dev/null +++ b/third_party/joni/src/module-info.java @@ -0,0 +1,7 @@ +open module org.jruby.joni { + exports org.joni; + exports org.joni.constants; + exports org.joni.exception; + + requires transitive org.jruby.jcodings; +} \ No newline at end of file diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java new file mode 100644 index 000000000..f7190c6b8 --- /dev/null +++ b/third_party/joni/src/org/joni/Analyser.java @@ -0,0 +1,2374 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAll; +import static org.joni.BitStatus.bsAt; +import static org.joni.BitStatus.bsClear; +import static org.joni.BitStatus.bsOnAt; +import static org.joni.BitStatus.bsOnAtSimple; +import static org.joni.Option.isCaptureGroup; +import static org.joni.Option.isFindCondition; +import static org.joni.Option.isIgnoreCase; +import static org.joni.Option.isMultiline; +import static org.joni.ast.ListNode.newAlt; +import static org.joni.ast.ListNode.newList; +import static org.joni.ast.QuantifierNode.isRepeatInfinite; + +import java.util.IllegalFormatConversionException; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.ObjPtr; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.joni.ast.AnchorNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.ListNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.EncloseType; +import org.joni.constants.internal.NodeType; +import org.joni.constants.internal.StackPopLevel; +import org.joni.constants.internal.TargetInfo; + +final class Analyser extends Parser { + + protected Analyser(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex, syntax, bytes, p, end, warnings); + } + + protected final void compile() { + if (Config.DEBUG) Config.log.println(encStringToString(bytes, getBegin(), getEnd())); + reset(); + + regex.numMem = 0; + regex.numRepeat = 0; + regex.numNullCheck = 0; + //regex.repeatRangeAlloc = 0; + regex.repeatRangeLo = null; + regex.repeatRangeHi = null; + regex.numCombExpCheck = 0; + + if (Config.USE_CEC) regex.numCombExpCheck = 0; + + + Node root = parseRegexp(); // onig_parse_make_tree + regex.numMem = env.numMem; + + if (Config.USE_NAMED_GROUP) { + /* mixed use named group and no-named group */ + if (env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(regex.options)) { + if (env.numNamed != env.numMem) { + root = disableNoNameGroupCapture(root); + } else { + numberedRefCheck(root); + } + } + } // USE_NAMED_GROUP + + if (Config.USE_NAMED_GROUP) { + if (env.numCall > 0) { + env.unsetAddrList = new UnsetAddrList(env.numCall); + setupSubExpCall(root); + // r != 0 ??? + subexpRecursiveCheckTrav(root); + // r < 0 -< err, FOUND_CALLED_NODE = 1 + subexpInfRecursiveCheckTrav(root); + // r != 0 recursion infinite ??? + regex.numCall = env.numCall; + } else { + regex.numCall = 0; + } + } // USE_NAMED_GROUP + + if (Config.DEBUG_PARSE_TREE && Config.DEBUG_PARSE_TREE_RAW) Config.log.println("\n" + root + "\n"); + + Node.TopNode top = Node.newTop(root); + setupTree(root, 0); + root = top.getRoot(); + + if (Config.DEBUG_PARSE_TREE) Config.log.println("\n" + root + "\n"); + + regex.captureHistory = env.captureHistory; + regex.btMemStart = env.btMemStart; + + if (isFindCondition(regex.options)) { + regex.btMemEnd = bsAll(); + } else { + regex.btMemEnd = env.btMemEnd; + regex.btMemEnd |= regex.captureHistory; + } + + if (Config.USE_CEC) { + if (env.backrefedMem == 0 || (Config.USE_SUBEXP_CALL && env.numCall == 0)) { + setupCombExpCheck(root, 0); + + if (Config.USE_SUBEXP_CALL && env.hasRecursion) { + env.numCombExpCheck = 0; + } else { // USE_SUBEXP_CALL + if (env.combExpMaxRegNum > 0) { + for (int i=1; i 1) { + int p_ = p; + while (p_ < end) { + int code = enc.mbcToCode(bytes, p_, end); + if (code >= 0x80) { + try { + sb.append(String.format(" 0x%04x ", code)); + } catch (IllegalFormatConversionException ifce) { + sb.append(code); + } + } else { + sb.append((char)code); + } + p_ += enc.length(bytes, p_, end); + } + } else { + while (p < end) { + sb.append(new String(bytes, p, 1)); + p++; + } + } + return sb.append("/").toString(); + } + + private void noNameDisableMapFor_listAlt(Node node, int[]map, Ptr counter) { + ListNode can = (ListNode)node; + do { + can.setValue(noNameDisableMap(can.value, map, counter)); + } while ((can = can.tail) != null); + } + + private void noNameDisableMapFor_quantifier(Node node, int[]map, Ptr counter) { + QuantifierNode qn = (QuantifierNode)node; + Node target = qn.target; + Node old = target; + target = noNameDisableMap(target, map, counter); + + if (target != old) { + qn.setTarget(target); + if (target.getType() == NodeType.QTFR) qn.reduceNestedQuantifier((QuantifierNode)target); + } + } + + private Node noNameDisableMapFor_enclose(Node node, int[]map, Ptr counter) { + EncloseNode en = (EncloseNode)node; + if (en.type == EncloseType.MEMORY) { + if (en.isNamedGroup()) { + counter.p++; + map[en.regNum] = counter.p; + en.regNum = counter.p; + en.setTarget(noNameDisableMap(en.target, map, counter)); + } else { + node = en.target; + en.target = null; // remove first enclose: /(a)(?c)/ + node = noNameDisableMap(node, map, counter); + } + } else { + en.setTarget(noNameDisableMap(en.target, map, counter)); + } + return node; + } + + private void noNameDisableMapFor_anchor(Node node, int[]map, Ptr counter) { + AnchorNode an = (AnchorNode)node; + if (an.target != null) an.setTarget(noNameDisableMap(an.target, map, counter)); + } + + private Node noNameDisableMap(Node node, int[]map, Ptr counter) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + noNameDisableMapFor_listAlt(node, map, counter); + break; + case NodeType.QTFR: + noNameDisableMapFor_quantifier(node, map, counter); + break; + case NodeType.ENCLOSE: + node = noNameDisableMapFor_enclose(node, map, counter); + break; + case NodeType.ANCHOR: + noNameDisableMapFor_anchor(node, map, counter); + break; + } // switch + return node; + } + + private void renumberByMap(Node node, int[]map) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + renumberByMap(can.value, map); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + renumberByMap(((QuantifierNode)node).target, map); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.type == EncloseType.CONDITION) { + en.regNum = map[en.regNum]; + } + renumberByMap(en.target, map); + break; + + case NodeType.BREF: + ((BackRefNode)node).renumber(map); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.target != null) renumberByMap(an.target, map); + break; + } // switch + } + + protected final void numberedRefCheck(Node node) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + numberedRefCheck(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + numberedRefCheck(((QuantifierNode)node).target); + break; + + case NodeType.ENCLOSE: + numberedRefCheck(((EncloseNode)node).target); + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (!br.isNameRef()) newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.target != null) numberedRefCheck(an.target); + break; + } // switch + } + + protected final Node disableNoNameGroupCapture(Node root) { + int[]map = new int[env.numMem + 1]; + + root = noNameDisableMap(root, map, new Ptr(0)); + renumberByMap(root, map); + + for (int i=1, pos=1; i<=env.numMem; i++) { + if (map[i] > 0) { + env.memNodes[pos] = env.memNodes[i]; + pos++; + } + } + + int loc = env.captureHistory; + env.captureHistory = bsClear(); + + for (int i=1; i<=Config.MAX_CAPTURE_HISTORY_GROUP; i++) { + if (bsAt(loc, i)) { + env.captureHistory = bsOnAtSimple(env.captureHistory, map[i]); + } + } + + env.numMem = env.numNamed; + regex.numMem = env.numNamed; + + regex.renumberNameTable(map); + + return root; + } + + // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + private int quantifiersMemoryInfo(Node node) { + int info = 0; + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + int v = quantifiersMemoryInfo(can.value); + if (v > info) info = v; + } while ((can = can.tail) != null); + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + return TargetInfo.IS_EMPTY_REC; /* tiny version */ + } else { + info = quantifiersMemoryInfo(cn.target); + } + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.upper != 0) { + info = quantifiersMemoryInfo(qn.target); + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + return TargetInfo.IS_EMPTY_MEM; + + case EncloseType.OPTION: + case EncloseNode.STOP_BACKTRACK: + case EncloseNode.CONDITION: + case EncloseNode.ABSENT: + info = quantifiersMemoryInfo(en.target); + break; + + default: + break; + } // inner switch + break; + + case NodeType.BREF: + case NodeType.STR: + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + case NodeType.ANCHOR: + default: + break; + } // switch + + return info; + } + + private int getMinMatchLength(Node node) { + int min = 0; + + switch (node.getType()) { + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (br.isRecursion()) break; + + if (br.back[0] > env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + min = getMinMatchLength(env.memNodes[br.back[0]]); + } + + for (int i=1; i env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + int tmin = getMinMatchLength(env.memNodes[br.back[i]]); + if (min > tmin) min = tmin; + } + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + EncloseNode en = cn.target; + if (en.isMinFixed()) min = en.minLength; + } else { + min = getMinMatchLength(cn.target); + } + } // USE_SUBEXP_CALL + break; + + case NodeType.LIST: + ListNode can = (ListNode)node; + do { + min += getMinMatchLength(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.ALT: + ListNode y = (ListNode)node; + do { + Node x = y.value; + int tmin = getMinMatchLength(x); + if (y == node) { + min = tmin; + } else if (min > tmin) { + min = tmin; + } + } while ((y = y.tail) != null); + break; + + case NodeType.STR: + min = ((StringNode)node).length(); + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + min = 1; + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.lower > 0) { + min = getMinMatchLength(qn.target); + min = MinMaxLen.distanceMultiply(min, qn.lower); + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isMinFixed()) { + min = en.minLength; + } else { + if (en.isMark1()) { + min = 0; /* recursive */ + } else { + en.setMark1(); + min = getMinMatchLength(en.target); + en.clearMark1(); + en.minLength = min; + en.setMinFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + min = getMinMatchLength(en.target); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + default: + break; + } // switch + + return min; + } + + private int getMaxMatchLength(Node node) { + int max = 0; + + switch (node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + do { + int tmax = getMaxMatchLength(ln.value); + max = MinMaxLen.distanceAdd(max, tmax); + } while ((ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + do { + int tmax = getMaxMatchLength(an.value); + if (max < tmax) max = tmax; + } while ((an = an.tail) != null); + break; + + case NodeType.STR: + max = ((StringNode)node).length(); + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + max = enc.maxLength(); + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (br.isRecursion()) { + max = MinMaxLen.INFINITE_DISTANCE; + break; + } + + for (int i=0; i env.numMem) { + if(!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + int tmax = getMaxMatchLength(env.memNodes[br.back[i]]); + if (max < tmax) max = tmax; + } + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (!cn.isRecursion()) { + max = getMaxMatchLength(cn.target); + } else { + max = MinMaxLen.INFINITE_DISTANCE; + } + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.upper != 0) { + max = getMaxMatchLength(qn.target); + if (max != 0) { + if (!isRepeatInfinite(qn.upper)) { + max = MinMaxLen.distanceMultiply(max, qn.upper); + } else { + max = MinMaxLen.INFINITE_DISTANCE; + } + } + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isMaxFixed()) { + max = en.maxLength; + } else { + if (en.isMark1()) { + max = MinMaxLen.INFINITE_DISTANCE; + } else { + en.setMark1(); + max = getMaxMatchLength(en.target); + en.clearMark1(); + en.maxLength = max; + en.setMaxFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + max = getMaxMatchLength(en.target); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + default: + break; + } // switch + + return max; + } + + private static final int GET_CHAR_LEN_VARLEN = -1; + private static final int GET_CHAR_LEN_TOP_ALT_VARLEN = -2; + protected final int getCharLengthTree(Node node) { + return getCharLengthTree(node, 0); + } + + private int getCharLengthTree(Node node, int level) { + level++; + + int len = 0; + returnCode = 0; + + switch(node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + do { + int tlen = getCharLengthTree(ln.value, level); + if (returnCode == 0) len = MinMaxLen.distanceAdd(len, tlen); + } while (returnCode == 0 && (ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + boolean varLen = false; + + int tlen = getCharLengthTree(an.value, level); + while (returnCode == 0 && (an = an.tail) != null) { + int tlen2 = getCharLengthTree(an.value, level); + if (returnCode == 0) { + if (tlen != tlen2) varLen = true; + } + } + + if (returnCode == 0) { + if (varLen) { + if (level == 1) { + returnCode = GET_CHAR_LEN_TOP_ALT_VARLEN; + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + } else { + len = tlen; + } + } + break; + + case NodeType.STR: + StringNode sn = (StringNode)node; + len = sn.length(enc); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.lower == qn.upper) { + tlen = getCharLengthTree(qn.target, level); + if (returnCode == 0) len = MinMaxLen.distanceMultiply(tlen, qn.lower); + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (!cn.isRecursion()) { + len = getCharLengthTree(cn.target, level); + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + } // USE_SUBEXP_CALL + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + len = 1; + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch(en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isCLenFixed()) { + len = en.charLength; + } else { + len = getCharLengthTree(en.target, level); + if (returnCode == 0) { + en.charLength = len; + en.setCLenFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + len = getCharLengthTree(en.target, level); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + break; + + default: + returnCode = GET_CHAR_LEN_VARLEN; + } // switch + return len; + } + + /* x is not included y ==> 1 : 0 */ + private boolean isNotIncluded(Node x, Node y) { + Node tmp; + + retry: while(true) { + int yType = y.getType(); + + switch(x.getType()) { + case NodeType.CTYPE: + switch(yType) { + case NodeType.CTYPE: + { + CTypeNode cny = (CTypeNode)y; + CTypeNode cnx = (CTypeNode)x; + return cny.ctype == cnx.ctype && cny.not != cnx.not && cny.asciiRange == cnx.asciiRange; + } + case NodeType.CCLASS: + // !swap:! + tmp = x; + x = y; + y = tmp; + // !goto retry;! + continue retry; + + case NodeType.STR: + // !goto swap;! + tmp = x; + x = y; + y = tmp; + continue retry; + + default: + break; + } // inner switch + break; + + case NodeType.CCLASS: + CClassNode xc = (CClassNode)x; + + switch(yType) { + case NodeType.CTYPE: + { + CTypeNode yc = (CTypeNode)y; + switch(yc.ctype) { + case CharacterType.WORD: + if (!yc.not) { + if (xc.mbuf == null && !xc.isNot()) { + for (int i=0; i ys.length()) len = ys.length(); + if (xs.isAmbig() || ys.isAmbig()) { + /* tiny version */ + return false; + } else { + for (int i=0, p=ys.p, q=xs.p; i 0) { + if (qn.headExact != null) { + n = qn.headExact; + } else { + n = getHeadValueNode(qn.target, exact); + } + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + + switch (en.type) { + case EncloseType.OPTION: + int options = regex.options; + regex.options = en.option; + n = getHeadValueNode(en.target, exact); + regex.options = options; + break; + + case EncloseType.MEMORY: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + n = getHeadValueNode(en.target, exact); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.type == AnchorType.PREC_READ) n = getHeadValueNode(an.target, exact); + break; + + default: + break; + } // switch + + return n; + } + + // true: invalid + private boolean checkTypeTree(Node node, int typeMask, int encloseMask, int anchorMask) { + if ((node.getType2Bit() & typeMask) == 0) return true; + + boolean invalid = false; + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + invalid = checkTypeTree(can.value, typeMask, encloseMask, anchorMask); + } while (!invalid && (can = can.tail) != null); + break; + + case NodeType.QTFR: + invalid = checkTypeTree(((QuantifierNode)node).target, typeMask, encloseMask, anchorMask); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if ((en.type & encloseMask) == 0) return true; + invalid = checkTypeTree(en.target, typeMask, encloseMask, anchorMask); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if ((an.type & anchorMask) == 0) return true; + + if (an.target != null) invalid = checkTypeTree(an.target, typeMask, encloseMask, anchorMask); + break; + + default: + break; + + } // switch + + return invalid; + } + + private static final int RECURSION_EXIST = 1; + private static final int RECURSION_INFINITE = 2; + private int subexpInfRecursiveCheck(Node node, boolean head) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + int min; + ListNode x = (ListNode)node; + do { + int ret = subexpInfRecursiveCheck(x.value, head); + if (ret == RECURSION_INFINITE) return ret; + r |= ret; + if (head) { + min = getMinMatchLength(x.value); + if (min != 0) head = false; + } + } while ((x = x.tail) != null); + break; + + case NodeType.ALT: + ListNode can = (ListNode)node; + r = RECURSION_EXIST; + do { + int ret = subexpInfRecursiveCheck(can.value, head); + if (ret == RECURSION_INFINITE) return ret; + r &= ret; + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + r = subexpInfRecursiveCheck(qn.target, head); + if (r == RECURSION_EXIST) { + if (qn.lower == 0) r = 0; + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpInfRecursiveCheck(an.target, head); + break; + } // inner switch + break; + + case NodeType.CALL: + r = subexpInfRecursiveCheck(((CallNode)node).target, head); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isMark2()) { + return 0; + } else if (en.isMark1()) { + return !head ? RECURSION_EXIST : RECURSION_INFINITE; + // throw exception here ??? + } else { + en.setMark2(); + r = subexpInfRecursiveCheck(en.target, head); + en.clearMark2(); + } + break; + + default: + break; + } // switch + return r; + } + + protected final int subexpInfRecursiveCheckTrav(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + r = subexpInfRecursiveCheckTrav(can.value); + } while (r == 0 && (can = can.tail) != null); + break; + + case NodeType.QTFR: + r = subexpInfRecursiveCheckTrav(((QuantifierNode)node).target); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpInfRecursiveCheckTrav(an.target); + break; + } // inner switch + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isRecursion()) { + en.setMark1(); + r = subexpInfRecursiveCheck(en.target, true); + if (r > 0) newValueException(NEVER_ENDING_RECURSION); + en.clearMark1(); + } + r = subexpInfRecursiveCheckTrav(en.target); + break; + + default: + break; + } // switch + + return r; + } + + private int subexpRecursiveCheck(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + r |= subexpRecursiveCheck(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + r = subexpRecursiveCheck(((QuantifierNode)node).target); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpRecursiveCheck(an.target); + break; + } // inner switch + break; + + case NodeType.CALL: + CallNode cn = (CallNode)node; + r = subexpRecursiveCheck(cn.target); + if (r != 0) cn.setRecursion(); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isMark2()) { + return 0; + } else if (en.isMark1()) { + return 1; /* recursion */ + } else { + en.setMark2(); + r = subexpRecursiveCheck(en.target); + en.clearMark2(); + } + break; + + default: + break; + } // switch + + return r; + } + + private static final int FOUND_CALLED_NODE = 1; + protected final int subexpRecursiveCheckTrav(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + int ret = subexpRecursiveCheckTrav(can.value); + if (ret == FOUND_CALLED_NODE) { + r = FOUND_CALLED_NODE; + } + // else if (ret < 0) return ret; ??? + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + r = subexpRecursiveCheckTrav(qn.target); + if (qn.upper == 0) { + if (r == FOUND_CALLED_NODE) qn.isRefered = true; + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpRecursiveCheckTrav(an.target); + break; + } // inner switch + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (!en.isRecursion()) { + if (en.isCalled()) { + en.setMark1(); + r = subexpRecursiveCheck(en.target); + if (r != 0) en.setRecursion(); + en.clearMark1(); + } + } + r = subexpRecursiveCheckTrav(en.target); + if (en.isCalled()) r |= FOUND_CALLED_NODE; + break; + + default: + break; + } // switch + + return r; + } + + private void setCallAttr(CallNode cn) { + EncloseNode en = env.memNodes[cn.groupNum]; + if (en == null) newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + en.setCalled(); + cn.setTarget(en); + env.btMemStart = BitStatus.bsOnAt(env.btMemStart, cn.groupNum); + cn.unsetAddrList = env.unsetAddrList; + } + + protected final void setupSubExpCall(Node node) { + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + setupSubExpCall(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + setupSubExpCall(((QuantifierNode)node).target); + break; + + case NodeType.ENCLOSE: + setupSubExpCall(((EncloseNode)node).target); + break; + + case NodeType.CALL: + CallNode cn = (CallNode)node; + if (cn.groupNum != 0) { + int gNum = cn.groupNum; + + if (Config.USE_NAMED_GROUP) { + if (env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(env.option)) { + newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + } + } // USE_NAMED_GROUP + if (gNum > env.numMem) newValueException(UNDEFINED_GROUP_REFERENCE, cn.nameP, cn.nameEnd); + setCallAttr(cn); + } else { + if (Config.USE_NAMED_GROUP) { + if (Config.USE_PERL_SUBEXP_CALL && cn.nameP == cn.nameEnd) { + setCallAttr(cn); + } else { + NameEntry ne = regex.nameToGroupNumbers(cn.name, cn.nameP, cn.nameEnd); + + if (ne == null) { + newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + } else if (ne.backNum > 1) { + newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); + } else { + cn.groupNum = ne.backRef1; // ne.backNum == 1 ? ne.backRef1 : ne.backRefs[0]; // ??? need to check ? + setCallAttr(cn); + } + } + } + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + setupSubExpCall(an.target); + break; + } + break; + + } // switch + } + + /* divide different length alternatives in look-behind. + (?<=A|B) ==> (?<=A)|(?<=B) + (? (? list */ + } while ((np = ((ListNode)np).tail) != null); + } + + return node; + } + + private Node setupLookBehind(AnchorNode node) { + int len = getCharLengthTree(node.target); + switch(returnCode) { + case 0: + node.charLength = len; + break; + case GET_CHAR_LEN_VARLEN: + newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + break; + case GET_CHAR_LEN_TOP_ALT_VARLEN: + if (syntax.differentLengthAltLookBehind()) { + return divideLookBehindAlternatives(node); + } else { + newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + } + } + return node; + } + + private void nextSetup(Node node, Node nextNode) { + retry: while(true) { + + int type = node.getType(); + if (type == NodeType.QTFR) { + QuantifierNode qn = (QuantifierNode)node; + if (qn.greedy && isRepeatInfinite(qn.upper)) { + if (Config.USE_QTFR_PEEK_NEXT) { + StringNode n = (StringNode)getHeadValueNode(nextNode, true); + /* '\0': for UTF-16BE etc... */ + if (n != null && n.bytes[n.p] != 0) { + qn.nextHeadExact = n; + } + } // USE_QTFR_PEEK_NEXT + + /* automatic possessification a*b ==> (?>a*)b */ + if (qn.lower <= 1) { + if (qn.target.isSimple()) { + Node x = getHeadValueNode(qn.target, false); + if (x != null) { + Node y = getHeadValueNode(nextNode, false); + if (y != null && isNotIncluded(x, y)) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); + en.setStopBtSimpleRepeat(); + node.replaceWith(en); + en.setTarget(node); + } + } + } + } + } + } else if (type == NodeType.ENCLOSE) { + EncloseNode en = (EncloseNode)node; + if (en.isMemory()) { + node = en.target; + continue retry; + } + } + + break; + } // while + } + + private void updateStringNodeCaseFoldSingleByte(StringNode sn, byte[]toLower) { + int end = sn.end; + byte[]bytes = sn.bytes; + int sp = 0; + int p = sn.p; + + while (p < end) { + byte lower = toLower[bytes[p] & 0xff]; + if (lower != bytes[p]) { + byte[]sbuf = new byte[end - sn.p]; + System.arraycopy(bytes, sn.p, sbuf, 0, sp); + + while (p < end) sbuf[sp++] = toLower[bytes[p++] & 0xff]; + + sn.set(sbuf, 0, sp); + break; + } else { + sp++; + p++; + } + } + } + + private void updateStringNodeCaseFoldMultiByte(StringNode sn) { + byte[]bytes = sn.bytes; + int end = sn.end; + value = sn.p; + int sp = 0; + byte[]buf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN]; + + while (value < end) { + int ovalue = value; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, buf); + + for (int i = 0; i < len; i++) { + if (bytes[ovalue + i] != buf[i]) { + + byte[]sbuf = new byte[sn.length() << 1]; + System.arraycopy(bytes, sn.p, sbuf, 0, ovalue - sn.p); + value = ovalue; + while (value < end) { + len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, buf); + for (i = 0; i < len; i++) { + if (sp >= sbuf.length) { + byte[]tmp = new byte[sbuf.length << 1]; + System.arraycopy(sbuf, 0, tmp, 0, sbuf.length); + sbuf = tmp; + } + sbuf[sp++] = buf[i]; + } + } + sn.set(sbuf, 0, sp); + return; + } + } + sp += len; + } + } + + private void updateStringNodeCaseFold(Node node) { + StringNode sn = (StringNode)node; + byte[] toLower = enc.toLowerCaseTable(); + if (toLower != null) { + updateStringNodeCaseFoldSingleByte(sn, toLower); + } else { + updateStringNodeCaseFoldMultiByte(sn); + } + } + + private Node expandCaseFoldMakeRemString(byte[]bytes, int p, int end) { + StringNode node = new StringNode(bytes, p, end); + + updateStringNodeCaseFold(node); + node.setAmbig(); + node.setDontGetOptInfo(); + return node; + } + + private boolean isCaseFoldVariableLength(int itemNum, CaseFoldCodeItem[] items, int slen) { + for(int i = 0; i < itemNum; i++) { + if (items[i].byteLen != slen || items[i].code.length != 1) return true; + } + return false; + } + + private boolean expandCaseFoldStringAlt(int itemNum, CaseFoldCodeItem[]items, + byte[]bytes, int p, int slen, int end, ObjPtr node) { + boolean varlen = false; + for (int i=0; i prevNode = new ObjPtr<>(); + StringNode stringNode = null; + + while (p < end) { + CaseFoldCodeItem[]items = enc.caseFoldCodesByString(regex.caseFoldFlag, bytes, p, end); + int len = enc.length(bytes, p, end); + + if (items.length == 0 || !isCaseFoldVariableLength(items.length, items, len)) { + if (stringNode == null) { + if (root == null && prevNode.p != null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + prevNode.p = stringNode = new StringNode(); // onig_node_new_str(NULL, NULL); + + if (root != null) ListNode.listAdd(root, stringNode); + + } + + stringNode.catBytes(bytes, p, p + len); + } else { + altNum *= (items.length + 1); + if (altNum > THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) break; + if (stringNode != null) { + updateStringNodeCaseFold(stringNode); + stringNode.setAmbig(); + } + + if (root == null && prevNode.p != null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + if (expandCaseFoldStringAlt(items.length, items, bytes, p, len, end, prevNode)) { // if (r == 1) + if (root == null) { + topRoot = (ListNode)prevNode.p; + } else { + ListNode.listAdd(root, prevNode.p); + } + + root = (ListNode)((ListNode)prevNode.p).value; + } else { /* r == 0 */ + if (root != null) ListNode.listAdd(root, prevNode.p); + } + stringNode = null; + } + p += len; + } + + if (stringNode != null) { + updateStringNodeCaseFold(stringNode); + stringNode.setAmbig(); + } + + if (p < end) { + Node srem = expandCaseFoldMakeRemString(bytes, p, end); + + if (prevNode.p != null && root == null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + if (root == null) { + prevNode.p = srem; + } else { + ListNode.listAdd(root, srem); + } + } + /* ending */ + Node xnode = topRoot != null ? topRoot : prevNode.p; + + node.replaceWith(xnode); + return xnode; + } + + private static final int CEC_THRES_NUM_BIG_REPEAT = 512; + private static final int CEC_INFINITE_NUM = 0x7fffffff; + + private static final int CEC_IN_INFINITE_REPEAT = (1<<0); + private static final int CEC_IN_FINITE_REPEAT = (1<<1); + private static final int CEC_CONT_BIG_REPEAT = (1<<2); + + protected final int setupCombExpCheck(Node node, int state) { + int r = state; + int ret; + + switch (node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + + do { + r = setupCombExpCheck(ln.value, r); + //prev = ((ConsAltNode)node).value; + } while (r >= 0 && (ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + do { + ret = setupCombExpCheck(an.value, state); + r |= ret; + } while (ret >= 0 && (an = an.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + int childState = state; + int addState = 0; + int varNum; + + if (!isRepeatInfinite(qn.upper)) { + if (qn.upper > 1) { + /* {0,1}, {1,1} are allowed */ + childState |= CEC_IN_FINITE_REPEAT; + + /* check (a*){n,m}, (a+){n,m} => (a*){n,n}, (a+){n,n} */ + if (env.backrefedMem == 0) { + if (qn.target.getType() == NodeType.ENCLOSE) { + EncloseNode en = (EncloseNode)qn.target; + if (en.type == EncloseType.MEMORY) { + if (en.target.getType() == NodeType.QTFR) { + QuantifierNode q = (QuantifierNode)en.target; + if (isRepeatInfinite(q.upper) && q.greedy == qn.greedy) { + qn.upper = qn.lower == 0 ? 1 : qn.lower; + if (qn.upper == 1) childState = state; + } + } + } + } + } + } + } + + if ((state & CEC_IN_FINITE_REPEAT) != 0) { + qn.combExpCheckNum = -1; + } else { + if (isRepeatInfinite(qn.upper)) { + varNum = CEC_INFINITE_NUM; + childState |= CEC_IN_INFINITE_REPEAT; + } else { + varNum = qn.upper - qn.lower; + } + + if (varNum >= CEC_THRES_NUM_BIG_REPEAT) addState |= CEC_CONT_BIG_REPEAT; + + if (((state & CEC_IN_INFINITE_REPEAT) != 0 && varNum != 0) || + ((state & CEC_CONT_BIG_REPEAT) != 0 && varNum >= CEC_THRES_NUM_BIG_REPEAT)) { + if (qn.combExpCheckNum == 0) { + env.numCombExpCheck++; + qn.combExpCheckNum = env.numCombExpCheck; + if (env.currMaxRegNum > env.combExpMaxRegNum) { + env.combExpMaxRegNum = env.currMaxRegNum; + } + } + } + } + r = setupCombExpCheck(qn.target, childState); + r |= addState; + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch( en.type) { + case EncloseNode.MEMORY: + if (env.currMaxRegNum < en.regNum) { + env.currMaxRegNum = en.regNum; + } + r = setupCombExpCheck(en.target, state); + break; + + default: + r = setupCombExpCheck(en.target, state); + } // inner switch + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + env.hasRecursion = true; + } else { + r = setupCombExpCheck(cn.target, state); + } + } // USE_SUBEXP_CALL + break; + + default: + break; + + } // switch + + return r; + } + + private static final int IN_ALT = (1<<0); + private static final int IN_NOT = (1<<1); + private static final int IN_REPEAT = (1<<2); + private static final int IN_VAR_REPEAT = (1<<3); + private static final int IN_CALL = (1<<4); + private static final int IN_RECCALL = (1<<5); + private static final int EXPAND_STRING_MAX_LENGTH = 100; + + /* setup_tree does the following work. + 1. check empty loop. (set qn->target_empty_info) + 2. expand ignore-case in char class. + 3. set memory status bit flags. (reg->mem_stats) + 4. set qn->head_exact for [push, exact] -> [push_or_jump_exact1, exact]. + 5. find invalid patterns in look-behind. + 6. expand repeated string. + */ + protected final Node setupTree(Node node, int state) { + restart: while (true) { + switch (node.getType()) { + case NodeType.LIST: + ListNode lin = (ListNode)node; + Node prev = null; + do { + setupTree(lin.value, state); + if (prev != null) { + nextSetup(prev, lin.value); + } + prev = lin.value; + } while ((lin = lin.tail) != null); + break; + + case NodeType.ALT: + ListNode aln = (ListNode)node; + do { + setupTree(aln.value, (state | IN_ALT)); + } while ((aln = aln.tail) != null); + break; + + case NodeType.CCLASS: + break; + + case NodeType.STR: + if (isIgnoreCase(regex.options) && !((StringNode)node).isRaw()) { + node = expandCaseFoldString(node); + } + break; + + case NodeType.CTYPE: + case NodeType.CANY: + break; + + case NodeType.CALL: // if (Config.USE_SUBEXP_CALL) ? + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + for (int i=0; i env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + env.backrefedMem = bsOnAt(env.backrefedMem, br.back[i]); + env.btMemStart = bsOnAt(env.btMemStart, br.back[i]); + if (Config.USE_BACKREF_WITH_LEVEL) { + if (br.isNestLevel()) { + env.btMemEnd = bsOnAt(env.btMemEnd, br.back[i]); + } + } // USE_BACKREF_AT_LEVEL + env.memNodes[br.back[i]].setMemBackrefed(); + } + } + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + Node target = qn.target; + + if ((state & IN_REPEAT) != 0) qn.setInRepeat(); + + if (isRepeatInfinite(qn.upper) || qn.lower >= 1) { + int d = getMinMatchLength(target); + if (d == 0) { + qn.targetEmptyInfo = TargetInfo.IS_EMPTY; + if (Config.USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT) { + int info = quantifiersMemoryInfo(target); + if (info > 0) qn.targetEmptyInfo = info; + } // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + // strange stuff here (turned off) + } + } + + state |= IN_REPEAT; + if (qn.lower != qn.upper) state |= IN_VAR_REPEAT; + + target = setupTree(target, state); + + /* expand string */ + if (target.getType() == NodeType.STR && !(target instanceof CalloutNode)) { + StringNode sn = (StringNode)target; + if (qn.lower > 1) { + StringNode str = new StringNode(sn.bytes, sn.p, sn.end); + str.flag = sn.flag; + + int i; + int n = qn.lower; + int len = sn.length(); + for (i = 1; i < n && (i + 1) * len <= EXPAND_STRING_MAX_LENGTH; i++) { + str.catBytes(sn.bytes, sn.p, sn.end); + } + + if (i < qn.upper || isRepeatInfinite(qn.upper)) { + qn.lower -= i; + if (!isRepeatInfinite(qn.upper)) qn.upper -= i; + ListNode list = ListNode.newList(str, null); + qn.replaceWith(list); + ListNode.listAdd(list, qn); + } else { + qn.replaceWith(str); + } + break; + } + } + + if (Config.USE_OP_PUSH_OR_JUMP_EXACT) { + if (qn.greedy && qn.targetEmptyInfo != 0) { + if (target.getType() == NodeType.QTFR) { + QuantifierNode tqn = (QuantifierNode)target; + if (tqn.headExact != null) { + qn.headExact = tqn.headExact; + tqn.headExact = null; + } + } else { + qn.headExact = getHeadValueNode(qn.target, true); + } + } + } // USE_OP_PUSH_OR_JUMP_EXACT + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.OPTION: + int options = regex.options; + regex.options = en.option; + setupTree(en.target, state); + regex.options = options; + break; + + case EncloseType.MEMORY: + if ((state & (IN_ALT | IN_NOT | IN_VAR_REPEAT | IN_CALL)) != 0) { + env.btMemStart = bsOnAt(env.btMemStart, en.regNum); + /* SET_ENCLOSE_STATUS(node, NST_MEM_IN_ALT_NOT); */ + } + if (en.isCalled()) state |= IN_CALL; + if (en.isRecursion()) { + state |= IN_RECCALL; + } else if ((state & IN_RECCALL) != 0){ + en.setRecursion(); + } + + setupTree(en.target, state); + break; + + case EncloseType.STOP_BACKTRACK: + setupTree(en.target, state); + if (en.target.getType() == NodeType.QTFR) { + QuantifierNode tqn = (QuantifierNode)en.target; + if (isRepeatInfinite(tqn.upper) && tqn.lower <= 1 && tqn.greedy) { + /* (?>a*), a*+ etc... */ + if (tqn.target.isSimple()) en.setStopBtSimpleRepeat(); + } + } + break; + + case EncloseNode.CONDITION: + if (Config.USE_NAMED_GROUP) { + if (!en.isNameRef() && env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(env.option)) { + newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + } + } + if (en.regNum > env.numMem) newValueException(INVALID_BACKREF); + setupTree(en.target, state); + break; + + case EncloseType.ABSENT: + setupTree(en.target, state); + break; + } // inner switch + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + setupTree(an.target, state); + break; + + case AnchorType.PREC_READ_NOT: + setupTree(an.target, (state | IN_NOT)); + break; + + case AnchorType.LOOK_BEHIND: + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB, AnchorType.ALLOWED_IN_LB)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + node = setupLookBehind(an); + if (node.getType() != NodeType.ANCHOR) continue restart; + setupTree(((AnchorNode)node).target, state); + node = setupLookBehind(an); + break; + + case AnchorType.LOOK_BEHIND_NOT: + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB_NOT, AnchorType.ALLOWED_IN_LB_NOT)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + node = setupLookBehind(an); + if (node.getType() != NodeType.ANCHOR) continue restart; + setupTree(((AnchorNode)node).target, (state | IN_NOT)); + node = setupLookBehind(an); + break; + + } // inner switch + break; + } // switch + return node; + } // restart: while + } + + private static final int MAX_NODE_OPT_INFO_REF_COUNT = 5; + private void optimizeNodeLeft(Node node, NodeOptInfo opt, OptEnvironment oenv) { // oenv remove, pass mmd + opt.clear(); + opt.setBoundNode(oenv.mmd); + + switch (node.getType()) { + case NodeType.LIST: { + OptEnvironment nenv = new OptEnvironment(); + NodeOptInfo nopt = new NodeOptInfo(); + nenv.copy(oenv); + ListNode lin = (ListNode)node; + do { + optimizeNodeLeft(lin.value, nopt, nenv); + nenv.mmd.add(nopt.length); + opt.concatLeftNode(nopt, enc); + } while ((lin = lin.tail) != null); + break; + } + + case NodeType.ALT: { + NodeOptInfo nopt = new NodeOptInfo(); + ListNode aln = (ListNode)node; + do { + optimizeNodeLeft(aln.value, nopt, oenv); + if (aln == node) { + opt.copy(nopt); + } else { + opt.altMerge(nopt, oenv); + } + } while ((aln = aln.tail) != null); + break; + } + + case NodeType.STR: { + StringNode sn = (StringNode)node; + + int slen = sn.length(); + + if (!sn.isAmbig()) { + opt.exb.concatStr(sn.bytes, sn.p, sn.end, sn.isRaw(), enc); + opt.exb.ignoreCase = 0; + + if (slen > 0) { + opt.map.addChar(sn.bytes[sn.p], enc); + } + + opt.length.set(slen, slen); + } else { + int max; + if (sn.isDontGetOptInfo()) { + int n = sn.length(enc); + max = enc.maxLength() * n; + } else { + opt.exb.concatStr(sn.bytes, sn.p, sn.end, sn.isRaw(), enc); + opt.exb.ignoreCase = 1; + + if (slen > 0) { + opt.map.addCharAmb(sn.bytes, sn.p, sn.end, enc, oenv.caseFoldFlag); + } + + max = slen; + } + opt.length.set(slen, max); + } + + if (opt.exb.length == slen) { + opt.exb.reachEnd = true; + } + break; + } + + case NodeType.CCLASS: { + CClassNode cc = (CClassNode)node; + /* no need to check ignore case. (setted in setup_tree()) */ + if (cc.mbuf != null || cc.isNot()) { + int min = enc.minLength(); + int max = enc.maxLength(); + opt.length.set(min, max); + } else { + for (int i=0; i= maxCode) { + opt.map.addChar((byte)i, enc); + } + } + } else { + for (int i=0; i 0) { + opt.expr.copy(nopt.exb); + } else if (nopt.exm.length > 0) { + opt.expr.copy(nopt.exm); + } + opt.expr.reachEnd = false; + if (nopt.map.value > 0) opt.map.copy(nopt.map); + break; + + case AnchorType.LOOK_BEHIND_NOT: + break; + + } // inner switch + break; + } + + case NodeType.BREF: { + BackRefNode br = (BackRefNode)node; + + if (br.isRecursion()) { + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + break; + } + + Node[]nodes = oenv.scanEnv.memNodes; + + int min = 0; + int max = 0; + + if (nodes != null && nodes[br.back[0]] != null) { + min = getMinMatchLength(nodes[br.back[0]]); + max = getMaxMatchLength(nodes[br.back[0]]); + } + + for (int i=1; i tmin) min = tmin; + if (max < tmax) max = tmax; + } + } + opt.length.set(min, max); + break; + } + + case NodeType.CALL: { + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + } else { + int safe = oenv.options; + oenv.options = cn.target.option; + optimizeNodeLeft(cn.target, opt, oenv); + oenv.options = safe; + } + } // USE_SUBEXP_CALL + break; + } + + case NodeType.QTFR: { + NodeOptInfo nopt = new NodeOptInfo(); + QuantifierNode qn = (QuantifierNode)node; + optimizeNodeLeft(qn.target, nopt, oenv); + if (/*qn.lower == 0 &&*/ isRepeatInfinite(qn.upper)) { + if (oenv.mmd.max == 0 && qn.target.getType() == NodeType.CANY && qn.greedy) { + if (isMultiline(oenv.options)) { + opt.anchor.add(AnchorType.ANYCHAR_STAR_ML); + } else { + opt.anchor.add(AnchorType.ANYCHAR_STAR); + } + } + } /*else*/ { + if (qn.lower > 0) { + opt.copy(nopt); + if (nopt.exb.length > 0) { + if (nopt.exb.reachEnd) { + int i; + for (i = 2; i <= qn.lower && !opt.exb.isFull(); i++) { + opt.exb.concat(nopt.exb, enc); + } + if (i < qn.lower) { + opt.exb.reachEnd = false; + } + } + } + if (qn.lower != qn.upper) { + opt.exb.reachEnd = false; + opt.exm.reachEnd = false; + } + if (qn.lower > 1) { + opt.exm.reachEnd = false; + } + + } + } + int min = MinMaxLen.distanceMultiply(nopt.length.min, qn.lower); + int max; + if (isRepeatInfinite(qn.upper)) { + max = nopt.length.max > 0 ? MinMaxLen.INFINITE_DISTANCE : 0; + } else { + max = MinMaxLen.distanceMultiply(nopt.length.max, qn.upper); + } + opt.length.set(min, max); + break; + } + + case NodeType.ENCLOSE: { + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.OPTION: + int save = oenv.options; + oenv.options = en.option; + optimizeNodeLeft(en.target, opt, oenv); + oenv.options = save; + break; + + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL && ++en.optCount > MAX_NODE_OPT_INFO_REF_COUNT) { + int min = 0; + int max = MinMaxLen.INFINITE_DISTANCE; + if (en.isMinFixed()) min = en.minLength; + if (en.isMaxFixed()) max = en.maxLength; + opt.length.set(min, max); + } else { // USE_SUBEXP_CALL + optimizeNodeLeft(en.target, opt, oenv); + if (opt.anchor.isSet(AnchorType.ANYCHAR_STAR_MASK)) { + if (bsAt(oenv.scanEnv.backrefedMem, en.regNum)) { + opt.anchor.remove(AnchorType.ANYCHAR_STAR_MASK); + } + } + } + break; + + case EncloseType.STOP_BACKTRACK: + case EncloseType.CONDITION: + optimizeNodeLeft(en.target, opt, oenv); + break; + + case EncloseType.ABSENT: + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + break; + } // inner switch + break; + } + + default: + newInternalException(PARSER_BUG); + } // switch + } + + protected final void setOptimizedInfoFromTree(Node node) { + NodeOptInfo opt = new NodeOptInfo(); + OptEnvironment oenv = new OptEnvironment(); + + oenv.enc = regex.enc; + oenv.options = regex.options; + oenv.caseFoldFlag = regex.caseFoldFlag; + oenv.scanEnv = env; + oenv.mmd.clear(); // ?? + + optimizeNodeLeft(node, opt, oenv); + + regex.anchor = opt.anchor.leftAnchor & (AnchorType.BEGIN_BUF | + AnchorType.BEGIN_POSITION | + AnchorType.ANYCHAR_STAR | + AnchorType.ANYCHAR_STAR_ML | + AnchorType.LOOK_BEHIND); + + if ((opt.anchor.leftAnchor & (AnchorType.LOOK_BEHIND | AnchorType.PREC_READ_NOT)) != 0) regex.anchor &= ~AnchorType.ANYCHAR_STAR_ML; + + regex.anchor |= opt.anchor.rightAnchor & (AnchorType.END_BUF | + AnchorType.SEMI_END_BUF | + AnchorType.PREC_READ_NOT); + + if ((regex.anchor & (AnchorType.END_BUF | AnchorType.SEMI_END_BUF)) != 0) { + regex.anchorDmin = opt.length.min; + regex.anchorDmax = opt.length.max; + } + + if (opt.exb.length > 0 || opt.exm.length > 0) { + opt.exb.select(opt.exm, enc); + if (opt.map.value > 0 && opt.exb.compare(opt.map) > 0) { + // !goto set_map;! + regex.setOptimizeMapInfo(opt.map); + regex.setSubAnchor(opt.map.anchor); + } else { + regex.setOptimizeExactInfo(opt.exb); + regex.setSubAnchor(opt.exb.anchor); + } + } else if (opt.map.value > 0) { + // !set_map:! + regex.setOptimizeMapInfo(opt.map); + regex.setSubAnchor(opt.map.anchor); + } else { + regex.subAnchor |= opt.anchor.leftAnchor & AnchorType.BEGIN_LINE; + if (opt.length.max == 0) regex.subAnchor |= opt.anchor.rightAnchor & AnchorType.END_LINE; + } + + if (Config.DEBUG_COMPILE || Config.DEBUG_MATCH) { + Config.log.println(regex.optimizeInfoToString()); + } + } +} diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java new file mode 100644 index 000000000..77b27fdda --- /dev/null +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -0,0 +1,109 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.ApplyAllCaseFoldFunction; +import org.jcodings.Encoding; +import org.joni.ast.CClassNode; +import org.joni.ast.ListNode; +import org.joni.ast.StringNode; + +final class ApplyCaseFold implements ApplyAllCaseFoldFunction { + + // i_apply_case_fold + @Override + public void apply(int from, int[]to, int length, Object o) { + ApplyCaseFoldArg arg = (ApplyCaseFoldArg)o; + + ScanEnvironment env = arg.env; + Encoding enc = env.enc; + CClassNode cc = arg.cc; + CClassNode ascCc = arg.ascCc; + BitSet bs = cc.bs; + boolean addFlag; + + if (ascCc == null) { + addFlag = false; + } else if (Encoding.isAscii(from) == Encoding.isAscii(to[0])) { + addFlag = true; + } else { + addFlag = ascCc.isCodeInCC(enc, from); + if (ascCc.isNot()) addFlag = !addFlag; + } + + if (length == 1) { + boolean inCC = cc.isCodeInCC(enc, from); + if (Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS) { + if ((inCC && !cc.isNot()) || (!inCC && cc.isNot())) { + if (addFlag) { + if (enc.minLength() > 1 || to[0] >= BitSet.SINGLE_BYTE_SIZE || enc.codeToMbcLength(to[0]) > 1) { + cc.addCodeRange(env, to[0], to[0], false); + } else { + /* /(?i:[^A-C])/.match("a") ==> fail. */ + bs.set(to[0]); + } + } + } + } else { + if (inCC) { + if (addFlag) { + if (enc.minLength() > 1 || to[0] >= BitSet.SINGLE_BYTE_SIZE) { + if (cc.isNot()) cc.clearNotFlag(env); + cc.addCodeRange(env, to[0], to[0], false); + } else { + if (cc.isNot()) { + bs.clear(to[0]); + } else { + bs.set(to[0]); + } + } + } + } + } // CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS + + } else { + if (cc.isCodeInCC(enc, from) && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { + StringNode node = null; + for (int i=0; i 1 || cc.bs.isEmpty()) { + len = OPSize.OPCODE; + } else { + len = OPSize.OPCODE + BitSet.BITSET_SIZE; + } + + len += OPSize.LENGTH + cc.mbuf.getUsed(); + } + return len; + } + + @Override + protected void compileCClassNode(CClassNode cc) { + if (cc.mbuf == null) { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_NOT); + } else { + addOpcode(OPCode.CCLASS); + } + addInts(cc.bs.bits, BitSet.BITSET_SIZE); // add_bitset + } else { + if (enc.minLength() > 1 || cc.bs.isEmpty()) { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_MB_NOT); + } else { + addOpcode(OPCode.CCLASS_MB); + } + addMultiByteCClass(cc.mbuf); + } else { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_MIX_NOT); + } else { + addOpcode(OPCode.CCLASS_MIX); + } + // store the bit set and mbuf themself! + addInts(cc.bs.bits, BitSet.BITSET_SIZE); // add_bitset + addMultiByteCClass(cc.mbuf); + } + } + } + + @Override + protected void compileCTypeNode(CTypeNode node) { + CTypeNode cn = node; + int op; + switch (cn.ctype) { + case CharacterType.WORD: + if (cn.not) { + if (cn.asciiRange) { + op = OPCode.ASCII_NOT_WORD; + } else { + op = OPCode.NOT_WORD; + } + } else { + if (cn.asciiRange) { + op = OPCode.ASCII_WORD; + } else { + op = OPCode.WORD; + } + } + break; + + default: + newInternalException(PARSER_BUG); + return; // not reached + } // inner switch + addOpcode(op); + } + + @Override + protected void compileAnyCharNode() { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML); + } else { + addOpcode(OPCode.ANYCHAR); + } + } + + @Override + protected void compileCallNode(CallNode node) { + addOpcode(OPCode.CALL); + node.unsetAddrList.add(codeLength, node.target); + addAbsAddr(0); /*dummy addr.*/ + } + + @Override + protected void compileBackrefNode(BackRefNode node) { + BackRefNode br = node; + if (Config.USE_BACKREF_WITH_LEVEL && br.isNestLevel()) { + addOpcode(OPCode.BACKREF_WITH_LEVEL); + addOption(regex.options & Option.IGNORECASE); + addLength(br.nestLevel); + // !goto add_bacref_mems;! + addLength(br.backNum); + for (int i=br.backNum-1; i>=0; i--) addMemNum(br.back[i]); + return; + } else { // USE_BACKREF_AT_LEVEL + if (br.backNum == 1) { + if (isIgnoreCase(regex.options)) { + addOpcode(OPCode.BACKREFN_IC); + addMemNum(br.back[0]); + } else { + switch (br.back[0]) { + case 1: + addOpcode(OPCode.BACKREF1); + break; + case 2: + addOpcode(OPCode.BACKREF2); + break; + default: + addOpcode(OPCode.BACKREFN); + addOpcode(br.back[0]); + break; + } // switch + } + } else { + if (isIgnoreCase(regex.options)) { + addOpcode(OPCode.BACKREF_MULTI_IC); + } else { + addOpcode(OPCode.BACKREF_MULTI); + } + // !add_bacref_mems:! + addLength(br.backNum); + for (int i=br.backNum-1; i>=0; i--) addMemNum(br.back[i]); + } + } + } + + private static final int REPEAT_RANGE_ALLOC = 8; + private void entryRepeatRange(int id, int lower, int upper) { + if (regex.repeatRangeLo == null) { + regex.repeatRangeLo = new int[REPEAT_RANGE_ALLOC]; + regex.repeatRangeHi = new int[REPEAT_RANGE_ALLOC]; + } else if (id >= regex.repeatRangeLo.length){ + int[]tmp = new int[regex.repeatRangeLo.length + REPEAT_RANGE_ALLOC]; + System.arraycopy(regex.repeatRangeLo, 0, tmp, 0, regex.repeatRangeLo.length); + regex.repeatRangeLo = tmp; + tmp = new int[regex.repeatRangeHi.length + REPEAT_RANGE_ALLOC]; + System.arraycopy(regex.repeatRangeHi, 0, tmp, 0, regex.repeatRangeHi.length); + regex.repeatRangeHi = tmp; + } + + regex.repeatRangeLo[id] = lower; + regex.repeatRangeHi[id] = isRepeatInfinite(upper) ? 0x7fffffff : upper; + } + + private void compileRangeRepeatNode(QuantifierNode qn, int targetLen, int emptyInfo) { + regex.requireStack = true; + int numRepeat = regex.numRepeat; + addOpcode(qn.greedy ? OPCode.REPEAT : OPCode.REPEAT_NG); + addMemNum(numRepeat); /* OP_REPEAT ID */ + regex.numRepeat++; + addRelAddr(targetLen + OPSize.REPEAT_INC); + + entryRepeatRange(numRepeat, qn.lower, qn.upper); + + compileTreeEmptyCheck(qn.target, emptyInfo); + + if ((Config.USE_SUBEXP_CALL && regex.numCall > 0) || qn.isInRepeat()) { + addOpcode(qn.greedy ? OPCode.REPEAT_INC_SG : OPCode.REPEAT_INC_NG_SG); + } else { + addOpcode(qn.greedy ? OPCode.REPEAT_INC : OPCode.REPEAT_INC_NG); + } + + addMemNum(numRepeat); /* OP_REPEAT ID */ + } + + private static final int QUANTIFIER_EXPAND_LIMIT_SIZE = 50; // was 50 + + private static boolean cknOn(int ckn) { + return ckn > 0; + } + + private int compileCECLengthQuantifierNode(QuantifierNode qn) { + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + int ckn = regex.numCombExpCheck > 0 ? qn.combExpCheckNum : 0; + int cklen = cknOn(ckn) ? OPSize.STATE_CHECK_NUM : 0; + + /* anychar repeat */ + if (qn.target.getType() == NodeType.CANY) { + if (qn.greedy && infinite) { + if (qn.nextHeadExact != null && !cknOn(ckn)) { + return OPSize.ANYCHAR_STAR_PEEK_NEXT + tlen * qn.lower + cklen; + } else { + return OPSize.ANYCHAR_STAR + tlen * qn.lower + cklen; + } + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + + int len; + if (infinite && qn.lower <= 1) { + if (qn.greedy) { + if (qn.lower == 1) { + len = OPSize.JUMP; + } else { + len = 0; + } + len += OPSize.PUSH + cklen + modTLen + OPSize.JUMP; + } else { + if (qn.lower == 0) { + len = OPSize.JUMP; + } else { + len = 0; + } + len += modTLen + OPSize.PUSH + cklen; + } + } else if (qn.upper == 0) { + if (qn.isRefered) { /* /(?..){0}/ */ + len = OPSize.JUMP + tlen; + } else { + len = 0; + } + } else if (qn.upper == 1 && qn.greedy) { + if (qn.lower == 0) { + if (cknOn(ckn)) { + len = OPSize.STATE_CHECK_PUSH + tlen; + } else { + len = OPSize.PUSH + tlen; + } + } else { + len = tlen; + } + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0) { /* '??' */ + len = OPSize.PUSH + cklen + OPSize.JUMP + tlen; + } else { + len = OPSize.REPEAT_INC + modTLen + OPSize.OPCODE + OPSize.RELADDR + OPSize.MEMNUM; + + if (cknOn(ckn)) { + len += OPSize.STATE_CHECK; + } + } + return len; + } + + @Override + protected void compileCECQuantifierNode(QuantifierNode qn) { + regex.requireStack = true; + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + int ckn = regex.numCombExpCheck > 0 ? qn.combExpCheckNum : 0; + + if (qn.isAnyCharStar()) { + compileTreeNTimes(qn.target, qn.lower); + if (qn.nextHeadExact != null && !cknOn(ckn)) { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR_PEEK_NEXT); + } else { + addOpcode(OPCode.ANYCHAR_STAR_PEEK_NEXT); + } + if (cknOn(ckn)) { + addStateCheckNum(ckn); + } + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + return; + } else { + if (isMultiline(regex.options)) { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_ANYCHAR_ML_STAR); + } else { + addOpcode(OPCode.ANYCHAR_ML_STAR); + } + } else { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_ANYCHAR_STAR); + } else { + addOpcode(OPCode.ANYCHAR_STAR); + } + } + if (cknOn(ckn)) { + addStateCheckNum(ckn); + } + return; + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + if (infinite && qn.lower <= 1) { + if (qn.greedy) { + if (qn.lower == 1) { + addOpcodeRelAddr(OPCode.JUMP, cknOn(ckn) ? OPSize.STATE_CHECK_PUSH : + OPSize.PUSH); + } + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(modTLen + OPSize.JUMP); + } else { + addOpcodeRelAddr(OPCode.PUSH, modTLen + OPSize.JUMP); + } + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + (cknOn(ckn) ? + OPSize.STATE_CHECK_PUSH : + OPSize.PUSH))); + } else { + if (qn.lower == 0) { + addOpcodeRelAddr(OPCode.JUMP, modTLen); + } + compileTreeEmptyCheck(qn.target, emptyInfo); + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH_OR_JUMP); + addStateCheckNum(ckn); + addRelAddr(-(modTLen + OPSize.STATE_CHECK_PUSH_OR_JUMP)); + } else { + addOpcodeRelAddr(OPCode.PUSH, -(modTLen + OPSize.PUSH)); + } + } + } else if (qn.upper == 0) { + if (qn.isRefered) { /* /(?..){0}/ */ + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } // else r=0 ??? + } else if (qn.upper == 1 && qn.greedy) { + if (qn.lower == 0) { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(tlen); + } else { + addOpcodeRelAddr(OPCode.PUSH, tlen); + } + } + compileTree(qn.target); + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0){ /* '??' */ + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(OPSize.JUMP); + } else { + addOpcodeRelAddr(OPCode.PUSH, OPSize.JUMP); + } + + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } else { + compileRangeRepeatNode(qn, modTLen, emptyInfo); + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK); + addStateCheckNum(ckn); + } + } + } + + private int compileNonCECLengthQuantifierNode(QuantifierNode qn) { + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + /* anychar repeat */ + if (qn.target.getType() == NodeType.CANY) { + if (qn.greedy && infinite) { + if (qn.nextHeadExact != null) { + return OPSize.ANYCHAR_STAR_PEEK_NEXT + tlen * qn.lower; + } else { + return OPSize.ANYCHAR_STAR + tlen * qn.lower; + } + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + + int len; + if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) { + len = OPSize.JUMP; + } else { + len = tlen * qn.lower; + } + + if (qn.greedy) { + if (qn.headExact != null) { + len += OPSize.PUSH_OR_JUMP_EXACT1 + modTLen + OPSize.JUMP; + } else if (qn.nextHeadExact != null) { + len += OPSize.PUSH_IF_PEEK_NEXT + modTLen + OPSize.JUMP; + } else { + len += OPSize.PUSH + modTLen + OPSize.JUMP; + } + } else { + len += OPSize.JUMP + modTLen + OPSize.PUSH; + } + + } else if (qn.upper == 0 && qn.isRefered) { /* /(?..){0}/ */ + len = OPSize.JUMP + tlen; + } else if (!infinite && qn.greedy && + (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE )) { + len = tlen * qn.lower; + len += (OPSize.PUSH + tlen) * (qn.upper - qn.lower); + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0) { /* '??' */ + len = OPSize.PUSH + OPSize.JUMP + tlen; + } else { + len = OPSize.REPEAT_INC + modTLen + OPSize.OPCODE + OPSize.RELADDR + OPSize.MEMNUM; + } + return len; + } + + @Override + protected void compileNonCECQuantifierNode(QuantifierNode qn) { + regex.requireStack = true; + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + if (qn.isAnyCharStar()) { + compileTreeNTimes(qn.target, qn.lower); + if (qn.nextHeadExact != null) { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR_PEEK_NEXT); + } else { + addOpcode(OPCode.ANYCHAR_STAR_PEEK_NEXT); + } + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + return; + } else { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR); + } else { + addOpcode(OPCode.ANYCHAR_STAR); + } + return; + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) { + if (qn.greedy) { + if (qn.headExact != null) { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH_OR_JUMP_EXACT1); + } else if (qn.nextHeadExact != null) { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH_IF_PEEK_NEXT); + } else { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH); + } + } else { + addOpcodeRelAddr(OPCode.JUMP, OPSize.JUMP); + } + } else { + compileTreeNTimes(qn.target, qn.lower); + } + + if (qn.greedy) { + if (qn.headExact != null) { + addOpcodeRelAddr(OPCode.PUSH_OR_JUMP_EXACT1, modTLen + OPSize.JUMP); + StringNode sn = (StringNode)qn.headExact; + addBytes(sn.bytes, sn.p, 1); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH_OR_JUMP_EXACT1)); + } else if (qn.nextHeadExact != null) { + addOpcodeRelAddr(OPCode.PUSH_IF_PEEK_NEXT, modTLen + OPSize.JUMP); + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH_IF_PEEK_NEXT)); + } else { + addOpcodeRelAddr(OPCode.PUSH, modTLen + OPSize.JUMP); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH)); + } + } else { + addOpcodeRelAddr(OPCode.JUMP, modTLen); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.PUSH, -(modTLen + OPSize.PUSH)); + } + } else if (qn.upper == 0 && qn.isRefered) { /* /(?..){0}/ */ + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } else if (!infinite && qn.greedy && + (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + int n = qn.upper - qn.lower; + compileTreeNTimes(qn.target, qn.lower); + + for (int i=0; i= code.length) { + int length = code.length << 1; + while (length <= size) length <<= 1; + int[]tmp = new int[length]; + System.arraycopy(code, 0, tmp, 0, code.length); + code = tmp; + } + } + + private void addInt(int i) { + if (codeLength >= code.length) { + int[]tmp = new int[code.length << 1]; + System.arraycopy(code, 0, tmp, 0, code.length); + code = tmp; + } + code[codeLength++] = i; + } + + void setInt(int i, int offset) { + ensure(offset); + regex.code[offset] = i; + } + + private void addBytes(byte[]bytes, int p ,int length) { + ensure(codeLength + length); + int end = p + length; + + while (p < end) code[codeLength++] = bytes[p++]; + } + + private void addInts(int[]ints, int length) { + ensure(codeLength + length); + System.arraycopy(ints, 0, code, codeLength, length); + codeLength += length; + } + + private void addOpcode(int opcode) { + addInt(opcode); + } + + private void addStateCheckNum(int num) { + addInt(num); + } + + private void addRelAddr(int addr) { + addInt(addr); + } + + private void addAbsAddr(int addr) { + addInt(addr); + } + + private void addLength(int length) { + addInt(length); + } + + private void addMemNum(int num) { + addInt(num); + } + + private void addOption(int option) { + addInt(option); + } + + private void addOpcodeRelAddr(int opcode, int addr) { + addOpcode(opcode); + addRelAddr(addr); + } + + private void addOpcodeOption(int opcode, int option) { + addOpcode(opcode); + addOption(option); + } + + private void addTemplate(byte[]bytes) { + if (templateNum == 0) { + templates = new byte[2][]; + } else if (templateNum == templates.length) { + byte[][]tmp = new byte[templateNum * 2][]; + System.arraycopy(templates, 0, tmp, 0, templateNum); + templates = tmp; + } + templates[templateNum++] = bytes; + } +} diff --git a/third_party/joni/src/org/joni/BitSet.java b/third_party/joni/src/org/joni/BitSet.java new file mode 100644 index 000000000..fe8d3c1aa --- /dev/null +++ b/third_party/joni/src/org/joni/BitSet.java @@ -0,0 +1,123 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class BitSet { + static final int BITS_PER_BYTE = 8; + public static final int SINGLE_BYTE_SIZE = (1 << BITS_PER_BYTE); + public static final int BITS_IN_ROOM = 4 * BITS_PER_BYTE; + public static final int BITSET_SIZE = (SINGLE_BYTE_SIZE / BITS_IN_ROOM); + static final int ROOM_SHIFT = log2(BITS_IN_ROOM); + + public final int[] bits = new int[BITSET_SIZE]; + + public boolean at(int pos) { + return (bits[pos >>> ROOM_SHIFT] & bit(pos)) != 0; + } + + public void set(ScanEnvironment env, int pos) { + if (at(pos)) env.ccDuplicateWarn(); + set(pos); + } + + public void set(int pos) { + bits[pos >>> ROOM_SHIFT] |= bit(pos); + } + + public void clear(int pos) { + bits[pos >>> ROOM_SHIFT] &= ~bit(pos); + } + + public void invert(int pos) { + bits[pos >>> ROOM_SHIFT] ^= bit(pos); + } + + public void clear() { + for (int i=0; i>>= 1) != 0) log++; + return log; + } + + private static final int BITS_TO_STRING_WRAP = 4; + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append("BitSet"); + for (int i=0; i s) ? s : pkeep) - str; + node.end = s - str; + + stkp = 0; + makeCaptureHistoryTree(region.getCaptureTree()); + } + + private byte[]cfbuf; + private byte[]cfbuf2; + + protected final byte[]cfbuf() { + return cfbuf == null ? cfbuf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : cfbuf; + } + + protected final byte[]cfbuf2() { + return cfbuf2 == null ? cfbuf2 = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : cfbuf2; + } + + private boolean stringCmpIC(int caseFlodFlag, int s1, IntHolder ps2, int mbLen, int textEnd) { + byte[]buf1 = cfbuf(); + byte[]buf2 = cfbuf2(); + + int s2 = ps2.value; + int end1 = s1 + mbLen; + + while (s1 < end1) { + value = s1; + int len1 = enc.mbcCaseFold(caseFlodFlag, bytes, this, textEnd, buf1); + s1 = value; + value = s2; + int len2 = enc.mbcCaseFold(caseFlodFlag, bytes, this, textEnd, buf2); + s2 = value; + + if (len1 != len2) return false; + int p1 = 0; + int p2 = 0; + + while (len1-- > 0) { + if (buf1[p1] != buf2[p2]) return false; + p1++; p2++; + } + } + ps2.value = s2; + return true; + } + + @Override + protected final int matchAt(int _range, int _sstart, int _sprev, boolean interrupt) throws InterruptedException { + range = _range; + sstart = _sstart; + sprev = _sprev; + stk = 0; + ip = 0; + + if (Config.DEBUG_MATCH) debugMatchBegin(); + stackInit(); + + bestLen = -1; + s = _sstart; + pkeep = _sstart; + try { + return enc.isSingleByte() || (msaOptions & Option.CR_7_BIT) != 0 + ? executeSb(interrupt) : execute(interrupt); + } finally { + unwindActiveCallouts(); + } + } + + private final int execute(final boolean checkThreadInterrupt) throws InterruptedException { + final int[] code = this.code; + int interruptCheckCounter = 0; + while (true) { + if (interruptCheckCounter++ >= interruptCheckEvery) { + if (timeout != -1) handleTimeout(); + handleInterrupted(checkThreadInterrupt); + interruptCheckCounter = 0; + } + + if (Config.DEBUG_MATCH) debugMatchLoop(); + + sbegin = s; + switch (code[ip++]) { + case OPCode.END: if (opEnd()) return finish(); break; + case OPCode.EXACT1: opExact1(); break; + case OPCode.EXACT2: opExact2(); continue; + case OPCode.EXACT3: opExact3(); continue; + case OPCode.EXACT4: opExact4(); continue; + case OPCode.EXACT5: opExact5(); continue; + case OPCode.EXACTN: opExactN(); continue; + + case OPCode.EXACTMB2N1: opExactMB2N1(); break; + case OPCode.EXACTMB2N2: opExactMB2N2(); continue; + case OPCode.EXACTMB2N3: opExactMB2N3(); continue; + case OPCode.EXACTMB2N: opExactMB2N(); continue; + case OPCode.EXACTMB3N: opExactMB3N(); continue; + case OPCode.EXACTMBN: opExactMBN(); continue; + + case OPCode.EXACT1_IC: opExact1IC(); break; + case OPCode.EXACTN_IC: opExactNIC(); continue; + + case OPCode.CCLASS: opCClass(); break; + case OPCode.CCLASS_MB: opCClassMB(); break; + case OPCode.CCLASS_MIX: opCClassMIX(); break; + case OPCode.CCLASS_NOT: opCClassNot(); break; + case OPCode.CCLASS_MB_NOT: opCClassMBNot(); break; + case OPCode.CCLASS_MIX_NOT: opCClassMIXNot(); break; + + case OPCode.ANYCHAR: opAnyChar(); break; + case OPCode.ANYCHAR_ML: opAnyCharML(); break; + case OPCode.ANYCHAR_STAR: opAnyCharStar(); break; + case OPCode.ANYCHAR_ML_STAR: opAnyCharMLStar(); break; + case OPCode.ANYCHAR_STAR_PEEK_NEXT: opAnyCharStarPeekNext(); break; + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: opAnyCharMLStarPeekNext(); break; + + case OPCode.WORD: opWord(); break; + case OPCode.NOT_WORD: opNotWord(); break; + case OPCode.WORD_BOUND: opWordBound(); continue; + case OPCode.NOT_WORD_BOUND: opNotWordBound(); continue; + case OPCode.WORD_BEGIN: opWordBegin(); continue; + case OPCode.WORD_END: opWordEnd(); continue; + + case OPCode.ASCII_WORD: opAsciiWord(); break; + case OPCode.ASCII_NOT_WORD: opNotAsciiWord(); break; + case OPCode.ASCII_WORD_BOUND: opAsciiWordBound(); break; + case OPCode.ASCII_NOT_WORD_BOUND: opNotAsciiWordBound(); continue; + case OPCode.ASCII_WORD_BEGIN: opAsciiWordBegin(); continue; + case OPCode.ASCII_WORD_END: opAsciiWordEnd(); continue; + + case OPCode.BEGIN_BUF: opBeginBuf(); continue; + case OPCode.END_BUF: opEndBuf(); continue; + case OPCode.BEGIN_LINE: opBeginLine(); continue; + case OPCode.END_LINE: opEndLine(); continue; + case OPCode.SEMI_END_BUF: opSemiEndBuf(); continue; + case OPCode.BEGIN_POSITION: opBeginPosition(); continue; + + case OPCode.MEMORY_START_PUSH: opMemoryStartPush(); continue; + case OPCode.MEMORY_START: opMemoryStart(); continue; + case OPCode.MEMORY_END_PUSH: opMemoryEndPush(); continue; + case OPCode.MEMORY_END: opMemoryEnd(); continue; // + case OPCode.KEEP: opKeep(); continue; + case OPCode.MEMORY_END_PUSH_REC: opMemoryEndPushRec(); continue; + case OPCode.MEMORY_END_REC: opMemoryEndRec(); continue; + + case OPCode.BACKREF1: opBackRef1(); continue; + case OPCode.BACKREF2: opBackRef2(); continue; + case OPCode.BACKREFN: opBackRefN(); continue; + case OPCode.BACKREFN_IC: opBackRefNIC(); continue; + case OPCode.BACKREF_MULTI: opBackRefMulti(); continue; + case OPCode.BACKREF_MULTI_IC: opBackRefMultiIC(); continue; + case OPCode.BACKREF_WITH_LEVEL: opBackRefAtLevel(); continue; + + case OPCode.NULL_CHECK_START: opNullCheckStart(); continue; + case OPCode.NULL_CHECK_END: opNullCheckEnd(); continue; + case OPCode.NULL_CHECK_END_MEMST: opNullCheckEndMemST(); continue; + case OPCode.NULL_CHECK_END_MEMST_PUSH: opNullCheckEndMemSTPush(); continue; + + case OPCode.JUMP: opJump(); continue; + case OPCode.PUSH: opPush(); continue; + + case OPCode.POP: opPop(); continue; + case OPCode.PUSH_OR_JUMP_EXACT1: opPushOrJumpExact1(); continue; + case OPCode.PUSH_IF_PEEK_NEXT: opPushIfPeekNext(); continue; + + case OPCode.REPEAT: opRepeat(); continue; + case OPCode.REPEAT_NG: opRepeatNG(); continue; + case OPCode.REPEAT_INC: opRepeatInc(); continue; + case OPCode.REPEAT_INC_SG: opRepeatIncSG(); continue; + case OPCode.REPEAT_INC_NG: opRepeatIncNG(); continue; + case OPCode.REPEAT_INC_NG_SG: opRepeatIncNGSG(); continue; + + case OPCode.PUSH_POS: opPushPos(); continue; + case OPCode.POP_POS: opPopPos(); continue; + case OPCode.PUSH_POS_NOT: opPushPosNot(); continue; + case OPCode.FAIL_POS: opFailPos(); continue; + case OPCode.PUSH_STOP_BT: opPushStopBT(); continue; + case OPCode.POP_STOP_BT: opPopStopBT(); continue; + + case OPCode.LOOK_BEHIND: opLookBehind(); continue; + case OPCode.PUSH_LOOK_BEHIND_NOT: opPushLookBehindNot(); continue; + case OPCode.FAIL_LOOK_BEHIND_NOT: opFailLookBehindNot(); continue; + + case OPCode.PUSH_ABSENT_POS: opPushAbsentPos(); continue; + case OPCode.ABSENT: opAbsent(); continue; + case OPCode.ABSENT_END: opAbsentEnd(); continue; + + case OPCode.CALL: opCall(); continue; + case OPCode.RETURN: opReturn(); continue; + case OPCode.CONDITION: opCondition(); continue; + case OPCode.FINISH: return finish(); + case OPCode.FAIL: opFail(); continue; + case OPCode.CALLOUT: opCallout(); continue; + + case OPCode.STATE_CHECK_ANYCHAR_STAR: if (USE_CEC) {opStateCheckAnyCharStar(); break;} + case OPCode.STATE_CHECK_ANYCHAR_ML_STAR:if (USE_CEC) {opStateCheckAnyCharMLStar();break;} + case OPCode.STATE_CHECK_PUSH: if (USE_CEC) {opStateCheckPush(); continue;} + case OPCode.STATE_CHECK_PUSH_OR_JUMP: if (USE_CEC) {opStateCheckPushOrJump(); continue;} + case OPCode.STATE_CHECK: if (USE_CEC) {opStateCheck(); continue;} + + default: + throw new InternalException(ErrorMessages.UNDEFINED_BYTECODE); + + } // main switch + } // main while + } + + private void handleTimeout() throws InterruptedException { + if (System.nanoTime() - startTime > timeout) throw TIMEOUT_EXCEPTION; + } + + private final int executeSb(final boolean checkThreadInterrupt) throws InterruptedException { + final int[] code = this.code; + int interruptCheckCounter = 0; + while (true) { + if (interruptCheckCounter++ >= interruptCheckEvery) { + if (timeout != -1) handleTimeout(); + handleInterrupted(checkThreadInterrupt); + interruptCheckCounter = 0; + } + + if (Config.DEBUG_MATCH) debugMatchLoop(); + + sbegin = s; + switch (code[ip++]) { + case OPCode.END: if (opEnd()) return finish(); break; + case OPCode.EXACT1: opExact1(); break; + case OPCode.EXACT2: opExact2(); continue; + case OPCode.EXACT3: opExact3(); continue; + case OPCode.EXACT4: opExact4(); continue; + case OPCode.EXACT5: opExact5(); continue; + case OPCode.EXACTN: opExactN(); continue; + + case OPCode.EXACTMB2N1: opExactMB2N1(); break; + case OPCode.EXACTMB2N2: opExactMB2N2(); continue; + case OPCode.EXACTMB2N3: opExactMB2N3(); continue; + case OPCode.EXACTMB2N: opExactMB2N(); continue; + case OPCode.EXACTMB3N: opExactMB3N(); continue; + case OPCode.EXACTMBN: opExactMBN(); continue; + + case OPCode.EXACT1_IC: opExact1IC(); break; + case OPCode.EXACTN_IC: opExactNIC(); continue; + + case OPCode.CCLASS: opCClassSb(); break; + case OPCode.CCLASS_MB: opCClassMBSb(); break; + case OPCode.CCLASS_MIX: opCClassMIXSb(); break; + case OPCode.CCLASS_NOT: opCClassNotSb(); break; + case OPCode.CCLASS_MB_NOT: opCClassMBNotSb(); break; + case OPCode.CCLASS_MIX_NOT: opCClassMIXNotSb(); break; + + case OPCode.ANYCHAR: opAnyCharSb(); break; + case OPCode.ANYCHAR_ML: opAnyCharMLSb(); break; + case OPCode.ANYCHAR_STAR: opAnyCharStarSb(); break; + case OPCode.ANYCHAR_ML_STAR: opAnyCharMLStarSb(); break; + case OPCode.ANYCHAR_STAR_PEEK_NEXT: opAnyCharStarPeekNextSb(); break; + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: opAnyCharMLStarPeekNextSb(); break; + + case OPCode.WORD: opWordSb(); break; + case OPCode.NOT_WORD: opNotWordSb(); break; + case OPCode.WORD_BOUND: opWordBoundSb(); continue; + case OPCode.NOT_WORD_BOUND: opNotWordBoundSb(); continue; + case OPCode.WORD_BEGIN: opWordBeginSb(); continue; + case OPCode.WORD_END: opWordEndSb(); continue; + + case OPCode.ASCII_WORD: opAsciiWord(); break; + case OPCode.ASCII_NOT_WORD: opNotAsciiWord(); break; + case OPCode.ASCII_WORD_BOUND: opAsciiWordBound(); break; + case OPCode.ASCII_NOT_WORD_BOUND: opNotAsciiWordBound(); continue; + case OPCode.ASCII_WORD_BEGIN: opAsciiWordBegin(); continue; + case OPCode.ASCII_WORD_END: opAsciiWordEnd(); continue; + + case OPCode.BEGIN_BUF: opBeginBuf(); continue; + case OPCode.END_BUF: opEndBuf(); continue; + case OPCode.BEGIN_LINE: opBeginLineSb(); continue; + case OPCode.END_LINE: opEndLineSb(); continue; + case OPCode.SEMI_END_BUF: opSemiEndBuf(); continue; + case OPCode.BEGIN_POSITION: opBeginPosition(); continue; + + case OPCode.MEMORY_START_PUSH: opMemoryStartPush(); continue; + case OPCode.MEMORY_START: opMemoryStart(); continue; + case OPCode.MEMORY_END_PUSH: opMemoryEndPush(); continue; + case OPCode.MEMORY_END: opMemoryEnd(); continue; + case OPCode.KEEP: opKeep(); continue; + case OPCode.MEMORY_END_PUSH_REC: opMemoryEndPushRec(); continue; + case OPCode.MEMORY_END_REC: opMemoryEndRec(); continue; + + case OPCode.BACKREF1: opBackRef1(); continue; + case OPCode.BACKREF2: opBackRef2(); continue; + case OPCode.BACKREFN: opBackRefN(); continue; + case OPCode.BACKREFN_IC: opBackRefNIC(); continue; + case OPCode.BACKREF_MULTI: opBackRefMulti(); continue; + case OPCode.BACKREF_MULTI_IC: opBackRefMultiIC(); continue; + case OPCode.BACKREF_WITH_LEVEL: opBackRefAtLevel(); continue; + + case OPCode.NULL_CHECK_START: opNullCheckStart(); continue; + case OPCode.NULL_CHECK_END: opNullCheckEnd(); continue; + case OPCode.NULL_CHECK_END_MEMST: opNullCheckEndMemST(); continue; + case OPCode.NULL_CHECK_END_MEMST_PUSH: opNullCheckEndMemSTPush(); continue; + + case OPCode.JUMP: opJump(); continue; + case OPCode.PUSH: opPush(); continue; + + case OPCode.POP: opPop(); continue; + case OPCode.PUSH_OR_JUMP_EXACT1: opPushOrJumpExact1(); continue; + case OPCode.PUSH_IF_PEEK_NEXT: opPushIfPeekNext(); continue; + + case OPCode.REPEAT: opRepeat(); continue; + case OPCode.REPEAT_NG: opRepeatNG(); continue; + case OPCode.REPEAT_INC: opRepeatInc(); continue; + case OPCode.REPEAT_INC_SG: opRepeatIncSG(); continue; + case OPCode.REPEAT_INC_NG: opRepeatIncNG(); continue; + case OPCode.REPEAT_INC_NG_SG: opRepeatIncNGSG(); continue; + + case OPCode.PUSH_POS: opPushPos(); continue; + case OPCode.POP_POS: opPopPos(); continue; + case OPCode.PUSH_POS_NOT: opPushPosNot(); continue; + case OPCode.FAIL_POS: opFailPos(); continue; + case OPCode.PUSH_STOP_BT: opPushStopBT(); continue; + case OPCode.POP_STOP_BT: opPopStopBT(); continue; + + case OPCode.LOOK_BEHIND: opLookBehindSb(); continue; + case OPCode.PUSH_LOOK_BEHIND_NOT: opPushLookBehindNot(); continue; + case OPCode.FAIL_LOOK_BEHIND_NOT: opFailLookBehindNot(); continue; + + case OPCode.PUSH_ABSENT_POS: opPushAbsentPos(); continue; + case OPCode.ABSENT: opAbsent(); continue; + case OPCode.ABSENT_END: opAbsentEnd(); continue; + + case OPCode.CALL: opCall(); continue; + case OPCode.RETURN: opReturn(); continue; + case OPCode.CONDITION: opCondition(); continue; + case OPCode.FINISH: return finish(); + case OPCode.FAIL: opFail(); continue; + case OPCode.CALLOUT: opCallout(); continue; + + case OPCode.EXACT1_IC_SB: opExact1ICSb(); break; + case OPCode.EXACTN_IC_SB: opExactNICSb(); continue; + + case OPCode.STATE_CHECK_ANYCHAR_STAR: if (USE_CEC) {opStateCheckAnyCharStarSb(); break;} + case OPCode.STATE_CHECK_ANYCHAR_ML_STAR:if (USE_CEC) {opStateCheckAnyCharMLStarSb();break;} + case OPCode.STATE_CHECK_PUSH: if (USE_CEC) {opStateCheckPush(); continue;} + case OPCode.STATE_CHECK_PUSH_OR_JUMP: if (USE_CEC) {opStateCheckPushOrJump(); continue;} + case OPCode.STATE_CHECK: if (USE_CEC) {opStateCheck(); continue;} + + default: + throw new InternalException(ErrorMessages.UNDEFINED_BYTECODE); + + } // main switch + } // main while + } + + private void handleInterrupted(final boolean checkThreadInterrupt) throws InterruptedException { + if (interrupted || (checkThreadInterrupt && Thread.currentThread().isInterrupted())) { + Thread.interrupted(); + throw INTERRUPTED_EXCEPTION; + } + interruptCheckEvery = Math.min(interruptCheckEvery << 1, MAX_INTERRUPT_CHECK_EVERY); + } + + private boolean opEnd() { + int n = s - sstart; + + if (n > bestLen) { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (isFindLongest(regex.options)) { + if (n > msaBestLen) { + msaBestLen = n; + msaBestS = sstart; + } else { + // goto end_best_len; + return endBestLength(); + } + } + } // USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE + + bestLen = n; + final Region region = msaRegion; + if (region != null) { + // USE_POSIX_REGION_OPTION ... else ... + region.setBeg(0, msaBegin = ((pkeep > s) ? s : pkeep) - str); + region.setEnd(0, msaEnd = s - str); + for (int i = 1; i <= regex.numMem; i++) { + int me = repeatStk[memEndStk + i]; + if (me != INVALID_INDEX) { + int ms = repeatStk[memStartStk + i]; + region.setBeg(i, (bsAt(regex.btMemStart, i) ? stack[ms].getMemPStr() : ms) - str); + region.setEnd(i, (bsAt(regex.btMemEnd, i) ? stack[me].getMemPStr() : me) - str); + } else { + region.setBeg(i, Region.REGION_NOTPOS); + region.setEnd(i, Region.REGION_NOTPOS); + } + } + + if (Config.USE_CAPTURE_HISTORY && regex.captureHistory != 0) checkCaptureHistory(region); + } else { + msaBegin = ((pkeep > s) ? s : pkeep) - str; + msaEnd = s - str; + } + } else { + Region region = msaRegion; + if (region != null) { + region.clear(); + } else { + msaBegin = msaEnd = 0; + } + } + // end_best_len: + /* default behavior: return first-matching result. */ + return endBestLength(); + } + + private boolean endBestLength() { + if (isFindCondition(regex.options)) { + if (isFindNotEmpty(regex.options) && s == sstart) { + bestLen = -1; + {opFail(); return false;} /* for retry */ + } + if (isFindLongest(regex.options) && s < range) { + {opFail(); return false;} /* for retry */ + } + } + // goto finish; + return true; + } + + private void opExact1() { + if (s >= range || code[ip] != bytes[s]) { + opFail(); + } else { + ip++; s++; + sprev = sbegin; // break; + } + } + + private void opExact2() { + if (s + 2 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] ) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact3() { + if (s + 3 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact4() { + if (s + 4 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact5() { + if (s + 5 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExactN() { + int tlen = code[ip++]; + if (s + tlen > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen-- > 0) if (bs[ps++] != bytes[s++]) {opFail(); return;} + + } else { + while (tlen-- > 0) if (code[ip++] != bytes[s++]) {opFail(); return;} + } + sprev = s - 1; + } + + private void opExactMB2N1() { + if (s + 2 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + ip++; s++; + sprev = sbegin; // break; + } + } + + private void opExactMB2N2() { + if (s + 4 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + sprev = s; + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + + private void opExactMB2N3() { + if (s + 6 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + sprev = s; + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + + private void opExactMB2N() { + int tlen = code[ip++]; + if (s + tlen * 2 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while(tlen-- > 0) { + if (bs[ps] != bytes[s] || bs[++ps] != bytes[++s]) {opFail(); return;} + ps++; s++; + } + } else { + while(tlen-- > 0) { + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + } + sprev = s - 2; + } + + private void opExactMB3N() { + int tlen = code[ip++]; + if (s + tlen * 3 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen-- > 0) { + if (bs[ps] != bytes[s] || bs[++ps] != bytes[++s] || bs[++ps] != bytes[++s]) {opFail(); return;} + ps++; s++; + } + } else { + while (tlen-- > 0) { + if (code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + } + + sprev = s - 3; + } + + private void opExactMBN() { + int tlen = code[ip++]; /* mb-len */ + int tlen2= code[ip++]; /* string len */ + + tlen2 *= tlen; + if (s + tlen2 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen2-- > 0) { + if (bs[ps] != bytes[s]) {opFail(); return;} + ps++; s++; + } + } else { + while (tlen2-- > 0) { + if (code[ip] != bytes[s]) {opFail(); return;} + ip++; s++; + } + } + + sprev = s - tlen; + } + + private void opExact1IC() { + if (s >= range) {opFail(); return;} + + byte[]lowbuf = cfbuf(); + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + + int q = 0; + while (len-- > 0) { + if (code[ip] != lowbuf[q]) {opFail(); return;} + ip++; q++; + } + sprev = sbegin; // break; + } + + private void opExact1ICSb() { + if (s >= range || code[ip] != enc.toLowerCaseTable()[bytes[s++] & 0xff]) {opFail(); return;} + ip++; + sprev = sbegin; // break; + } + + private void opExactNIC() { + int tlen = code[ip++]; + byte[]lowbuf = cfbuf(); + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + int endp = ps + tlen; + + while (ps < endp) { + sprev = s; + if (s >= range) {opFail(); return;} + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + int q = 0; + while (len-- > 0) { + if (bs[ps] != lowbuf[q]) {opFail(); return;} + ps++; q++; + } + } + } else { + int endp = ip + tlen; + + while (ip < endp) { + sprev = s; + if (s >= range) {opFail(); return;} + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + int q = 0; + while (len-- > 0) { + if (code[ip] != lowbuf[q]) {opFail(); return;} + ip++; q++; + } + } + } + + } + + private void opExactNICSb() { + int tlen = code[ip++]; + if (s + tlen > range) {opFail(); return;} + byte[]toLowerTable = enc.toLowerCaseTable(); + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + while (tlen-- > 0) if (bs[ps++] != toLowerTable[bytes[s++] & 0xff]) {opFail(); return;} + } else { + while (tlen-- > 0) if (code[ip++] != toLowerTable[bytes[s++] & 0xff]) {opFail(); return;} + } + sprev = s - 1; + } + + private void opCondition() { + int mem = code[ip++]; + int addr = code[ip++]; + if (mem > regex.numMem || repeatStk[memEndStk + mem] == INVALID_INDEX || repeatStk[memStartStk + mem] == INVALID_INDEX) { + ip += addr; + } + } + + private boolean isInBitSet() { + int c = bytes[s] & 0xff; + return ((code[ip + (c >>> BitSet.ROOM_SHIFT)] & (1 << c)) != 0); + } + + private void opCClass() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s += enc.length(bytes, s, end); /* OP_CCLASS can match mb-code. \D, \S */ + if (s > end) s = end; + sprev = sbegin; // break; + } + + private void opCClassSb() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + sprev = sbegin; // break; + } + + private boolean isInClassMB() { + int tlen = code[ip++]; + if (s >= range) return false; + int mbLen = enc.length(bytes, s, end); + if (s + mbLen > range) return false; + int ss = s; + s += mbLen; + int c = enc.mbcToCode(bytes, ss, s); + if (!CodeRange.isInCodeRange(code, ip, c)) return false; + ip += tlen; + return true; + } + + private void opCClassMB() { + // beyond string check + if (s >= range || !enc.isMbcHead(bytes, s, end)) {opFail(); return;} + if (!isInClassMB()) {opFail(); return;} // not!!! + sprev = sbegin; // break; + } + + private void opCClassMBSb() { + opFail(); + } + + private void opCClassMIX() { + if (s >= range) {opFail(); return;} + if (enc.isMbcHead(bytes, s, end)) { + ip += BitSet.BITSET_SIZE; + if (!isInClassMB()) {opFail(); return;} + } else { + if (!isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; // by code range length + ip += tlen; + s++; + } + sprev = sbegin; // break; + } + + private void opCClassMIXSb() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; + ip += tlen; + s++; + sprev = sbegin; // break; + } + + private void opCClassNot() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s += enc.length(bytes, s, end); + if (s > end) s = end; + sprev = sbegin; // break; + } + + private void opCClassNotSb() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + sprev = sbegin; // break; + } + + private boolean isNotInClassMB() { + int tlen = code[ip++]; + int mbLen = enc.length(bytes, s, end); + + if (!(s + mbLen <= range)) { + if (s >= range) return false; + s = end; + ip += tlen; + return true; + } + + int ss = s; + s += mbLen; + int c = enc.mbcToCode(bytes, ss, s); + + if (CodeRange.isInCodeRange(code, ip, c)) return false; + ip += tlen; + return true; + } + + private void opCClassMBNot() { + if (s >= range) {opFail(); return;} + if (!enc.isMbcHead(bytes, s, end)) { + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + return; + } + if (!isNotInClassMB()) {opFail(); return;} + sprev = sbegin; // break; + } + + private void opCClassMBNotSb() { + if (s >= range) {opFail(); return;} + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + } + + private void opCClassMIXNot() { + if (s >= range) {opFail(); return;} + if (enc.isMbcHead(bytes, s, end)) { + ip += BitSet.BITSET_SIZE; + if (!isNotInClassMB()) {opFail(); return;} + } else { + if (isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; + ip += tlen; + s++; + } + sprev = sbegin; // break; + } + + private void opCClassMIXNotSb() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + } + + private void opAnyChar() { + final int n; + if (s >= range || s + (n = enc.length(bytes, s, end)) > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + s += n; + sprev = sbegin; // break; + } + + private void opAnyCharSb() { + if (s >= range || bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAnyCharML() { + if (s >= range) {opFail(); return;} + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + s += n; + sprev = sbegin; // break; + } + + private void opAnyCharMLSb() { + if (s >= range) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAnyCharStar() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + if (enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + } + + private void opAnyCharStarSb() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + if (bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + } + + private void opAnyCharMLStar() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + } + + private void opAnyCharMLStarSb() { + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + sprev = s; + s++; + } + } + + private void opAnyCharStarPeekNext() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharStarPeekNextSb() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + byte b = bytes[s]; + if (c == b) pushAlt(ip + 1, s, sprev, pkeep); + if (b == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharMLStarPeekNext() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharMLStarPeekNextSb() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + sprev = s; + s++; + } + ip++; + sprev = sbegin; // break; + } + + // CEC + private void opStateCheckAnyCharStar() { + int mem = code[ip++]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + sprev = sbegin; // break; + } + + private void opStateCheckAnyCharStarSb() { + int mem = code[ip++]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + if (bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + sprev = sbegin; // break; + } + + // CEC + private void opStateCheckAnyCharMLStar() { + int mem = code[ip++]; + + final byte[]bytes = this.bytes; + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + sprev = sbegin; // break; + } + + private void opStateCheckAnyCharMLStarSb() { + int mem = code[ip++]; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + sprev = s; + s++; + } + sprev = sbegin; // break; + } + + private void opWord() { + if (s >= range || !enc.isMbcWord(bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opWordSb() { + if (s >= range || !enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAsciiWord() { + if (s >= range || !isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opNotWord() { + if (s >= range || enc.isMbcWord(bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opNotWordSb() { + if (s >= range || enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opNotAsciiWord() { + if (s >= range || isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opWordBound() { + if (s == str) { + if (s >= range || !enc.isMbcWord(bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } else { + if (enc.isMbcWord(bytes, s, end) == enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } + } + + private void opWordBoundSb() { + if (s == str) { + if (s >= range || !enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } else { + if (enc.isWord(bytes[s] & 0xff) == enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } + } + + private void opAsciiWordBound() { + if (s == str) { + if (s >= range || !isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } else { + if (isMbcAsciiWord(enc, bytes, s, end) == isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } + } + + private void opNotWordBound() { + if (s == str) { + if (s < range && enc.isMbcWord(bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } else { + if (enc.isMbcWord(bytes, s, end) != enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } + } + + private void opNotWordBoundSb() { + if (s == str) { + if (s < range && enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } else { + if (enc.isWord(bytes[s] & 0xff) != enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } + } + + private void opNotAsciiWordBound() { + if (s == str) { + if (s < range && isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } else { + if (isMbcAsciiWord(enc, bytes, s, end) != isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } + } + + private void opWordBegin() { + if (s < range && enc.isMbcWord(bytes, s, end)) { + if (s == str || !enc.isMbcWord(bytes, sprev, end)) return; + } + opFail(); + } + + private void opWordBeginSb() { + if (s < range && enc.isWord(bytes[s] & 0xff)) { + if (s == str || !enc.isWord(bytes[sprev] & 0xff)) return; + } + opFail(); + } + + private void opAsciiWordBegin() { + if (s < range && isMbcAsciiWord(enc, bytes, s, end)) { + if (s == str || !isMbcAsciiWord(enc, bytes, sprev, end)) return; + } + opFail(); + } + + private void opWordEnd() { + if (s != str && enc.isMbcWord(bytes, sprev, end)) { + if (s == end || !enc.isMbcWord(bytes, s, end)) return; + } + opFail(); + } + + private void opWordEndSb() { + if (s != str && enc.isWord(bytes[sprev] & 0xff)) { + if (s == end || !enc.isWord(bytes[s] & 0xff)) return; + } + opFail(); + } + + private void opAsciiWordEnd() { + if (s != str && isMbcAsciiWord(enc, bytes, sprev, end)) { + if (s == end || !isMbcAsciiWord(enc, bytes, s, end)) return; + } + opFail(); + } + + private void opBeginBuf() { + if (s != str) opFail(); + } + + private void opEndBuf() { + if (s != end) opFail(); + } + + private void opBeginLine() { + if (s == str) { + if (isNotBol(msaOptions)) opFail(); + return; + } else if (enc.isNewLine(bytes, sprev, end) && s != end) { + return; + } + opFail(); + } + + private void opBeginLineSb() { + if (s == str) { + if (isNotBol(msaOptions)) opFail(); + return; + } else if (bytes[sprev] == Encoding.NEW_LINE && s != end) { + return; + } + opFail(); + } + + private void opEndLine() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !enc.isNewLine(bytes, sprev, end)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (enc.isNewLine(bytes, s, end) || (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end))) { + return; + } + opFail(); + } + + private void opEndLineSb() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !(sprev < end && bytes[sprev] == Encoding.NEW_LINE)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (bytes[s] == Encoding.NEW_LINE || (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end))) { + return; + } + opFail(); + } + + private void opSemiEndBuf() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !enc.isNewLine(bytes, sprev, end)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (enc.isNewLine(bytes, s, end) && (s + enc.length(bytes, s, end)) == end) { + return; + } else if (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end)) { + int ss = s + enc.length(bytes, s, end); + ss += enc.length(bytes, ss, end); + if (ss == end) return; + } + opFail(); + } + + private void opBeginPosition() { + if (s != msaGpos) opFail(); + } + + private void opMemoryStartPush() { + int mem = code[ip++]; + pushMemStart(mem, s); + } + + private void opMemoryStart() { + int mem = code[ip++]; + repeatStk[memStartStk + mem] = s; + repeatStk[memEndStk + mem] = -1; + } + + private void opMemoryEndPush() { + int mem = code[ip++]; + pushMemEnd(mem, s); + } + + private void opMemoryEnd() { + int mem = code[ip++]; + repeatStk[memEndStk + mem] = s; + } + + private void opKeep() { + pkeep = s; + } + + private void opMemoryEndPushRec() { + int mem = code[ip++]; + int stkp = getMemStart(mem); /* should be before push mem-end. */ + pushMemEnd(mem, s); + repeatStk[memStartStk + mem] = stkp; + } + + private void opMemoryEndRec() { + int mem = code[ip++]; + repeatStk[memEndStk + mem] = s; + int stkp = getMemStart(mem); + repeatStk[memStartStk + mem] = bsAt(regex.btMemStart, mem) ? stkp : stack[stkp].getMemPStr(); + pushMemEndMark(mem); + } + + private boolean backrefInvalid(int mem) { + return repeatStk[memEndStk + mem] == INVALID_INDEX || repeatStk[memStartStk + mem] == INVALID_INDEX; + } + + private int backrefStart(int mem) { + int ms = repeatStk[memStartStk + mem]; + return bsAt(regex.btMemStart, mem) ? stack[ms].getMemPStr() : ms; + } + + private int backrefEnd(int mem) { + int me = repeatStk[memEndStk + mem]; + return bsAt(regex.btMemEnd, mem) ? stack[me].getMemPStr() : me; + } + + private void backref(int mem) { + if (mem > regex.numMem || backrefInvalid(mem)) {opFail(); return;} + int pstart = backrefStart(mem); + int pend = backrefEnd(mem); + int n = pend - pstart; + if (s + n > range) {opFail(); return;} + sprev = s; + + while (n-- > 0) if (bytes[pstart++] != bytes[s++]) {opFail(); return;} + + if (sprev < range) { // beyond string check + int len; + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + } + + private void opBackRef1() { + backref(1); + } + + private void opBackRef2() { + backref(2); + } + + private void opBackRefN() { + backref(code[ip++]); + } + + private void opBackRefNIC() { + int mem = code[ip++]; + if (mem > regex.numMem || backrefInvalid(mem)) {opFail(); return;} + int pstart = backrefStart(mem); + int pend = backrefEnd(mem); + int n = pend - pstart; + if (s + n > range) {opFail(); return;} + sprev = s; + + value = s; + if (!stringCmpIC(regex.caseFoldFlag, pstart, this, n, end)) {opFail(); return;} + s = value; + + if (sprev < range) { + int len; + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + } + + private void opBackRefMulti() { + int tlen = code[ip++]; + + int i; + loop:for (i=0; i range) continue; + + sprev = s; + int swork = s; + + while (n-- > 0) { + if (bytes[pstart++] != bytes[swork++]) continue loop; + } + + s = swork; + + int len; + + // beyond string check + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + + ip += tlen - i - 1; // * SIZE_MEMNUM (1) + break; /* success */ + } + if (i == tlen) {opFail(); return;} + } + + private void opBackRefMultiIC() { + int tlen = code[ip++]; + + int i; + loop:for (i=0; i range) continue; + + sprev = s; + + value = s; + if (!stringCmpIC(regex.caseFoldFlag, pstart, this, n, end)) continue loop; // STRING_CMP_VALUE_IC + s = value; + + int len; + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + + ip += tlen - i - 1; // * SIZE_MEMNUM (1) + break; /* success */ + } + if (i == tlen) {opFail(); return;} + } + + private boolean memIsInMemp(int mem, int num, int memp) { + for (int i=0; i= 0) { + StackEntry e = stack[k]; + + if (e.type == CALL_FRAME) { + level--; + } else if (e.type == RETURN) { + level++; + } else if (level == nest) { + if (e.type == MEM_START) { + if (memIsInMemp(e.getMemNum(), memNum, memp)) { + int pstart = e.getMemPStr(); + if (pend != -1) { + if (pend - pstart > end - s) return false; /* or goto next_mem; */ + int p = pstart; + + value = s; + if (ignoreCase) { + if (!stringCmpIC(caseFoldFlag, pstart, this, pend - pstart, end)) { + return false; /* or goto next_mem; */ + } + } else { + while (p < pend) { + if (bytes[p++] != bytes[value++]) return false; /* or goto next_mem; */ + } + } + s = value; + + return true; + } + } + } else if (e.type == MEM_END) { + if (memIsInMemp(e.getMemNum(), memNum, memp)) { + pend = e.getMemPStr(); + } + } + } + k--; + } + return false; + } + + private void opBackRefAtLevel() { + int ic = code[ip++]; + int level = code[ip++]; + int tlen = code[ip++]; + + sprev = s; + if (backrefMatchAtNestedLevel(ic != 0, regex.caseFoldFlag, level, tlen, ip)) { // (s) and (end) implicit + int len; + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + ip += tlen; // * SIZE_MEMNUM + } else { + {opFail(); return;} + } + } + + /* no need: IS_DYNAMIC_OPTION() == 0 */ + @SuppressWarnings("unused") + private void opSetOptionPush() { + // option = code[ip++]; // final for now + pushAlt(ip, s, sprev, pkeep); + ip += OPSize.SET_OPTION + OPSize.FAIL; + } + + @SuppressWarnings("unused") + private void opSetOption() { + // option = code[ip++]; // final for now + } + + private void opNullCheckStart() { + int mem = code[ip++]; + pushNullCheckStart(mem, s); + } + + private void nullCheckFound() { + // null_check_found: + /* empty loop founded, skip next instruction */ + switch(code[ip++]) { + case OPCode.JUMP: + case OPCode.PUSH: + ip++; // p += SIZE_RELADDR; + break; + case OPCode.REPEAT_INC: + case OPCode.REPEAT_INC_NG: + case OPCode.REPEAT_INC_SG: + case OPCode.REPEAT_INC_NG_SG: + ip++; // p += SIZE_MEMNUM; + break; + default: + throw new InternalException(ErrorMessages.UNEXPECTED_BYTECODE); + } // switch + } + + private void opNullCheckEnd() { + int mem = code[ip++]; + int isNull = nullCheck(mem, s); /* mem: null check id */ + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END: skip id:" + mem + ", s:" + s); + } + + nullCheckFound(); + } + } + + // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + private void opNullCheckEndMemST() { + int mem = code[ip++]; /* mem: null check id */ + int isNull = nullCheckMemSt(mem, s); + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END_MEMST: skip id:" + mem + ", s:" + s); + } + + if (isNull == -1) {opFail(); return;} + nullCheckFound(); + } + } + + // USE_SUBEXP_CALL + private void opNullCheckEndMemSTPush() { + int mem = code[ip++]; /* mem: null check id */ + + int isNull; + if (Config.USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT) { + isNull = nullCheckMemStRec(mem, s); + } else { + isNull = nullCheckRec(mem, s); + } + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END_MEMST_PUSH: skip id:" + mem + ", s:" + s); + } + + if (isNull == -1) {opFail(); return;} + nullCheckFound(); + } else { + pushNullCheckEnd(mem); + } + } + + private void opJump() { + ip += code[ip] + 1; + } + + private void opPush() { + int addr = code[ip++]; + pushAlt(ip + addr, s, sprev, pkeep); + } + + // CEC + private void opStateCheckPush() { + int mem = code[ip++]; + if (stateCheckVal(s, mem)) {opFail(); return;} + int addr = code[ip++]; + pushAltWithStateCheck(ip + addr, s, sprev, mem, pkeep); + } + + // CEC + private void opStateCheckPushOrJump() { + int mem = code[ip++]; + int addr= code[ip++]; + + if (stateCheckVal(s, mem)) { + ip += addr; + } else { + pushAltWithStateCheck(ip + addr, s, sprev, mem, pkeep); + } + } + + // CEC + private void opStateCheck() { + int mem = code[ip++]; + if (stateCheckVal(s, mem)) {opFail(); return;} + pushStateCheck(s, mem); + } + + private void opPop() { + popOne(); + } + + private void opPushOrJumpExact1() { + int addr = code[ip++]; + // beyond string check + if (s < range && code[ip] == bytes[s]) { + ip++; + pushAlt(ip + addr, s, sprev, pkeep); + return; + } + ip += addr + 1; + } + + private void opPushIfPeekNext() { + int addr = code[ip++]; + // beyond string check + if (s < range && code[ip] == bytes[s]) { + ip++; + pushAlt(ip + addr, s, sprev, pkeep); + return; + } + ip++; + } + + private void opRepeat() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int addr= code[ip++]; + + // ensure1(); + repeatStk[mem] = stk; + pushRepeat(mem, ip); + + if (regex.repeatRangeLo[mem] == 0) { // lower + pushAlt(ip + addr, s, sprev, pkeep); + } + } + + private void opRepeatNG() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int addr= code[ip++]; + + // ensure1(); + repeatStk[mem] = stk; + pushRepeat(mem, ip); + + if (regex.repeatRangeLo[mem] == 0) { + pushAlt(ip, s, sprev, pkeep); + ip += addr; + } + } + + private void repeatInc(int mem, int si) { + StackEntry e = stack[si]; + + e.increaseRepeatCount(); + + if (e.getRepeatCount() >= regex.repeatRangeHi[mem]) { + /* end of repeat. Nothing to do. */ + } else if (e.getRepeatCount() >= regex.repeatRangeLo[mem]) { + pushAlt(ip, s, sprev, pkeep); + ip = e.getRepeatPCode(); /* Don't use stkp after PUSH. */ + } else { + ip = e.getRepeatPCode(); + } + pushRepeatInc(si); + } + + private void opRepeatInc() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int si = repeatStk[mem]; + repeatInc(mem, si); + } + + private void opRepeatIncSG() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int si = getRepeat(mem); + repeatInc(mem, si); + } + + private void repeatIncNG(int mem, int si) { + StackEntry e = stack[si]; + + e.increaseRepeatCount(); + + if (e.getRepeatCount() < regex.repeatRangeHi[mem]) { + if (e.getRepeatCount() >= regex.repeatRangeLo[mem]) { + int pcode = e.getRepeatPCode(); + pushRepeatInc(si); + pushAlt(pcode, s, sprev, pkeep); + } else { + ip = e.getRepeatPCode(); + pushRepeatInc(si); + } + } else if (e.getRepeatCount() == regex.repeatRangeHi[mem]) { + pushRepeatInc(si); + } + } + + private void opRepeatIncNG() { + int mem = code[ip++]; + int si = repeatStk[mem]; + repeatIncNG(mem, si); + } + + private void opRepeatIncNGSG() { + int mem = code[ip++]; + int si = getRepeat(mem); + repeatIncNG(mem, si); + } + + private void opPushPos() { + pushPos(s, sprev, pkeep); + } + + private void opPopPos() { + StackEntry e = stack[posEnd()]; + s = e.getStatePStr(); + sprev= e.getStatePStrPrev(); + } + + private void opPushPosNot() { + int addr = code[ip++]; + pushPosNot(ip + addr, s, sprev, pkeep); + } + + private void opFailPos() { + popTilPosNot(); + opFail(); + } + + private void opPushStopBT() { + pushStopBT(); + } + + private void opPopStopBT() { + stopBtEnd(); + } + + private void opLookBehind() { + int tlen = code[ip++]; + s = enc.stepBack(bytes, str, s, end, tlen); + if (s == -1) {opFail(); return;} + sprev = enc.prevCharHead(bytes, str, s, end); + } + + private void opLookBehindSb() { + int tlen = code[ip++]; + s -= tlen; + if (s < str) {opFail(); return;} + sprev = s == str ? -1 : s - 1; + } + + private void opPushLookBehindNot() { + int addr = code[ip++]; + int tlen = code[ip++]; + int q = enc.stepBack(bytes, str, s, end, tlen); + if (q == -1) { + /* too short case -> success. ex. /(? aend && s > absent) { + pop(); + opFail(); + return; + } else if (s >= aend && s > absent) { + if (s > aend || s > end) { + opFail(); + return; + } + ip += addr; + } else { + + pushAlt(ip + addr, s, sprev, pkeep); + int n = (s >= end) ? 1 : enc.length(bytes, s, end); + pushAbsentPos(absent, range); + pushAlt(selfip, s + n, s, pkeep); + pushAbsent(); + range = aend; + } + } + + private void opAbsentEnd() { + if (sprev < range) range = sprev; + if (Config.DEBUG_MATCH) System.out.println("ABSENT_END: end:" + range); + popTilAbsent(); + opFail(); + return; + // sprev = sbegin; // break; + } + + private void opCall() { + int addr = code[ip++]; + pushCallFrame(ip); + ip = addr; // absolute address + } + + private void opCallout() { + int calloutId = code[ip++]; + CalloutHandler handler = getCalloutHandler(); + if (handler == null) throw new IllegalStateException("callout pattern has no handler"); + CalloutResult result = handler.execute(calloutId, this); + if (result == null) throw new NullPointerException("callout result"); + if (result.getBacktrackToken() != null) pushCallout(result.getBacktrackToken()); + if (result.getAction() == CalloutAction.FAIL) opFail(); + } + + @Override + public int currentBytePosition() { + return s - str; + } + + @Override + public int captureCount() { + return regex.numMem; + } + + @Override + public int captureBegin(int capture) { + checkCapture(capture); + if (capture == 0) return sstart - str; + int value = repeatStk[memStartStk + capture]; + if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + return (bsAt(regex.btMemStart, capture) ? stack[value].getMemPStr() : value) - str; + } + + @Override + public int captureEnd(int capture) { + checkCapture(capture); + if (capture == 0) return s - str; + int value = repeatStk[memEndStk + capture]; + if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + return (bsAt(regex.btMemEnd, capture) ? stack[value].getMemPStr() : value) - str; + } + + private void checkCapture(int capture) { + if (capture < 0 || capture > regex.numMem) { + throw new IndexOutOfBoundsException("capture " + capture); + } + } + + private void opReturn() { + ip = sreturn(); + pushReturn(); + } + + private void opFail() { + if (stack == null) { + ip = regex.codeLength - 1; + return; + } + + + StackEntry e = pop(); + ip = e.getStatePCode(); + s = e.getStatePStr(); + sprev = e.getStatePStrPrev(); + pkeep = e.getPKeep(); + + if (USE_CEC) { + if (((SCStackEntry)e).getStateCheck() != 0) { + e.type = STATE_CHECK_MARK; + stk++; + } + } + } + + private int finish() { + return bestLen; + } + + private void debugMatchBegin() { + Config.log.println("match_at: " + "str: " + str + ", end: " + end + ", start: " + sstart + ", sprev: " + sprev); + Config.log.println("size: " + (end - str) + ", start offset: " + (sstart - str)); + } + + private void debugMatchLoop() { + Config.log.printf("%4d", (s - str)).print("> \""); + int q, i; + for (i = 0, q = s; i < 7 && q < end && s >= 0; i++) { + int len = enc.length(bytes, q, end); + while (len-- > 0) { + if (q < end) { + Config.log.print(new String(bytes, q++, 1)); + } + } + } + String str = q < end ? "...\"" : "\""; + q += str.length(); + Config.log.print(str); + for (i = 0; i < 20 - (q - s); i++) + Config.log.print(" "); + StringBuilder sb = new StringBuilder(); + new ByteCodePrinter(regex).compiledByteCodeToString(sb, ip); + Config.log.println(sb.toString()); + } +} diff --git a/third_party/joni/src/org/joni/ByteCodePrinter.java b/third_party/joni/src/org/joni/ByteCodePrinter.java new file mode 100644 index 000000000..80c4b7615 --- /dev/null +++ b/third_party/joni/src/org/joni/ByteCodePrinter.java @@ -0,0 +1,392 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.constants.internal.Arguments; +import org.joni.constants.internal.OPCode; +import org.joni.constants.internal.OPSize; +import org.joni.exception.InternalException; + +class ByteCodePrinter { + final int[] code; + final int codeLength; + final byte[][] templates; + final Encoding enc; + + public ByteCodePrinter(Regex regex) { + code = regex.code; + codeLength = regex.codeLength; + templates = regex.templates; + enc = regex.enc; + } + + public String byteCodeListToString() { + return compiledByteCodeListToString(); + } + + private void pString(StringBuilder sb, int len, int s) { + sb.append(':'); + while (len-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + } + + private void pLenString(StringBuilder sb, int len, int mbLen, int s) { + int x = len * mbLen; + sb.append(':').append(len).append(':'); + while (x-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + } + + private void pLenStringFromTemplate(StringBuilder sb, int len, int mbLen, byte[]tm, int idx) { + int x = len * mbLen; + sb.append(":T:").append(len).append(':'); + while (x-- > 0) sb.append(new String(tm, idx++, 1)); + } + + public int compiledByteCodeToString(StringBuilder sb, int bp) { + int len, n, mem, addr, scn, cod; + BitSet bs; + int tm, idx; + + sb.append('[').append(OPCode.OpCodeNames[code[bp]]); + int argType = OPCode.OpCodeArgTypes[code[bp]]; + int ip = bp; + if (argType != Arguments.SPECIAL) { + bp++; + switch (argType) { + case Arguments.NON: + break; + + case Arguments.RELADDR: + sb.append(":(").append(code[bp]).append(')'); + bp += OPSize.RELADDR; + break; + + case Arguments.ABSADDR: + sb.append(":(").append(code[bp]).append(')'); + bp += OPSize.ABSADDR; + break; + + case Arguments.LENGTH: + sb.append(':').append(code[bp]); + bp += OPSize.LENGTH; + break; + + case Arguments.MEMNUM: + sb.append(':').append(code[bp]); + bp += OPSize.MEMNUM; + break; + + case Arguments.OPTION: + sb.append(':').append(code[bp]); + bp += OPSize.OPTION; + break; + + case Arguments.STATE_CHECK: + sb.append(':').append(code[bp]); + bp += OPSize.STATE_CHECK; + break; + } + } else { + switch (code[bp++]) { + case OPCode.EXACT1: + case OPCode.ANYCHAR_STAR_PEEK_NEXT: + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: + pString(sb, 1, bp++); + break; + + case OPCode.EXACT2: + pString(sb, 2, bp); + bp += 2; + break; + + case OPCode.EXACT3: + pString(sb, 3, bp); + bp += 3; + break; + + case OPCode.EXACT4: + pString(sb, 4, bp); + bp += 4; + break; + + case OPCode.EXACT5: + pString(sb, 5, bp); + bp += 5; + break; + + case OPCode.EXACTN: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 1, templates[tm], idx); + } else { + pLenString(sb, len, 1, bp); + bp += len; + } + break; + + case OPCode.EXACTMB2N1: + pString(sb, 2, bp); + bp += 2; + break; + + case OPCode.EXACTMB2N2: + pString(sb, 4, bp); + bp += 4; + break; + + case OPCode.EXACTMB2N3: + pString(sb, 6, bp); + bp += 6; + break; + + case OPCode.EXACTMB2N: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 2, templates[tm], idx); + } else { + pLenString(sb, len, 2, bp); + bp += len * 2; + } + break; + + case OPCode.EXACTMB3N: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 3, templates[tm], idx); + } else { + pLenString(sb, len, 3, bp); + bp += len * 3; + } + break; + + case OPCode.EXACTMBN: + int mbLen = code[bp]; + bp += OPSize.LENGTH; + len = code[bp]; + bp += OPSize.LENGTH; + n = len * mbLen; + + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + sb.append(":T:").append(mbLen).append(":").append(len).append(":"); + while (n-- > 0) sb.append(new String(templates[tm], idx++, 1)); + } else { + sb.append(":").append(mbLen).append(":").append(len).append(":"); + while (n-- > 0) sb.append(new String(new byte[]{(byte)code[bp++]})); + } + + break; + + case OPCode.EXACT1_IC: + case OPCode.EXACT1_IC_SB: + final int MAX_CHAR_LENGTH = 6; + byte[]bytes = new byte[MAX_CHAR_LENGTH]; + for (int i = 0; bp + i < code.length && i < MAX_CHAR_LENGTH; i++) bytes[i] = (byte)code[bp + i]; + len = enc.length(bytes, 0, MAX_CHAR_LENGTH); + pString(sb, len, bp); + bp += len; + break; + + case OPCode.EXACTN_IC: + case OPCode.EXACTN_IC_SB: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 1, templates[tm], idx); + } else { + pLenString(sb, len, 1, bp); + bp += len; + } + break; + + case OPCode.CCLASS: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + sb.append(':').append(n); + break; + + case OPCode.CCLASS_NOT: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + sb.append(':').append(n); + break; + + case OPCode.CCLASS_MB: + case OPCode.CCLASS_MB_NOT: + len = code[bp]; + bp += OPSize.LENGTH; + cod = code[bp]; + //bp += OPSize.CODE_POINT; + bp += len; + sb.append(':').append(cod).append(':').append(len); + break; + + case OPCode.CCLASS_MIX: + case OPCode.CCLASS_MIX_NOT: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + len = code[bp]; + bp += OPSize.LENGTH; + cod = code[bp]; + //bp += OPSize.CODE_POINT; + bp += len; + sb.append(':').append(n).append(':').append(cod).append(':').append(len); + break; + + case OPCode.BACKREFN_IC: + mem = code[bp]; + bp += OPSize.MEMNUM; + sb.append(':').append(mem); + break; + + case OPCode.BACKREF_MULTI_IC: + case OPCode.BACKREF_MULTI: + sb.append(' '); + len = code[bp]; + bp += OPSize.LENGTH; + for (int i=0; i 0) sb.append(", "); + sb.append(mem); + } + break; + + case OPCode.BACKREF_WITH_LEVEL: { + int option = code[bp]; + bp += OPSize.OPTION; + sb.append(':').append(option); + int level = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(level); + sb.append(' '); + len = code[bp]; + bp += OPSize.LENGTH; + for (int i=0; i 0) sb.append(", "); + sb.append(mem); + } + break; + } + + case OPCode.REPEAT: + case OPCode.REPEAT_NG: + mem = code[bp]; + bp += OPSize.MEMNUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(mem).append(':').append(addr); + break; + + case OPCode.PUSH_OR_JUMP_EXACT1: + case OPCode.PUSH_IF_PEEK_NEXT: + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(":(").append(addr).append(')'); + pString(sb, 1, bp); + bp++; + break; + + case OPCode.LOOK_BEHIND: + len = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(len); + break; + + case OPCode.PUSH_LOOK_BEHIND_NOT: + addr = code[bp]; + bp += OPSize.RELADDR; + len = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(len).append(":(").append(addr).append(')'); + break; + + case OPCode.STATE_CHECK_PUSH: + case OPCode.STATE_CHECK_PUSH_OR_JUMP: + scn = code[bp]; + bp += OPSize.STATE_CHECK_NUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(scn).append(":(").append(addr).append(')'); + break; + + case OPCode.CONDITION: + mem = code[bp]; + bp += OPSize.MEMNUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(mem).append(":").append(addr); + break; + + default: + throw new InternalException("undefined code: " + code[--bp]); + } + } + + sb.append(']'); + if (Config.DEBUG_COMPILE_BYTE_CODE_INFO) sb.append('@').append(ip).append('(').append(bp - ip).append(')'); + return bp; + } + + private String compiledByteCodeListToString() { + StringBuilder sb = new StringBuilder("code length: ").append(codeLength).append('\n'); + int ncode = -1; + int bp = 0; + int end = codeLength; + + while (bp < end) { + ncode++; + sb.append(ncode % 5 == 0 ? '\n' : ' '); + bp = compiledByteCodeToString(sb, bp); + } + sb.append("\n"); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/CalloutAction.java b/third_party/joni/src/org/joni/CalloutAction.java new file mode 100644 index 000000000..a92d779f2 --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutAction.java @@ -0,0 +1,26 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Match control requested by a generic callout handler. */ +public enum CalloutAction { + CONTINUE, + FAIL +} diff --git a/third_party/joni/src/org/joni/CalloutHandler.java b/third_party/joni/src/org/joni/CalloutHandler.java new file mode 100644 index 000000000..3722b2caa --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutHandler.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Runtime-neutral bridge invoked from the matcher bytecode loop. */ +public interface CalloutHandler { + CalloutResult execute(int calloutId, MatchView match); + + void unwind(Object backtrackToken); +} diff --git a/third_party/joni/src/org/joni/CalloutResult.java b/third_party/joni/src/org/joni/CalloutResult.java new file mode 100644 index 000000000..194ee57bc --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutResult.java @@ -0,0 +1,53 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Immutable result of a match-time callout. */ +public final class CalloutResult { + public static final CalloutResult CONTINUE = new CalloutResult(CalloutAction.CONTINUE, null); + public static final CalloutResult FAIL = new CalloutResult(CalloutAction.FAIL, null); + + private final CalloutAction action; + private final Object backtrackToken; + + private CalloutResult(CalloutAction action, Object backtrackToken) { + if (action == null) throw new NullPointerException("action"); + this.action = action; + this.backtrackToken = backtrackToken; + } + + public static CalloutResult continueWith(Object backtrackToken) { + return backtrackToken == null ? CONTINUE + : new CalloutResult(CalloutAction.CONTINUE, backtrackToken); + } + + public static CalloutResult failWith(Object backtrackToken) { + return backtrackToken == null ? FAIL + : new CalloutResult(CalloutAction.FAIL, backtrackToken); + } + + public CalloutAction getAction() { + return action; + } + + public Object getBacktrackToken() { + return backtrackToken; + } +} diff --git a/third_party/joni/src/org/joni/CaptureTreeNode.java b/third_party/joni/src/org/joni/CaptureTreeNode.java new file mode 100644 index 000000000..ff0c52fde --- /dev/null +++ b/third_party/joni/src/org/joni/CaptureTreeNode.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +final class CaptureTreeNode { + int group; + int beg; + int end; + // int allocated; + int numChildren; + CaptureTreeNode[] children; + + CaptureTreeNode() { + beg = Region.REGION_NOTPOS; + end = Region.REGION_NOTPOS; + group = -1; + } + + static final int HISTORY_TREE_INIT_ALLOC_SIZE = 8; + + void addChild(CaptureTreeNode child) { + if (children == null) { + children = new CaptureTreeNode[HISTORY_TREE_INIT_ALLOC_SIZE]; + } else if (numChildren >= children.length) { + CaptureTreeNode[] tmp = new CaptureTreeNode[children.length << 1]; + System.arraycopy(children, 0, tmp, 0, children.length); + children = tmp; + } + + children[numChildren] = child; + numChildren++; + } + + void clear() { + for (int i = 0; i < numChildren; i++) { + children[i] = null; // ??? + } + numChildren = 0; + beg = end = Region.REGION_NOTPOS; + group = -1; + } + + CaptureTreeNode cloneTree() { + CaptureTreeNode clone = new CaptureTreeNode(); + clone.beg = beg; + clone.end = end; + + for (int i = 0; i < numChildren; i++) { + CaptureTreeNode child = children[i].cloneTree(); + clone.addChild(child); + } + return clone; + } +} diff --git a/third_party/joni/src/org/joni/CodeRangeBuffer.java b/third_party/joni/src/org/joni/CodeRangeBuffer.java new file mode 100644 index 000000000..c5d0e6ca4 --- /dev/null +++ b/third_party/joni/src/org/joni/CodeRangeBuffer.java @@ -0,0 +1,403 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.exception.ErrorMessages; +import org.joni.exception.ValueException; + +public final class CodeRangeBuffer { + private static final int INIT_MULTI_BYTE_RANGE_SIZE = 5; + public static final int LAST_CODE_POINT = 0x7fffffff; + + private int[]p; + private int used; + + public CodeRangeBuffer() { + p = new int[INIT_MULTI_BYTE_RANGE_SIZE]; + writeCodePoint(0, 0); + } + + public int[]getCodeRange() { + return p; + } + + public int getUsed() { + return used; + } + + private CodeRangeBuffer(CodeRangeBuffer orig) { + p = new int[orig.p.length]; + System.arraycopy(orig.p, 0, p, 0, p.length); + used = orig.used; + } + + public void expand(int low) { + int length = p.length; + do { length <<= 1; } while (length < low); + int[]tmp = new int[length]; + System.arraycopy(p, 0, tmp, 0, used); + p = tmp; + } + + public void ensureSize(int size) { + int length = p.length; + while (length < size ) { length <<= 1; } + if (p.length != length) { + int[]tmp = new int[length]; + System.arraycopy(p, 0, tmp, 0, used); + p = tmp; + } + } + + private void moveRight(int from, int to, int n) { + if (to + n > p.length) expand(to + n); + System.arraycopy(p, from, p, to, n); + if (to + n > used) used = to + n; + } + + protected void moveLeft(int from, int to, int n) { + System.arraycopy(p, from, p, to, n); + } + + private void moveLeftAndReduce(int from, int to) { + System.arraycopy(p, from, p, to, used - from); + used -= from - to; + } + + public void writeCodePoint(int pos, int b) { + int u = pos + 1; + if (p.length < u) expand(u); + p[pos] = b; + if (used < u) used = u; + } + + @Override + public CodeRangeBuffer clone() { + return new CodeRangeBuffer(this); + } + + // add_code_range_to_buf + public static CodeRangeBuffer addCodeRangeToBuff(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to) { + return addCodeRangeToBuff(pbuf, env, from, to, true); + } + + public static CodeRangeBuffer addCodeRangeToBuff(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to, boolean checkDup) { + if (from > to) { + int n = from; + from = to; + to = n; + } + + int n; + + if (pbuf == null) { + pbuf = new CodeRangeBuffer(); // move to CClassNode + n = 0; + } else { + n = pbuf.p[0]; + } + + int[] data = pbuf.p; // data = bbuf.p + final int dataOff = 1; // data++ + + int bound = from == 0 ? 0 : n; + int low; + for (low = 0; low < bound; ) { + int x = (low + bound) >>> 1; + if (from - 1 > data[dataOff + x * 2 + 1]) { + low = x + 1; + } else { + bound = x; + } + } + + int high = to == LAST_CODE_POINT ? n : low; + for (bound = n; high < bound; ) { + int x = (high + bound) >>> 1; + if (to + 1 >= data[dataOff + x * 2]) { + high = x + 1; + } else { + bound = x; + } + } + + int incN = low + 1 - high; + + if (n + incN > Config.MAX_MULTI_BYTE_RANGES_NUM) throw new ValueException(ErrorMessages.TOO_MANY_MULTI_BYTE_RANGES); + + if (incN != 1) { + if (checkDup) { + if (from <= data[dataOff + low * 2 + 1] && (data[dataOff + low * 2] <= from || data[dataOff + low * 2 + 1] <= to)) env.ccDuplicateWarn(); + } + + if (from > data[dataOff + low * 2]) from = data[dataOff + low * 2]; + if (to < data[dataOff + (high - 1) * 2 + 1]) to = data[dataOff + (high - 1) * 2 + 1]; + } + + if (incN != 0) { + int fromPos = 1 + high * 2; + int toPos = 1 + (low + 1) * 2; + + if (incN > 0) { + if (high < n) { + int size = (n - high) * 2; + pbuf.moveRight(fromPos, toPos, size); + } + } else { + pbuf.moveLeftAndReduce(fromPos, toPos); + } + } + + int pos = 1 + low * 2; + // pbuf.ensureSize(pos + 2); + pbuf.writeCodePoint(pos, from); + pbuf.writeCodePoint(pos + 1, to); + n += incN; + pbuf.writeCodePoint(0, n); + + return pbuf; + } + + // add_code_range, be aware of it returning null! + public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to) { + return addCodeRange(pbuf, env, from, to, true); + } + + public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to, boolean checkDup) { + if (from >to) { + if (env.syntax.allowEmptyRangeInCC()) { + return pbuf; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + return addCodeRangeToBuff(pbuf, env, from, to, checkDup); + } + + private static int mbcodeStartPosition(Encoding enc) { + return enc.minLength() > 1 ? 0 : 0x80; + } + + + // SET_ALL_MULTI_BYTE_RANGE + protected static CodeRangeBuffer setAllMultiByteRange(ScanEnvironment env, CodeRangeBuffer pbuf) { + return addCodeRangeToBuff(pbuf, env, mbcodeStartPosition(env.enc), LAST_CODE_POINT); + } + + // ADD_ALL_MULTI_BYTE_RANGE + public static CodeRangeBuffer addAllMultiByteRange(ScanEnvironment env, CodeRangeBuffer pbuf) { + if (!env.enc.isSingleByte()) return setAllMultiByteRange(env, pbuf); + return pbuf; + } + + // not_code_range_buf + public static CodeRangeBuffer notCodeRangeBuff(ScanEnvironment env, CodeRangeBuffer bbuf) { + CodeRangeBuffer pbuf = null; + + if (bbuf == null) return setAllMultiByteRange(env, pbuf); + + int[]p = bbuf.p; + int n = p[0]; + + if (n <= 0) return setAllMultiByteRange(env, pbuf); + + int pre = mbcodeStartPosition(env.enc); + + int from; + int to = 0; + for (int i=0; i to1) break; + } + + if (from1 <= to1) { + pbuf = addCodeRangeToBuff(pbuf, env, from1, to1); + } + + return pbuf; + } + + // and_code_range_buf + public static CodeRangeBuffer andCodeRangeBuff(CodeRangeBuffer bbuf1, boolean not1, + CodeRangeBuffer bbuf2, boolean not2, ScanEnvironment env) { + CodeRangeBuffer pbuf = null; + + if (bbuf1 == null) { + if (not1 && bbuf2 != null) return bbuf2.clone(); /* not1 != 0 -> not2 == 0 */ + return null; + } else if (bbuf2 == null) { + if (not2) return bbuf1.clone(); + return null; + } + + if (not1) { + CodeRangeBuffer tbuf; + boolean tnot; + // swap + tnot = not1; not1 = not2; not2 = tnot; + tbuf = bbuf1; bbuf1 = bbuf2; bbuf2 = tbuf; + } + + int[]p1 = bbuf1.p; + int n1 = p1[0]; + int[]p2 = bbuf2.p; + int n2 = p2[0]; + + if (!not2 && !not1) { /* 1 AND 2 */ + for (int i=0; i to1) break; + if (to2 < from1) continue; + int from = from1 > from2 ? from1 : from2; + int to = to1 < to2 ? to1 : to2; + pbuf = addCodeRangeToBuff(pbuf, env, from, to); + } + } + } else if (!not1) { /* 1 AND (not 2) */ + for (int i=0; i 0 && i % 6 == 0) buf.append("\n "); + } + + return buf.toString(); + } + + private static String rangeNumToString(int num){ + return "0x" + Integer.toString(num, 16); + } +} diff --git a/third_party/joni/src/org/joni/Compiler.java b/third_party/joni/src/org/joni/Compiler.java new file mode 100644 index 000000000..2cffd94ba --- /dev/null +++ b/third_party/joni/src/org/joni/Compiler.java @@ -0,0 +1,196 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.ast.AnchorNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.ListNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.NodeType; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.SyntaxException; + +abstract class Compiler implements ErrorMessages { + protected final Analyser analyser; + protected final Encoding enc; + protected final Regex regex; + + protected Compiler(Analyser analyser) { + this.analyser = analyser; + this.regex = analyser.regex; + this.enc = regex.enc; + } + + final void compile(Node root) { + prepare(); + compileTree(root); + finish(); + } + + protected abstract void prepare(); + protected abstract void finish(); + + protected abstract void compileAltNode(ListNode node); + + private void compileStringRawNode(StringNode sn) { + if (sn.length() <= 0) return; + addCompileString(sn.bytes, sn.p, 1 /*sb*/, sn.length(), false); + } + + private void compileStringNode(StringNode node) { + StringNode sn = node; + if (sn.length() <= 0) return; + + boolean ambig = sn.isAmbig(); + + int p, prev; + p = prev = sn.p; + int end = sn.end; + byte[]bytes = sn.bytes; + int prevLen = enc.length(bytes, p, end); + p += prevLen; + int blen = prevLen; + + while (p < end) { + int len = enc.length(bytes, p, end); + if (len == prevLen || ambig) { + blen += len; + } else { + addCompileString(bytes, prev, prevLen, blen, ambig); + prev = p; + blen = len; + prevLen = len; + } + p += len; + } + addCompileString(bytes, prev, prevLen, blen, ambig); + } + + protected abstract void addCompileString(byte[]bytes, int p, int mbLength, int strLength, boolean ignoreCase); + + protected abstract void compileCClassNode(CClassNode node); + protected abstract void compileCTypeNode(CTypeNode node); + protected abstract void compileAnyCharNode(); + protected abstract void compileCallNode(CallNode node); + protected abstract void compileCalloutNode(CalloutNode node); + protected abstract void compileBackrefNode(BackRefNode node); + protected abstract void compileCECQuantifierNode(QuantifierNode node); + protected abstract void compileNonCECQuantifierNode(QuantifierNode node); + protected abstract void compileOptionNode(EncloseNode node); + protected abstract void compileEncloseNode(EncloseNode node); + protected abstract void compileAnchorNode(AnchorNode node); + + protected final void compileTree(Node node) { + if (node instanceof CalloutNode) { + compileCalloutNode((CalloutNode)node); + return; + } + switch (node.getType()) { + case NodeType.LIST: + ListNode lin = (ListNode)node; + do { + compileTree(lin.value); + } while ((lin = lin.tail) != null); + break; + + case NodeType.ALT: + compileAltNode((ListNode)node); + break; + + case NodeType.STR: + StringNode sn = (StringNode)node; + if (sn.isRaw()) { + compileStringRawNode(sn); + } else { + compileStringNode(sn); + } + break; + + case NodeType.CCLASS: + compileCClassNode((CClassNode)node); + break; + + case NodeType.CTYPE: + compileCTypeNode((CTypeNode)node); + break; + + case NodeType.CANY: + compileAnyCharNode(); + break; + + case NodeType.BREF: + compileBackrefNode((BackRefNode)node); + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + compileCallNode((CallNode)node); + break; + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + if (Config.USE_CEC) { + compileCECQuantifierNode((QuantifierNode)node); + } else { + compileNonCECQuantifierNode((QuantifierNode)node); + } + break; + + case NodeType.ENCLOSE: + EncloseNode enode = (EncloseNode)node; + if (enode.isOption()) { + compileOptionNode(enode); + } else { + compileEncloseNode(enode); + } + break; + + case NodeType.ANCHOR: + compileAnchorNode((AnchorNode)node); + break; + + default: + // undefined node type + newInternalException(PARSER_BUG); + } // switch + } + + protected final void compileTreeNTimes(Node node, int n) { + for (int i=0; i, \k */ + + boolean USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT = ConfigSupport.getBoolean("joni.use_monomaniac_check_captures_in_endless_repeat", true); /* /(?:()|())*\2/ */ + boolean USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE = ConfigSupport.getBoolean("joni.use_newline_at_end_of_string_has_empty_line", true); /* /\n$/ =~ "\n" */ + boolean USE_WARNING_REDUNDANT_NESTED_REPEAT_OPERATOR = ConfigSupport.getBoolean("joni.use_warning_redundant_nested_repeat_operator", true); + + boolean CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS = ConfigSupport.getBoolean("joni.case_fold_is_applied_inside_negative_cclass", true); + + boolean USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE = ConfigSupport.getBoolean("joni.use_match_range_must_be_inside_of_specified_range", false); + boolean USE_CAPTURE_HISTORY = ConfigSupport.getBoolean("joni.use_capture_history", false); + boolean USE_VARIABLE_META_CHARS = ConfigSupport.getBoolean("joni.use_variable_meta_chars", true); + boolean USE_WORD_BEGIN_END = ConfigSupport.getBoolean("joni.use_word_begin_end", true); /* "\<": word-begin, "\>": word-end */ + boolean USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE = ConfigSupport.getBoolean("joni.use_find_longest_search_all_of_range", true); + boolean USE_SUNDAY_QUICK_SEARCH = ConfigSupport.getBoolean("joni.use_sunday_quick_search", true); + boolean USE_CEC = ConfigSupport.getBoolean("joni.use_cec", false); + boolean USE_DYNAMIC_OPTION = ConfigSupport.getBoolean("joni.use_dynamic_option", false); + boolean USE_BYTE_MAP = ConfigSupport.getBoolean("joni.use_byte_map", OptExactInfo.OPT_EXACT_MAXLEN <= CHAR_TABLE_SIZE); + boolean USE_INT_MAP_BACKWARD = ConfigSupport.getBoolean("joni.use_int_map_backward", false); + + int NREGION = ConfigSupport.getInt("joni.nregion", 10); + int MAX_BACKREF_NUM = ConfigSupport.getInt("joni.max_backref_num", 1000); + int MAX_CAPTURE_GROUP_NUM = ConfigSupport.getInt("joni.max_capture_group_num", 32767); + int MAX_REPEAT_NUM = ConfigSupport.getInt("joni.max_multi_byte_ranges_num", 100000); + int MAX_MULTI_BYTE_RANGES_NUM = ConfigSupport.getInt("joni.max_multi_byte_ranges_num", 10000); + + // internal config + boolean USE_OP_PUSH_OR_JUMP_EXACT = ConfigSupport.getBoolean("joni.use_op_push_or_jump_exact", true); + boolean USE_QTFR_PEEK_NEXT = ConfigSupport.getBoolean("joni.use_qtfr_peek_next", true); + + int INIT_MATCH_STACK_SIZE = ConfigSupport.getInt("joni.init_match_stack_size", 64); + + boolean OPTIMIZE = ConfigSupport.getBoolean("joni.optimize", true); + @Deprecated boolean DONT_OPTIMIZE = !OPTIMIZE; + + // use embedded string templates in Regex object as byte arrays instead of compiling them into int bytecode array + boolean USE_STRING_TEMPLATES = ConfigSupport.getBoolean("joni.use_string_templates", true); + + + int MAX_CAPTURE_HISTORY_GROUP = ConfigSupport.getInt("joni.max_capture_history_group", 31); + + + int CHECK_STRING_THRESHOLD_LEN = ConfigSupport.getInt("joni.check_string_threshold_len", 7); + int CHECK_BUFF_MAX_SIZE = ConfigSupport.getInt("joni.check_buff_max_size", 0x4000); + + PrintStream log = System.out; + PrintStream err = System.err; + + boolean DEBUG_ALL = ConfigSupport.getBoolean("joni.debug.all", false); + + boolean DEBUG = ConfigSupport.getBoolean("joni.debug", false) || DEBUG_ALL; + boolean DEBUG_PARSE_TREE = ConfigSupport.getBoolean("joni.debug.parse.tree", false) || DEBUG_ALL; + boolean DEBUG_PARSE_TREE_RAW = ConfigSupport.getBoolean("joni.debug.parse.tree.raw", true) || DEBUG_ALL; + boolean DEBUG_COMPILE = ConfigSupport.getBoolean("joni.debug.compile", false) || DEBUG_ALL; + boolean DEBUG_COMPILE_BYTE_CODE_INFO = ConfigSupport.getBoolean("joni.debug.compile.bytecode.info", false) || DEBUG_ALL; + boolean DEBUG_SEARCH = ConfigSupport.getBoolean("joni.debug.search", false) || DEBUG_ALL; + boolean DEBUG_MATCH = ConfigSupport.getBoolean("joni.debug.match", false) || DEBUG_ALL; +} diff --git a/third_party/joni/src/org/joni/ConfigSupport.java b/third_party/joni/src/org/joni/ConfigSupport.java new file mode 100644 index 000000000..99e263352 --- /dev/null +++ b/third_party/joni/src/org/joni/ConfigSupport.java @@ -0,0 +1,13 @@ +package org.joni; + +public class ConfigSupport { + static boolean getBoolean(String property, boolean def) { + String value = System.getProperty(property, def ? "true" : "false"); + return !value.equals("false"); + } + + static int getInt(String property, int def) { + String value = System.getProperty(property); + return value != null ? Integer.parseInt(value) : def; + } +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java new file mode 100644 index 000000000..d9ed2b1cb --- /dev/null +++ b/third_party/joni/src/org/joni/Lexer.java @@ -0,0 +1,1336 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.Option.isAsciiRange; +import static org.joni.Option.isSingleline; +import static org.joni.Option.isWordBoundAllRange; +import static org.joni.ast.QuantifierNode.isRepeatInfinite; + +import org.jcodings.Encoding; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.jcodings.exception.CharacterPropertyException; +import org.jcodings.exception.EncodingError; +import org.joni.ast.QuantifierNode; +import org.joni.constants.MetaChar; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.TokenType; +import org.joni.exception.ErrorMessages; + +class Lexer extends ScannerSupport { + protected final Regex regex; + protected final ScanEnvironment env; + protected final Syntax syntax; // fast access to syntax + protected final Token token = new Token(); // current token + + protected Lexer(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex.enc, bytes, p, end); + this.regex = regex; + this.env = new ScanEnvironment(regex, syntax, warnings); + this.syntax = env.syntax; + } + + /** + * @return 0: normal {n,m}, 2: fixed {n} + * !introduce returnCode here + */ + private int fetchRangeQuantifier() { + mark(); + boolean synAllow = syntax.allowInvalidInterval(); + + if (!left()) { + if (synAllow) { + return 1; /* "....{" : OK! */ + } else { + newSyntaxException(END_PATTERN_AT_LEFT_BRACE); + } + } + + if (!synAllow) { + c = peek(); + if (c == ')' || c == '(' || c == '|') { + newSyntaxException(END_PATTERN_AT_LEFT_BRACE); + } + } + + int low = scanUnsignedNumber(); + if (low < 0) newSyntaxException(ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + if (low > Config.MAX_REPEAT_NUM) newSyntaxException(ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + + boolean nonLow = false; + if (p == _p) { /* can't read low */ + if (syntax.allowIntervalLowAbbrev()) { + low = 0; + nonLow = true; + } else { + return invalidRangeQuantifier(synAllow); + } + } + + if (!left()) return invalidRangeQuantifier(synAllow); + + fetch(); + int up; + int ret = 0; + if (c == ',') { + int prev = p; // ??? last + up = scanUnsignedNumber(); + if (up < 0) newValueException(TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + if (up > Config.MAX_REPEAT_NUM) newValueException(TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + + if (p == prev) { + if (nonLow) return invalidRangeQuantifier(synAllow); + up = QuantifierNode.REPEAT_INFINITE; /* {n,} : {n,infinite} */ + } + } else { + if (nonLow) return invalidRangeQuantifier(synAllow); + unfetch(); + up = low; /* {n} : exact n times */ + ret = 2; /* fixed */ + } + + if (!left()) return invalidRangeQuantifier(synAllow); + fetch(); + + if (syntax.opEscBraceInterval()) { + if (c != syntax.metaCharTable.esc) return invalidRangeQuantifier(synAllow); + if (!left()) return invalidRangeQuantifier(synAllow); + fetch(); + } + + if (c != '}') return invalidRangeQuantifier(synAllow); + + if (!isRepeatInfinite(up) && low > up) { + newValueException(UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE); + } + + token.type = TokenType.INTERVAL; + token.setRepeatLower(low); + token.setRepeatUpper(up); + + return ret; /* 0: normal {n,m}, 2: fixed {n} */ + } + + private int invalidRangeQuantifier(boolean synAllow) { + if (synAllow) { + restore(); + return 1; + } else { + newSyntaxException(INVALID_REPEAT_RANGE_PATTERN); + return 0; // not reached + } + } + + /* \M-, \C-, \c, or \... */ + private void fetchEscapedValue() { + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + fetch(); + + switch(c) { + + case 'M': + if (syntax.op2EscCapitalMBarMeta()) { + if (!left()) newSyntaxException(END_PATTERN_AT_META); + fetch(); + if (c != '-') newSyntaxException(META_CODE_SYNTAX); + if (!left()) newSyntaxException(END_PATTERN_AT_META); + fetch(); + if (c == syntax.metaCharTable.esc) fetchEscapedValue(); + c = ((c & 0xff) | 0x80); + } else { + fetchEscapedValueBackSlash(); + } + break; + + case 'C': + if (syntax.op2EscCapitalCBarControl()) { + if (!left()) newSyntaxException(END_PATTERN_AT_CONTROL); + fetch(); + if (c != '-') newSyntaxException(CONTROL_CODE_SYNTAX); + fetchEscapedValueControl(); + } else { + fetchEscapedValueBackSlash(); + } + break; + + case 'c': + if (syntax.opEscCControl()) { + fetchEscapedValueControl(); + } + /* fall through */ + + default: + fetchEscapedValueBackSlash(); + } // switch + } + + private void fetchEscapedValueBackSlash() { + c = env.convertBackslashValue(c); + } + + private void fetchEscapedValueControl() { + if (!left()) { + if (syntax.op3OptionECMAScript()) { + return; + } else { + newSyntaxException(END_PATTERN_AT_CONTROL); + } + } + fetch(); + if (c == '?') { + c = 0177; + } else { + if (c == syntax.metaCharTable.esc) fetchEscapedValue(); + c &= 0x9f; + } + } + + private int nameEndCodePoint(int start) { + switch(start) { + case '<': + return '>'; + case '\'': + return '\''; + case '(': + return ')'; + case '{': + return '}'; + default: + return 0; + } + } + + // USE_NAMED_GROUP && USE_BACKREF_AT_LEVEL + /* + \k, \k + \k, \k + \k<-num+n>, \k<-num-n> + */ + + // value implicit (rnameEnd) + private boolean fetchNameWithLevel(int startCode, Ptr rbackNum, Ptr rlevel) { + int src = p; + boolean existLevel = false; + int isNum = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + if (enc.isDigit(c)) { + isNum = 1; + } else if (c == '-') { + isNum = 2; + sign = -1; + pnumHead = p; + } + } + + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')' || c == '+' || c == '-') { + if (isNum == 2) err = INVALID_GROUP_NAME; + break; + } + + if (isNum != 0) { + if (enc.isDigit(c)) { + isNum = 1; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } + } + + boolean isEndCode = false; + if (err == null && c != endCode) { + if (c == '+' || c == '-') { + int flag = c == '-' ? -1 : 1; + + if (!left()) newValueException(INVALID_CHAR_IN_GROUP_NAME); + fetch(); + if (!enc.isDigit(c)) newValueException(INVALID_GROUP_NAME, src, stop); + unfetch(); + int level = scanUnsignedNumber(); + if (level < 0) newValueException(TOO_BIG_NUMBER); + rlevel.p = level * flag; + existLevel = true; + + if (left()) { + fetch(); + isEndCode = c == endCode; + } + } + + if (!isEndCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + } + } + + if (err == null) { + if (isNum != 0) { + mark(); + p = pnumHead; + int backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0) { + newValueException(INVALID_GROUP_NAME, src, stop); + } + rbackNum.p = backNum * sign; + } + value = nameEnd; + return existLevel; + } else { + newValueException(INVALID_GROUP_NAME, src, nameEnd); + return false; // not reached + } + } + + // USE_NAMED_GROUP + // ref: 0 -> define name (don't allow number name) + // 1 -> reference name (allow number name) + private int fetchNameForNamedGroup(int startCode, boolean ref) { + int src = p; + value = 0; + + int isNum = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + if (enc.isDigit(c)) { + if (ref) { + isNum = 1; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } else if (c == '-') { + if (ref) { + isNum = 2; + sign = -1; + pnumHead = p; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } + } + + if (err == null) { + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')') { + if (isNum == 2) { + err = INVALID_GROUP_NAME; + return fetchNameTeardown(src, endCode, nameEnd, err); + } + break; + } + + if (isNum != 0) { + if (enc.isDigit(c)) { + isNum = 1; + } else { + if (!enc.isWord(c)) { + err = INVALID_CHAR_IN_GROUP_NAME; + } else { + err = INVALID_GROUP_NAME; + } + return fetchNameTeardown(src, endCode, nameEnd, err); + } + } + } + + if (c != endCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + return fetchNameErr(src, nameEnd, err); + } + + int backNum = 0; + if (isNum != 0) { + mark(); + p = pnumHead; + backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0) { + newValueException(INVALID_GROUP_NAME, src, nameEnd); + } + backNum *= sign; + } + value = nameEnd; + return backNum; + } else { + return fetchNameTeardown(src, endCode, nameEnd, err); + } + } + + private int fetchNameErr(int src, int nameEnd, String err) { + newValueException(err, src, nameEnd); + return 0; // not reached + } + + private int fetchNameTeardown(int src, int endCode, int nameEnd, String err) { + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')') break; + } + if (!left()) nameEnd = stop; + return fetchNameErr(src, nameEnd, err); + } + + // #else USE_NAMED_GROUP + // make it return nameEnd! + private final int fetchNameForNoNamedGroup(int startCode, boolean ref) { + int src = p; + value = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + + if (enc.isDigit(c)) { + } else if (c == '-') { + sign = -1; + pnumHead = p; + } else { + err = INVALID_CHAR_IN_GROUP_NAME; + } + } + + while(left()) { + nameEnd = p; + + fetch(); + if (c == endCode || c == ')') break; + if (!enc.isDigit(c)) err = INVALID_CHAR_IN_GROUP_NAME; + } + + if (err == null && c != endCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + } + + if (err == null) { + mark(); + p = pnumHead; + int backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0){ + newValueException(INVALID_GROUP_NAME, src, nameEnd); + } + backNum *= sign; + + value = nameEnd; + return backNum; + } else { + newValueException(err, src, nameEnd); + return 0; // not reached + } + } + + protected final int fetchName(int startCode, boolean ref) { + if (Config.USE_NAMED_GROUP) { + return fetchNameForNamedGroup(startCode, ref); + } else { + return fetchNameForNoNamedGroup(startCode, ref); + } + } + + private boolean strExistCheckWithEsc(int[]s, int n, int bad) { + int p = this.p; + int to = this.stop; + + boolean inEsc = false; + int i; + while(p < to) { + if (inEsc) { + inEsc = false; + p += enc.length(bytes, p, to); + } else { + int x = enc.mbcToCode(bytes, p, to); + int q = p + enc.length(bytes, p, to); + if (x == s[0]) { + for (i=1; i= n) return true; + p += enc.length(bytes, p, to); + } else { + x = enc.mbcToCode(bytes, p, to); + if (x == bad) return false; + else if (x == syntax.metaCharTable.esc) inEsc = true; + p = q; + } + } + } + return false; + } + + private static final int[] send = new int[]{':', ']'}; + + private void fetchTokenInCCFor_charType(boolean flag, int type) { + token.type = TokenType.CHAR_TYPE; + token.setPropCType(type); + token.setPropNot(flag); + } + + private void fetchTokenInCCFor_p() { + int c2 = peek(); // !!! migrate to peekIs + if (c2 == '{' && syntax.op2EscPBraceCharProperty()) { + inc(); + token.type = TokenType.CHAR_PROPERTY; + token.setPropNot(c == 'P'); + + if (left() && syntax.op2EscPBraceCircumflexNot()) { + c2 = fetchTo(); + if (c2 == '^') { + token.setPropNot(!token.getPropNot()); + } else { + unfetch(); + } + } + } else { + syntaxWarn("invalid Unicode Property \\<%n>", (char)c); + } + } + + private void fetchTokenInCCFor_x() { + if (!left()) return; + int last = p; + + if (peekIs('{') && syntax.opEscXBraceHex8()) { + inc(); + int num = scanUnsignedHexadecimalNumber(0, 8); + if (num < 0) newValueException(ERR_TOO_BIG_WIDE_CHAR_VALUE); + if (left()) { + int c2 = peek(); + if (enc.isXDigit(c2)) newValueException(ERR_TOO_LONG_WIDE_CHAR_VALUE); + } + + if (p > last + enc.length(bytes, last, stop) && left() && peekIs('}')) { + inc(); + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } else { + /* can't read nothing or invalid format */ + p = last; + } + } else if (syntax.opEscXHex2()) { + int num = scanUnsignedHexadecimalNumber(0, 2); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 16; + token.setC(num); + } + } + + private void fetchTokenInCCFor_u() { + if (!left()) return; + int last = p; + + if (syntax.op2EscUHex4()) { + int num = scanUnsignedHexadecimalNumber(4, 4); + if (num < -1) newValueException(TOO_SHORT_DIGITS); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } + } + + private void fetchTokenInCCFor_digit() { + if (syntax.opEscOctal3()) { + unfetch(); + int last = p; + int num = scanUnsignedOctalNumber(3); + if (num < 0 || num > 0xff) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 8; + token.setC(num); + } + } + + private void fetchTokenInCCFor_posixBracket() { + if (syntax.opPosixBracket() && peekIs(':')) { + token.backP = p; /* point at '[' is readed */ + inc(); + if (strExistCheckWithEsc(send, send.length, ']')) { + token.type = TokenType.POSIX_BRACKET_OPEN; + } else { + unfetch(); + // remove duplication, goto cc_in_cc; + if (syntax.op2CClassSetOp()) { + token.type = TokenType.CC_CC_OPEN; + } else { + env.ccEscWarn("["); + } + } + } else { // cc_in_cc: + if (syntax.op2CClassSetOp()) { + token.type = TokenType.CC_CC_OPEN; + } else { + env.ccEscWarn("["); + } + } + } + + private void fetchTokenInCCFor_and() { + if (syntax.op2CClassSetOp() && left() && peekIs('&')) { + inc(); + token.type = TokenType.CC_AND; + } + } + + protected final TokenType fetchTokenInCC() { + if (!left()) { + token.type = TokenType.EOT; + return token.type; + } + + fetch(); + token.type = TokenType.CHAR; + token.base = 0; + token.setC(c); + token.escaped = false; + + if (c == ']') { + token.type = TokenType.CC_CLOSE; + } else if (c == '-') { + token.type = TokenType.CC_RANGE; + } else if (c == syntax.metaCharTable.esc) { + if (!syntax.backSlashEscapeInCC()) return token.type; + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + fetch(); + token.escaped = true; + token.setC(c); + + switch (c) { + case 'w': + fetchTokenInCCFor_charType(false, CharacterType.WORD); + break; + case 'W': + fetchTokenInCCFor_charType(true, CharacterType.WORD); + break; + case 'd': + fetchTokenInCCFor_charType(false, CharacterType.DIGIT); + break; + case 'D': + fetchTokenInCCFor_charType(true, CharacterType.DIGIT); + break; + case 's': + fetchTokenInCCFor_charType(false, CharacterType.SPACE); + break; + case 'S': + fetchTokenInCCFor_charType(true, CharacterType.SPACE); + break; + case 'h': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + break; + case 'H': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + break; + case 'p': + case 'P': + if (!left()) break; + fetchTokenInCCFor_p(); + break; + case 'x': + fetchTokenInCCFor_x(); + break; + case 'u': + fetchTokenInCCFor_u(); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + fetchTokenInCCFor_digit(); + break; + + default: + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.setCode(c); + token.type = TokenType.CODE_POINT; + } + break; + } // switch + + } else if (c == '[') { + fetchTokenInCCFor_posixBracket(); + } else if (c == '&') { + fetchTokenInCCFor_and(); + } + return token.type; + } + + protected final int backrefRelToAbs(int relNo) { + return env.numMem + 1 + relNo; + } + + private void fetchTokenFor_repeat(int lower, int upper) { + token.type = TokenType.OP_REPEAT; + token.setRepeatLower(lower); + token.setRepeatUpper(upper); + greedyCheck(); + } + + private void fetchTokenFor_openBrace() { + switch (fetchRangeQuantifier()) { + case 0: + greedyCheck(); + break; + case 2: + if (syntax.fixedIntervalIsGreedyOnly()) { + possessiveCheck(); + } else { + greedyCheck(); + } + break; + default: /* 1 : normal char */ + } // inner switch + } + + private void fetchTokenFor_anchor(int subType) { + token.type = TokenType.ANCHOR; + token.setAnchorSubtype(subType); + } + + private void fetchTokenFor_xBrace() { + if (!left()) return; + + int last = p; + if (peekIs('{') && syntax.opEscXBraceHex8()) { + inc(); + int num = scanUnsignedHexadecimalNumber(0, 8); + if (num < 0) newValueException(ERR_TOO_BIG_WIDE_CHAR_VALUE); + if (left()) { + if (enc.isXDigit(peek())) newValueException(ERR_TOO_LONG_WIDE_CHAR_VALUE); + } + + if (p > last + enc.length(bytes, last, stop) && left() && peekIs('}')) { + inc(); + token.type = TokenType.CODE_POINT; + token.setCode(num); + } else { + /* can't read nothing or invalid format */ + p = last; + } + } else if (syntax.opEscXHex2()) { + int num = scanUnsignedHexadecimalNumber(0, 2); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 16; + token.setC(num); + } + } + + private void fetchTokenFor_uHex() { + if (!left()) return; + int last = p; + + if (syntax.op2EscUHex4()) { + int num = scanUnsignedHexadecimalNumber(4, 4); + if (num < -1) newValueException(TOO_SHORT_DIGITS); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } + } + + private void fetchTokenFor_digit() { + unfetch(); + int last = p; + int num = scanUnsignedNumber(); + if (num < 0 || num > Config.MAX_BACKREF_NUM) { // goto skip_backref + } else if (syntax.opDecimalBackref() && (num <= env.numMem || num <= 9)) { /* This spec. from GNU regex */ + if (syntax.strictCheckBackref()) { + if (num > env.numMem || env.memNodes == null || env.memNodes[num] == null) newValueException(INVALID_BACKREF); + } + token.type = TokenType.BACKREF; + token.setBackrefNum(1); + token.setBackrefRef1(num); + token.setBackrefByName(false); + if (Config.USE_BACKREF_WITH_LEVEL) token.setBackrefExistLevel(false); + return; + } + + if (c == '8' || c == '9') { /* normal char */ // skip_backref: + p = last; + inc(); + return; + } + p = last; + + fetchTokenFor_zero(); /* fall through */ + } + + private void fetchTokenFor_zero() { + if (syntax.opEscOctal3()) { + int last = p; + int num = scanUnsignedOctalNumber(c == '0' ? 2 : 3); + if (num < 0 || num > 0xff) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 8; + token.setC(num); + } else if (c != '0') { + inc(); + } + } + + private void fetchTokenFor_NamedBackref() { + if (Config.USE_NAMED_GROUP) { + if (syntax.op2EscKNamedBackref() && left()) { + fetch(); + if (c =='<' || c == '\'') { + fetchNamedBackrefToken(); + } else { + unfetch(); + syntaxWarn("invalid back reference"); + } + } + } + } + + private void fetchTokenFor_subexpCall() { + if (Config.USE_NAMED_GROUP) { + if (syntax.op2EscGBraceBackref() && left()) { + fetch(); + if (c == '{') { + fetchNamedBackrefToken(); + } else { + unfetch(); + } + } + } + if (Config.USE_SUBEXP_CALL) { + if (syntax.op2EscGSubexpCall() && left()) { + fetch(); + if (c == '<' || c == '\'') { + int gNum = -1; + boolean rel = false; + int cnext = peek(); + int nameEnd = 0; + if (cnext == '0') { + inc(); + if (peekIs(nameEndCodePoint(c))) { /* \g<0>, \g'0' */ + inc(); + nameEnd = p; + gNum = 0; + } + } else if (cnext == '+') { + inc(); + rel = true; + } + int prev = p; + if (gNum < 0) { + gNum = fetchName(c, true); + nameEnd = value; + } + token.type = TokenType.CALL; + token.setCallNameP(prev); + token.setCallNameEnd(nameEnd); + token.setCallGNum(gNum); + token.setCallRel(rel); + } else { + syntaxWarn("invalid subexp call"); + unfetch(); + } + } + } + } + + protected void fetchNamedBackrefToken() { + int last = p; + int backNum; + if (Config.USE_BACKREF_WITH_LEVEL) { + Ptr rbackNum = new Ptr(); + Ptr rlevel = new Ptr(); + token.setBackrefExistLevel(fetchNameWithLevel(c, rbackNum, rlevel)); + token.setBackrefLevel(rlevel.p); + backNum = rbackNum.p; + } else { + backNum = fetchName(c, true); + } // USE_BACKREF_AT_LEVEL + int nameEnd = value; // set by fetchNameWithLevel/fetchName + + if (backNum != 0) { + if (backNum < 0) { + backNum = backrefRelToAbs(backNum); + if (backNum <= 0) newValueException(INVALID_BACKREF); + } + + if (syntax.strictCheckBackref() && (backNum > env.numMem || env.memNodes == null)) { + newValueException(INVALID_BACKREF); + } + token.type = TokenType.BACKREF; + token.setBackrefByName(false); + token.setBackrefNum(1); + token.setBackrefRef1(backNum); + } else { + NameEntry e = regex.nameToGroupNumbers(bytes, last, nameEnd); + if (e == null) newValueException(UNDEFINED_NAME_REFERENCE, last, nameEnd); + + if (syntax.strictCheckBackref()) { + if (e.backNum == 1) { + if (e.backRef1 > env.numMem || + env.memNodes == null || + env.memNodes[e.backRef1] == null) newValueException(INVALID_BACKREF); + } else { + for (int i=0; i env.numMem || + env.memNodes == null || + env.memNodes[e.backRefs[i]] == null) newValueException(INVALID_BACKREF); + } + } + } + + token.type = TokenType.BACKREF; + token.setBackrefByName(true); + + if (e.backNum == 1) { + token.setBackrefNum(1); + token.setBackrefRef1(e.backRef1); + } else { + token.setBackrefNum(e.backNum); + token.setBackrefRefs(e.backRefs); + } + } + } + + private void fetchTokenFor_charProperty() { + if (peekIs('{') && syntax.op2EscPBraceCharProperty()) { + inc(); + token.type = TokenType.CHAR_PROPERTY; + token.setPropNot(c == 'P'); + + if (left() && syntax.op2EscPBraceCircumflexNot()) { + fetch(); + if (c == '^') { + token.setPropNot(!token.getPropNot()); + } else { + unfetch(); + } + } + } else { + syntaxWarn("invalid Unicode Property \\<%n>", (char)c); + } + } + + private void fetchTokenFor_metaChars() { + if (c == syntax.metaCharTable.anyChar) { + token.type = TokenType.ANYCHAR; + } else if (c == syntax.metaCharTable.anyTime) { + fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + } else if (c == syntax.metaCharTable.zeroOrOneTime) { + fetchTokenFor_repeat(0, 1); + } else if (c == syntax.metaCharTable.oneOrMoreTime) { + fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + } else if (c == syntax.metaCharTable.anyCharAnyTime) { + token.type = TokenType.ANYCHAR_ANYTIME; + // goto out + } + } + + protected final void fetchToken() { + int src = p; + // mark(); // out + start: + while(true) { + if (!left()) { + token.type = TokenType.EOT; + return; + } + + token.type = TokenType.STRING; + token.base = 0; + token.backP = p; + + fetch(); + + if (c == syntax.metaCharTable.esc && !syntax.op2IneffectiveEscape()) { // IS_MC_ESC_CODE(code, syn) + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + + token.backP = p; + fetch(); + + token.setC(c); + token.escaped = true; + switch(c) { + + case '*': + if (syntax.opEscAsteriskZeroInf()) fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + break; + case '+': + if (syntax.opEscPlusOneInf()) fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + break; + case '?': + if (syntax.opEscQMarkZeroOne()) fetchTokenFor_repeat(0, 1); + break; + case '{': + if (syntax.opEscBraceInterval()) fetchTokenFor_openBrace(); + break; + case '|': + if (syntax.opEscVBarAlt()) token.type = TokenType.ALT; + break; + case '(': + if (syntax.opEscLParenSubexp()) token.type = TokenType.SUBEXP_OPEN; + break; + case ')': + if (syntax.opEscLParenSubexp()) token.type = TokenType.SUBEXP_CLOSE; + break; + case 'w': + if (syntax.opEscWWord()) fetchTokenInCCFor_charType(false, CharacterType.WORD); + break; + case 'W': + if (syntax.opEscWWord()) fetchTokenInCCFor_charType(true, CharacterType.WORD); + break; + case 'b': + if (syntax.opEscBWordBound()) { + fetchTokenFor_anchor(AnchorType.WORD_BOUND); + token.setAnchorASCIIRange(isAsciiRange(env.option) && !isWordBoundAllRange(env.option)); + } + break; + case 'B': + if (syntax.opEscBWordBound()) { + fetchTokenFor_anchor(AnchorType.NOT_WORD_BOUND); + token.setAnchorASCIIRange(isAsciiRange(env.option) && !isWordBoundAllRange(env.option)); + } + break; + case '<': + if (Config.USE_WORD_BEGIN_END && syntax.opEscLtGtWordBeginEnd()) { + fetchTokenFor_anchor(AnchorType.WORD_BEGIN); + token.setAnchorASCIIRange(isAsciiRange(env.option)); + } + break; + case '>': + if (Config.USE_WORD_BEGIN_END && syntax.opEscLtGtWordBeginEnd()) { + fetchTokenFor_anchor(AnchorType.WORD_END); + token.setAnchorASCIIRange(isAsciiRange(env.option)); + } + break; + case 's': + if (syntax.opEscSWhiteSpace()) fetchTokenInCCFor_charType(false, CharacterType.SPACE); + break; + case 'S': + if (syntax.opEscSWhiteSpace()) fetchTokenInCCFor_charType(true, CharacterType.SPACE); + break; + case 'd': + if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(false, CharacterType.DIGIT); + break; + case 'D': + if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(true, CharacterType.DIGIT); + break; + case 'h': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + break; + case 'H': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + break; + case 'A': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); + break; + case 'Z': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.SEMI_END_BUF); + break; + case 'z': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.END_BUF); + break; + case 'G': + if (syntax.opEscCapitalGBeginAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_POSITION); + break; + case '`': + if (syntax.op2EscGnuBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); + break; + case '\'': + if (syntax.op2EscGnuBufAnchor()) fetchTokenFor_anchor(AnchorType.END_BUF); + break; + case 'x': + fetchTokenFor_xBrace(); + break; + case 'u': + fetchTokenFor_uHex(); + break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + fetchTokenFor_digit(); + break; + case '0': + fetchTokenFor_zero(); + break; + case 'k': + fetchTokenFor_NamedBackref(); + break; + case 'g': + fetchTokenFor_subexpCall(); + break; + case 'Q': + if (syntax.op2EscCapitalQQuote()) token.type = TokenType.QUOTE_OPEN; + break; + case 'p': + case 'P': + fetchTokenFor_charProperty(); + break; + case 'R': + if (syntax.op2EscCapitalRLinebreak()) token.type = TokenType.LINEBREAK; + break; + case 'X': + if (syntax.op2EscCapitalXExtendedGraphemeCluster()) token.type = TokenType.EXTENDED_GRAPHEME_CLUSTER; + break; + case 'K': + if (syntax.op2EscCapitalKKeep()) token.type = TokenType.KEEP; + break; + default: + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { /* set_raw: */ + token.type = TokenType.CODE_POINT; + token.setCode(c); + } else { /* string */ + int encLength = enc.length(bytes, token.backP, stop); + /* Check and ensure the encoded character is valid */ + if (encLength == Encoding.CHAR_INVALID) { + throw new IllegalArgumentException("Invalid character found."); + } + p = token.backP + encLength; + } + break; + } // switch (c) + } else { + token.setC(c); + token.escaped = false; + + if (Config.USE_VARIABLE_META_CHARS && (c != MetaChar.INEFFECTIVE_META_CHAR && syntax.opVariableMetaCharacters())) { + fetchTokenFor_metaChars(); + break; + } + + { + switch(c) { + case '.': + if (syntax.opDotAnyChar()) token.type = TokenType.ANYCHAR; + break; + case '*': + if (syntax.opAsteriskZeroInf()) fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + break; + case '+': + if (syntax.opPlusOneInf()) fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + break; + case '?': + if (syntax.opQMarkZeroOne()) fetchTokenFor_repeat(0, 1); + break; + case '{': + if (syntax.opBraceInterval()) fetchTokenFor_openBrace(); + break; + case '|': + if (syntax.opVBarAlt()) token.type = TokenType.ALT; + break; + + case '(': + if (peekIs('?') && syntax.op2QMarkGroupEffect()) { + inc(); + if (peekIs('#')) { + fetch(); + while (true) { + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c == syntax.metaCharTable.esc) { + if (left()) fetch(); + } else { + if (c == ')') break; + } + } + continue start; // goto start + } + unfetch(); + } + + if (syntax.opLParenSubexp()) token.type = TokenType.SUBEXP_OPEN; + break; + case ')': + if (syntax.opLParenSubexp()) token.type = TokenType.SUBEXP_CLOSE; + break; + case '^': + if (syntax.opLineAnchor()) fetchTokenFor_anchor(isSingleline(env.option) ? AnchorType.BEGIN_BUF : AnchorType.BEGIN_LINE); + break; + case '$': + if (syntax.opLineAnchor()) fetchTokenFor_anchor(isSingleline(env.option) ? AnchorType.SEMI_END_BUF : AnchorType.END_LINE); + break; + case '[': + if (syntax.opBracketCC()) token.type = TokenType.CC_OPEN; + break; + case ']': + if (src > getBegin()) { /* /].../ is allowed. */ + env.closeBracketWithoutEscapeWarn("]"); + } + break; + case '#': + if (Option.isExtend(env.option)) { + while (left()) { + fetch(); + if (enc.isNewLine(c)) break; + } + continue start; // goto start + } + break; + + case ' ': + case '\t': + case '\n': + case '\r': + case '\f': + if (Option.isExtend(env.option)) continue start; // goto start + break; + + default: // string + break; + + } // switch + } + } + + break; + } // while + } + + private void greedyCheck() { + if (left() && peekIs('?') && syntax.opQMarkNonGreedy()) { + + fetch(); + + token.setRepeatGreedy(false); + token.setRepeatPossessive(false); + } else { + possessiveCheck(); + } + } + + private void possessiveCheck() { + if (left() && peekIs('+') && + (syntax.op2PlusPossessiveRepeat() && token.type != TokenType.INTERVAL || + syntax.op2PlusPossessiveInterval() && token.type == TokenType.INTERVAL)) { + + fetch(); + + token.setRepeatGreedy(true); + token.setRepeatPossessive(true); + } else { + token.setRepeatGreedy(true); + token.setRepeatPossessive(false); + } + } + + protected final int fetchCharPropertyToCType() { + mark(); + + while (left()) { + int last = p; + fetch(); + if (c == '}') { + return enc.propertyNameToCType(bytes, _p, last); + } else if (c == '(' || c == ')' || c == '{' || c == '|') { + throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); + } + } + newValueException(PROPERTY_NAME_NEVER_TERMINATED, _p, stop); + return 0; // not reached + } + + protected final void syntaxWarn(String message, char c) { + syntaxWarn(message.replace("<%n>", Character.toString(c))); + } + + protected final void syntaxWarn(String message) { + if (env.warnings != WarnCallback.NONE) { + env.warnings.warn(message + ": /" + new String(bytes, getBegin(), getEnd() - getBegin()) + "/"); + } + } +} diff --git a/third_party/joni/src/org/joni/MatchView.java b/third_party/joni/src/org/joni/MatchView.java new file mode 100644 index 000000000..46227575a --- /dev/null +++ b/third_party/joni/src/org/joni/MatchView.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Read-only provisional matcher state, valid only during a callout. */ +public interface MatchView { + int currentBytePosition(); + + int captureCount(); + + int captureBegin(int capture); + + int captureEnd(int capture); +} diff --git a/third_party/joni/src/org/joni/Matcher.java b/third_party/joni/src/org/joni/Matcher.java new file mode 100644 index 000000000..79abcc291 --- /dev/null +++ b/third_party/joni/src/org/joni/Matcher.java @@ -0,0 +1,672 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.joni; + +import static org.joni.Option.isFindLongest; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; +import org.jcodings.constants.CharacterType; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.constants.internal.AnchorType; +import org.joni.exception.TimeoutException; + +public abstract class Matcher extends IntHolder { + static final InterruptedException INTERRUPTED_EXCEPTION = new InterruptedException(); + static final InterruptedException TIMEOUT_EXCEPTION = new TimeoutException(); + public static final int FAILED = -1; + public static final int INTERRUPTED = -2; + + protected final Regex regex; + protected final Encoding enc; + + protected final byte[]bytes; + protected final int str; + protected final int end; + + protected int msaStart; + protected int msaOptions; + protected final Region msaRegion; + protected int msaBestLen; + protected int msaBestS; + protected int msaGpos; + + protected int msaBegin; + protected int msaEnd; + + protected long timeout; // nanoseconds + private CalloutHandler calloutHandler; + + // nanoseconds since entering searchCommon (underlying machines will check during interrupt checks + // which will cheapen how often we look but also it should be granular enough to not matter). + protected long startTime; + + Matcher(Regex regex, Region region, byte[]bytes, int p, int end) { + this(regex, region, bytes, p, end, -1); + } + + // FIXME: For next major version this should be the main constructor and MatcherFactory should + // have the abstract method be for this. + Matcher(Regex regex, Region region, byte[]bytes, int p, int end, long timeout) { + this.regex = regex; + this.enc = regex.enc; + this.bytes = bytes; + this.str = p; + this.end = end; + this.msaRegion = region; + this.timeout = timeout; + } + + // main matching method + protected abstract int matchAt(int range, int sstart, int sprev, boolean interrupt) throws InterruptedException; + + protected abstract void stateCheckBuffInit(int strLength, int offset, int stateNum); + protected abstract void stateCheckBuffClear(); + + public abstract void interrupt(); + + public final Region getRegion() { + return msaRegion; + } + + public final Region getEagerRegion() { + return msaRegion != null ? msaRegion : Region.newRegion(msaBegin, msaEnd); + } + + public final int getBegin() { + return msaBegin; + } + + public final int getEnd() { + return msaEnd; + } + + protected final void msaInit(int option, int start, int gpos) { + msaOptions = option; + msaStart = start; + msaGpos = gpos; + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) msaBestLen = -1; + } + + public final int match(int at, int range, int option) { + try { + return matchCommon(at, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int matchInterruptible(int at, int range, int option) throws InterruptedException { + return matchCommon(at, range, option, true); + } + + private final int matchCommon(int at, int range, int option, boolean interrupt) throws InterruptedException { + msaInit(option, at, at); + + if (Config.USE_CEC) { + int offset = at = str; + stateCheckBuffInit(end - str, offset, regex.numCombExpCheck); // move it to construction? + } // USE_COMBINATION_EXPLOSION_CHECK + + int prev = enc.prevCharHead(bytes, str, at, end); + + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + return matchAt(end /*range*/, at, prev, interrupt); + } else { + return matchAt(range /*range*/, at, prev, interrupt); + } + } + + int low, high; // these are the return values + private final boolean forwardSearchRange(byte[]bytes, int str, int end, int s, int range, IntHolder lowPrev) { + int pprev = -1; + int p = s; + + if (Config.DEBUG_SEARCH) debugForwardSearchRange(str, end, s, range); + + if (regex.dMin > 0) { + if (enc.isSingleByte()) { + p += regex.dMin; + } else { + int q = p + regex.dMin; + while (p < q && p < end) p += enc.length(bytes, p, end); + } + } + + retry:while (true) { + if (Config.DEBUG_SEARCH) debugSearch(regex.forward.getName(), p, end, range); + p = regex.forward.search(this, bytes, p, end, range); + + if (p != -1 && p < range) { + if (p - regex.dMin < s) { + // retry_gate: + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + + if (regex.subAnchor != 0) { + switch (regex.subAnchor) { + case AnchorType.BEGIN_LINE: + if (p != str) { + int prev = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + if (!enc.isNewLine(bytes, prev, end)) { + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + } + break; + + case AnchorType.END_LINE: + if (p == end) { + if (!Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + int prev = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + if (prev != -1 && enc.isNewLine(bytes, prev, end)) { + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + } + } else if (!enc.isNewLine(bytes, p, end) && (!Config.USE_CRNL_AS_LINE_TERMINATOR || !enc.isMbcCrnl(bytes, p, end))) { + //if () break; + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + break; + } // switch + } + + if (regex.dMax == 0) { + low = p; + if (lowPrev != null) { // ??? // remove null checks + if (low > s) { + lowPrev.value = enc.prevCharHead(bytes, s, p, end); + } else { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + } + } + } else { + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + low = p - regex.dMax; + + if (low > s) { + low = enc.rightAdjustCharHeadWithPrev(bytes, s, low, end, lowPrev); + if (lowPrev != null && lowPrev.value == -1) { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : s, low, end); + } + } else { + if (lowPrev != null) { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, low, end); + } + } + } + } + /* no needs to adjust *high, *high is used as range check only */ + high = p - regex.dMin; + if (Config.DEBUG_SEARCH) debugForwardSearchRangeSuccess(str, low, high); + return true; /* success */ + } + return false; /* fail */ + } //while + } + + // low, high + private final boolean backwardSearchRange(byte[]bytes, int str, int end, int s, int range, int adjrange) { + range += regex.dMin; + int p = s; + + retry:while (true) { + p = regex.backward.search(this, bytes, range, adjrange, end, p, s, range); + + if (p != -1) { + if (regex.subAnchor != 0) { + switch (regex.subAnchor) { + case AnchorType.BEGIN_LINE: + if (p != str) { + int prev = enc.prevCharHead(bytes, str, p, end); + if (!enc.isNewLine(bytes, prev, end)) { + p = prev; + continue retry; + } + } + break; + + case AnchorType.END_LINE: + if (p == end) { + if (!Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + int prev = enc.prevCharHead(bytes, adjrange, p, end); + if (prev == -1) return false; + if (enc.isNewLine(bytes, prev, end)) { + p = prev; + continue retry; + } + } + } else if (!enc.isNewLine(bytes, p, end) && (!Config.USE_CRNL_AS_LINE_TERMINATOR || !enc.isMbcCrnl(bytes, p, end))) { + p = enc.prevCharHead(bytes, adjrange, p, end); + if (p == -1) return false; + continue retry; + } + break; + } // switch + } + + /* no needs to adjust *high, *high is used as range check only */ + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + low = p - regex.dMax; + high = p - regex.dMin; + high = enc.rightAdjustCharHead(bytes, adjrange, high, end); + } + + if (Config.DEBUG_SEARCH) debugBackwardSearchRange(str, low, high); + return true; + } + + if (Config.DEBUG_SEARCH) Config.log.println("backward_search_range: fail."); + return false; + } // while + } + + // MATCH_AND_RETURN_CHECK + private boolean matchCheck(int upperRange, int s, int prev, boolean interrupt) throws InterruptedException { + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + //range = upperRange; + if (matchAt(upperRange, s, prev, interrupt) != -1) { + if (!isFindLongest(regex.options)) return true; + } + } else { + //range = upperRange; + if (matchAt(upperRange, s, prev, interrupt) != -1) return true; + } + } else { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (matchAt(end, s, prev, interrupt) != -1) { + //range = upperRange; + if (!isFindLongest(regex.options)) return true; + } + } else { + //range = upperRange; + if (matchAt(end, s, prev, interrupt) != -1) return true; + } + } + return false; + } + + public final int search(int start, int range, int option) { + try { + return searchCommon(start, start, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int search(int gpos, int start, int range, int option) { + try { + return searchCommon(gpos, start, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int searchInterruptible(int start, int range, int option) throws InterruptedException { + return searchCommon(start, start, range, option, true); + } + + public final int searchInterruptible(int gpos, int start, int range, int option) throws InterruptedException { + return searchCommon(gpos, start, range, option, true); + } + + private final int searchCommon(int gpos, int start, int range, int option, boolean interrupt) throws InterruptedException { + if (timeout != -1) startTime = System.nanoTime(); + int s, prev; + int origStart = start; + int origRange = range; + + if (Config.DEBUG_SEARCH) debugSearch(str, end, start, range); + + if (start > end || start < str) return FAILED; + + /* anchor optimize: resume search range */ + if (regex.anchor != 0 && str < end) { + int minSemiEnd, maxSemiEnd; + + if ((regex.anchor & AnchorType.BEGIN_POSITION) != 0) { + /* search start-position only */ + // !begin_position:! + if (range > start) { + if (gpos > start) { + if (gpos < range) range = gpos + 1; + } else { + range = start + 1; + } + } else { + range = start; + } + } else if ((regex.anchor & AnchorType.BEGIN_BUF) != 0) { + /* search str-position only */ + if (range > start) { + if (start != str) return FAILED; // mismatch_no_msa; + range = str + 1; + } else { + if (range <= str) { + start = str; + range = str; + } else { + return FAILED; // mismatch_no_msa; + } + } + } else if ((regex.anchor & AnchorType.END_BUF) != 0) { + minSemiEnd = maxSemiEnd = end; + // !end_buf:! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } else if ((regex.anchor & AnchorType.SEMI_END_BUF) != 0) { + int preEnd = enc.stepBack(bytes, str, end, end, 1); + maxSemiEnd = end; + if (enc.isNewLine(bytes, preEnd, end)) { + minSemiEnd = preEnd; + if (Config.USE_CRNL_AS_LINE_TERMINATOR) { + preEnd = enc.stepBack(bytes, str, preEnd, end, 1); + if (preEnd != -1 && enc.isMbcCrnl(bytes, preEnd, end)) { + minSemiEnd = preEnd; + } + } + if (minSemiEnd > str && start <= minSemiEnd) { + // !goto end_buf;! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } + } else { + minSemiEnd = end; + // !goto end_buf;! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } + } else if ((regex.anchor & AnchorType.ANYCHAR_STAR_ML) != 0) { + // goto !begin_position;! + if (range > start) { + if (gpos > start) { + if (gpos < range) range = gpos + 1; + } else { + range = start + 1; + } + } else { + range = start; + } + } + + } else if (str == end) { /* empty string */ + // empty address ? + if (Config.DEBUG_SEARCH) Config.log.println("onig_search: empty string."); + + if (regex.thresholdLength == 0) { + s = start = str; + prev = -1; + msaInit(option, start, start); + + if (Config.USE_CEC) stateCheckBuffClear(); + + if (matchCheck(end, s, prev, interrupt)) return match(s); + return mismatch(); + } + return FAILED; // goto mismatch_no_msa; + } + + if (Config.DEBUG_SEARCH) debugSearch(str, end, start, range); + + msaInit(option, origStart, gpos); + if (Config.USE_CEC) { + int offset = Math.min(start, range) - str; + stateCheckBuffInit(end - str, offset, regex.numCombExpCheck); + } + + s = start; + if (range > start) { /* forward search */ + if (s > str) { + prev = enc.prevCharHead(bytes, str, s, end); + } else { + prev = 0; // -1 + } + + if (regex.forward != null) { + int schRange = range; + if (regex.dMax != 0) { + if (regex.dMax == MinMaxLen.INFINITE_DISTANCE) { + schRange = end; + } else { + schRange += regex.dMax; + if (schRange > end) schRange = end; + } + } + if ((end - start) < regex.thresholdLength) return mismatch(); + + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + do { + if (!forwardSearchRange(bytes, str, end, s, schRange, this)) return mismatch(); // low, high, lowPrev + if (s < low) { + s = low; + prev = value; + } + while (s <= high) { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); // ??? + prev = s; + s += enc.length(bytes, s, end); + } + } while (s < range); + return mismatch(); + + } else { /* check only. */ + if (!forwardSearchRange(bytes, str, end, s, schRange, null)) return mismatch(); + + if ((regex.anchor & AnchorType.ANYCHAR_STAR) != 0) { + do { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + prev = s; + s += enc.length(bytes, s, end); + + if ((regex.anchor & (AnchorType.LOOK_BEHIND | AnchorType.PREC_READ_NOT)) == 0) { + while (!enc.isNewLine(bytes, prev, end) && s < range) { + prev = s; + s += enc.length(bytes, s, end); + } + } + } while (s < range); + return mismatch(); + } + } + } + + do { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + prev = s; + s += enc.length(bytes, s, end); + } while (s < range); + + if (s == range) { /* because empty match with /$/. */ + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + } + } else { /* backward search */ + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + if (origStart < end) { + origStart += enc.length(bytes, origStart, end); // /* is upper range */ + } + } + + if (regex.backward != null) { + int adjrange; + if (range < end) { + adjrange = enc.leftAdjustCharHead(bytes, str, range, end); + } else { + adjrange = end; + } + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE && (end - range) >= regex.thresholdLength) { + do { + int schStart = s + regex.dMax; + if (schStart > end) schStart = end; + if (!backwardSearchRange(bytes, str, end, schStart, range, adjrange)) return mismatch(); // low, high + if (s > high) s = high; + while (s != -1 && s >= low) { + prev = enc.prevCharHead(bytes, str, s, end); + if (matchCheck(origStart, s, prev, interrupt)) return match(s); + s = prev; + } + } while (s >= range); + return mismatch(); + } else { /* check only. */ + if ((end - range) < regex.thresholdLength) return mismatch(); + + int schStart = s; + if (regex.dMax != 0) { + if (regex.dMax == MinMaxLen.INFINITE_DISTANCE) { + schStart = end; + } else { + schStart += regex.dMax; + if (schStart > end) { + schStart = end; + } else { + schStart = enc.leftAdjustCharHead(bytes, start, schStart, end); + } + } + } + if (!backwardSearchRange(bytes, str, end, schStart, range, adjrange)) return mismatch(); + } + } + + do { + prev = enc.prevCharHead(bytes, str, s, end); + if (matchCheck(origStart, s, prev, interrupt)) return match(s); + s = prev; + } while (s >= range); + + } + return mismatch(); + } + + private final boolean endBuf(int start, int range, int minSemiEnd, int maxSemiEnd) { + if ((maxSemiEnd - str) < regex.anchorDmin) return true; // mismatch_no_msa; + + if (range > start) { + if ((minSemiEnd - start) > regex.anchorDmax) { + start = minSemiEnd - regex.anchorDmax; + if (start < end) { + start = enc.rightAdjustCharHead(bytes, str, start, end); + } else { /* match with empty at end */ + start = enc.prevCharHead(bytes, str, end, end); + } + } + if ((maxSemiEnd - (range - 1)) < regex.anchorDmin) { + range = maxSemiEnd - regex.anchorDmin + 1; + } + if (start >= range) return true; // mismatch_no_msa; + } else { + if ((minSemiEnd - range) > regex.anchorDmax) { + range = minSemiEnd - regex.anchorDmax; + } + if ((maxSemiEnd - start) < regex.anchorDmin) { + start = maxSemiEnd - regex.anchorDmin; + start = enc.leftAdjustCharHead(bytes, str, start, end); + } + if (range > start) return true; // mismatch_no_msa; + } + return false; + } + + private final int match(int s) { + return s - str; // sstart ??? + } + + private final int mismatch() { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (msaBestLen >= 0) { + int s = msaBestS; + return match(s); + } + } + // falls through finish: + return FAILED; + } + + private byte[]icbuf; + + protected final byte[]icbuf() { + return icbuf == null ? icbuf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : icbuf; + } + + static boolean isMbcAsciiWord(Encoding enc, byte[]bytes, int p, int end) { // ONIGENC_IS_MBC_ASCII_WORD + return ASCIIEncoding.INSTANCE.isCodeCType(enc.mbcToCode(bytes, p, end), CharacterType.WORD); + } + + private final void debugForwardSearchRange(int str, int end, int s, int range) { + if (Config.DEBUG_SEARCH) { + Config.log.println("forward_search_range: " + + "str: " + str + + ", end: " + end + + ", s: " + s + + ", range: " + range); + } + } + + private final void debugForwardSearchRangeSuccess(int str, int low, int high) { + if (Config.DEBUG_SEARCH) { + Config.log.println("forward_search_range success: " + + "low: " + (low - str) + + ", high: " + (high - str) + + ", dmin: " + regex.dMin + + ", dmax: " + regex.dMax); + } + } + + private final void debugSearch(int str, int end, int start, int range) { + if (Config.DEBUG_SEARCH) { + Config.log.println("onig_search (entry point): " + + "str: " + str + + ", end: " + (end - str) + + ", start: " + (start - str) + + ", range " + (range - str)); + } + } + + private final void debugBackwardSearchRange(int str, int low, int high) { + if (Config.DEBUG_SEARCH) { + Config.log.println("backward_search_range: "+ + "low: " + (low - str) + + ", high: " + (high - str)); + } + } + + static void debugSearch(String name, int textP, int textEnd, int textRange) { + Config.log.println(name + ": text: " + textP + ", text_end: " + textEnd + ", text_range: " + textRange); + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public final void setCalloutHandler(CalloutHandler handler) { + calloutHandler = handler; + } + + final CalloutHandler getCalloutHandler() { + return calloutHandler; + } +} diff --git a/third_party/joni/src/org/joni/MatcherFactory.java b/third_party/joni/src/org/joni/MatcherFactory.java new file mode 100644 index 000000000..48f3ae445 --- /dev/null +++ b/third_party/joni/src/org/joni/MatcherFactory.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +abstract class MatcherFactory { + abstract Matcher create(Regex regex, Region region, byte[]bytes, int p, int end); + + public Matcher create(Regex regex, Region region, byte[]bytes, int p, int end, long timeout) { + Matcher matcher = create(regex, region, bytes, p, end); + matcher.setTimeout(timeout); + return matcher; + } + + static final MatcherFactory DEFAULT = new MatcherFactory() { + @Override + Matcher create(Regex regex, Region region, byte[]bytes, int p, int end) { + return new ByteCodeMachine(regex, region, bytes, p, end); + } + }; +} diff --git a/third_party/joni/src/org/joni/MinMaxLen.java b/third_party/joni/src/org/joni/MinMaxLen.java new file mode 100644 index 000000000..0c6d18f46 --- /dev/null +++ b/third_party/joni/src/org/joni/MinMaxLen.java @@ -0,0 +1,129 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +final class MinMaxLen { + int min; /* min byte length */ + int max; /* max byte length */ + + /* 1000 / (min-max-dist + 1) */ + private static final short[] distValues = { + 1000, 500, 333, 250, 200, 167, 143, 125, 111, 100, + 91, 83, 77, 71, 67, 63, 59, 56, 53, 50, + 48, 45, 43, 42, 40, 38, 37, 36, 34, 33, + 32, 31, 30, 29, 29, 28, 27, 26, 26, 25, + 24, 24, 23, 23, 22, 22, 21, 21, 20, 20, + 20, 19, 19, 19, 18, 18, 18, 17, 17, 17, + 16, 16, 16, 16, 15, 15, 15, 15, 14, 14, + 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, + 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 10, 10, 10, 10, 10 + }; + + int distanceValue() { + if (max == INFINITE_DISTANCE) return 0; + int d = max - min; + /* return dist_vals[d] * 16 / (mm->min + 12); */ + return d < distValues.length ? distValues[d] : 1; + } + + int compareDistanceValue(MinMaxLen other, int v1, int v2) { + if (v2 <= 0) return -1; + if (v1 <= 0) return 1; + + v1 *= distanceValue(); + v2 *= other.distanceValue(); + + if (v2 > v1) return 1; + if (v2 < v1) return -1; + + return Integer.compare(min, other.min); + } + + boolean equal(MinMaxLen other) { + return min == other.min && max == other.max; + } + + void set(int min, int max) { + this.min = min; + this.max = max; + } + + void clear() { + min = max = 0; + } + + void copy(MinMaxLen other) { + min = other.min; + max = other.max; + } + + void add(MinMaxLen other) { + min = distanceAdd(min, other.min); + max = distanceAdd(max, other.max); + } + + void addLength(int len) { + min = distanceAdd(min, len); + max = distanceAdd(max, len); + } + + void altMerge(MinMaxLen other) { + if (min > other.min) min = other.min; + if (max < other.max) max = other.max; + } + + static final int INFINITE_DISTANCE = 0x7FFFFFFF; + static int distanceAdd(int d1, int d2) { + if (d1 == INFINITE_DISTANCE || d2 == INFINITE_DISTANCE) { + return INFINITE_DISTANCE; + } else { + if (d1 <= INFINITE_DISTANCE - d2) return d1 + d2; + else return INFINITE_DISTANCE; + } + } + + static int distanceMultiply(int d, int m) { + if (m == 0) return 0; + if (d < INFINITE_DISTANCE / m) { + return d * m; + } else { + return INFINITE_DISTANCE; + } + } + + static String distanceRangeToString(int a, int b) { + String s = ""; + if (a == INFINITE_DISTANCE) { + s += "inf"; + } else { + s += "(" + a + ")"; + } + + s += "-"; + + if (b == INFINITE_DISTANCE) { + s += "inf"; + } else { + s += "(" + b + ")"; + } + return s; + } +} diff --git a/third_party/joni/src/org/joni/MultiRegion.java b/third_party/joni/src/org/joni/MultiRegion.java new file mode 100644 index 000000000..baf0eb407 --- /dev/null +++ b/third_party/joni/src/org/joni/MultiRegion.java @@ -0,0 +1,72 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import java.util.Arrays; + +public final class MultiRegion extends Region { + private final int[] begEnd; + + public MultiRegion(int num) { + this.begEnd = new int[num * 2]; + } + + public MultiRegion(int begin, int end) { + this.begEnd = new int[]{begin, end}; + } + + @Override + public final int getNumRegs() { + return begEnd.length / 2; + } + + @Override + public MultiRegion clone() { + MultiRegion region = new MultiRegion(getNumRegs()); + System.arraycopy(begEnd, 0, region.begEnd, 0, begEnd.length); + if (getCaptureTree() != null) region.setCaptureTree(getCaptureTree().cloneTree()); + return region; + } + + @Override + public int getBeg(int index) { + return begEnd[index * 2]; + } + + @Override + public int setBeg(int index, int value) { + return begEnd[index * 2] = value; + } + + @Override + public int getEnd(int index) { + return begEnd[index * 2 + 1]; + } + + @Override + public int setEnd(int index, int value) { + return begEnd[index * 2 + 1] = value; + } + + @Override + void clear() { + Arrays.fill(begEnd, REGION_NOTPOS); + } +} diff --git a/third_party/joni/src/org/joni/NameEntry.java b/third_party/joni/src/org/joni/NameEntry.java new file mode 100644 index 000000000..2b6251305 --- /dev/null +++ b/third_party/joni/src/org/joni/NameEntry.java @@ -0,0 +1,98 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class NameEntry { + static final int INIT_NAME_BACKREFS_ALLOC_NUM = 8; + + public final byte[]name; + public final int nameP; + public final int nameEnd; + + int backNum; + int backRef1; + int[] backRefs; + + public NameEntry(byte[]bytes, int p, int end) { + name = bytes; + nameP = p; + nameEnd = end; + } + + public int[] getBackRefs() { + switch (backNum) { + case 0: + return new int[]{}; + case 1: + return new int[]{backRef1}; + default: + int[]result = new int[backNum]; + System.arraycopy(backRefs, 0, result, 0, backNum); + return result; + } + } + + private void alloc() { + backRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; + } + + private void ensureSize() { + if (backNum > backRefs.length) { + int[]tmp = new int[backRefs.length << 1]; + System.arraycopy(backRefs, 0, tmp, 0, backRefs.length); + backRefs = tmp; + } + } + + public void addBackref(int backRef) { + backNum++; + + switch (backNum) { + case 1: + backRef1 = backRef; + break; + case 2: + alloc(); + backRefs[0] = backRef1; + backRefs[1] = backRef; + break; + default: + ensureSize(); + backRefs[backNum - 1] = backRef; + } + } + + @Override + public String toString() { + StringBuilder buff = new StringBuilder(new String(name, nameP, nameEnd - nameP) + " "); + if (backNum == 0) { + buff.append("-"); + } else if (backNum == 1){ + buff.append(backRef1); + } else { + for (int i=0; i 0) buff.append(", "); + buff.append(backRefs[i]); + } + } + return buff.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/NativeMachine.java b/third_party/joni/src/org/joni/NativeMachine.java new file mode 100644 index 000000000..d086123a5 --- /dev/null +++ b/third_party/joni/src/org/joni/NativeMachine.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +abstract class NativeMachine extends Matcher { + + protected NativeMachine(Regex regex, Region region, byte[]bytes, int p, int end) { + super(regex, region, bytes, p, end); + } +} diff --git a/third_party/joni/src/org/joni/NodeOptInfo.java b/third_party/joni/src/org/joni/NodeOptInfo.java new file mode 100644 index 000000000..9eb737577 --- /dev/null +++ b/third_party/joni/src/org/joni/NodeOptInfo.java @@ -0,0 +1,127 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +final class NodeOptInfo { + final MinMaxLen length = new MinMaxLen(); + final OptAnchorInfo anchor = new OptAnchorInfo(); + final OptExactInfo exb = new OptExactInfo(); /* boundary */ + final OptExactInfo exm = new OptExactInfo(); /* middle */ + final OptExactInfo expr = new OptExactInfo(); /* prec read (?=...) */ + final OptMapInfo map = new OptMapInfo(); /* boundary */ + + public void setBoundNode(MinMaxLen mmd) { + exb.mmd.copy(mmd); + expr.mmd.copy(mmd); + map.mmd.copy(mmd); + } + + public void clear() { + length.clear(); + anchor.clear(); + exb.clear(); + exm.clear(); + expr.clear(); + map.clear(); + } + + public void copy(NodeOptInfo other) { + length.copy(other.length); + anchor.copy(other.anchor); + exb.copy(other.exb); + exm.copy(other.exm); + expr.copy(other.expr); + map.copy(other.map); + } + + public void concatLeftNode(NodeOptInfo other, Encoding enc) { + OptAnchorInfo tanchor = new OptAnchorInfo(); // remove it somehow ? + tanchor.concat(anchor, other.anchor, length.max, other.length.max); + anchor.copy(tanchor); + + if (other.exb.length > 0 && length.max == 0) { + tanchor.concat(anchor, other.exb.anchor, length.max, other.length.max); + other.exb.anchor.copy(tanchor); + } + + if (other.map.value > 0 && length.max == 0) { + if (other.map.mmd.max == 0) { + other.map.anchor.leftAnchor |= anchor.leftAnchor; + } + } + + boolean exbReach = exb.reachEnd; + boolean exmReach = exm.reachEnd; + + if (other.length.max != 0) { + exb.reachEnd = exm.reachEnd = false; + } + + if (other.exb.length > 0) { + if (exbReach) { + exb.concat(other.exb, enc); + other.exb.clear(); + } else if (exmReach) { + exm.concat(other.exb, enc); + other.exb.clear(); + } + } + + exm.select(other.exb, enc); + exm.select(other.exm, enc); + + if (expr.length > 0) { + if (other.length.max > 0) { + // TODO: make sure it is not an Oniguruma bug (casting unsigned int to int for arithmetic comparison) + int otherLengthMax = other.length.max; + if (otherLengthMax == MinMaxLen.INFINITE_DISTANCE) otherLengthMax = -1; + if (expr.length > otherLengthMax) expr.length = otherLengthMax; + if (expr.mmd.max == 0) { + exb.select(expr, enc); + } else { + exm.select(expr, enc); + } + } + } else if (other.expr.length > 0) { + expr.copy(other.expr); + } + + map.select(other.map); + length.add(other.length); + } + + public void altMerge(NodeOptInfo other, OptEnvironment env) { + anchor.altMerge(other.anchor); + exb.altMerge(other.exb, env); + exm.altMerge(other.exm, env); + expr.altMerge(other.expr, env); + map.altMerge(other.map, env.enc); + length.altMerge(other.length); + } + + public void setBound(MinMaxLen mmd) { + exb.mmd.copy(mmd); + expr.mmd.copy(mmd); + map.mmd.copy(mmd); + } + +} diff --git a/third_party/joni/src/org/joni/OptAnchorInfo.java b/third_party/joni/src/org/joni/OptAnchorInfo.java new file mode 100644 index 000000000..6774e20af --- /dev/null +++ b/third_party/joni/src/org/joni/OptAnchorInfo.java @@ -0,0 +1,96 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.constants.internal.AnchorType; + +final class OptAnchorInfo implements AnchorType { + int leftAnchor; + int rightAnchor; + + void clear() { + leftAnchor = rightAnchor = 0; + } + + void copy(OptAnchorInfo other) { + leftAnchor = other.leftAnchor; + rightAnchor = other.rightAnchor; + } + + void concat(OptAnchorInfo left, OptAnchorInfo right, int leftLength, int rightLength) { + leftAnchor = left.leftAnchor; + if (leftLength == 0) leftAnchor |= right.leftAnchor; + + rightAnchor = right.rightAnchor; + if (rightLength == 0) { + rightAnchor |= left.rightAnchor; + } else { + rightAnchor |= left.rightAnchor & AnchorType.PREC_READ_NOT; + } + } + + boolean isSet(int anchor) { + if ((leftAnchor & anchor) != 0) return true; + return (rightAnchor & anchor) != 0; + } + + void add(int anchor) { + if (isLeftAnchor(anchor)) { + leftAnchor |= anchor; + } else { + rightAnchor |= anchor; + } + } + + void remove(int anchor) { + if (isLeftAnchor(anchor)) { + leftAnchor &= ~anchor; + } else { + rightAnchor &= ~anchor; + } + } + + void altMerge(OptAnchorInfo other) { + leftAnchor &= other.leftAnchor; + rightAnchor &= other.rightAnchor; + } + + static boolean isLeftAnchor(int anchor) { // make a mask for it ? + return !(anchor == END_BUF || anchor == SEMI_END_BUF || + anchor == END_LINE || anchor == PREC_READ || + anchor == PREC_READ_NOT); + } + + static String anchorToString(int anchor) { + StringBuilder s = new StringBuilder("["); + + if ((anchor & AnchorType.BEGIN_BUF) !=0 ) s.append("begin-buf "); + if ((anchor & AnchorType.BEGIN_LINE) !=0 ) s.append("begin-line "); + if ((anchor & AnchorType.BEGIN_POSITION) !=0 ) s.append("begin-pos "); + if ((anchor & AnchorType.END_BUF) !=0 ) s.append("end-buf "); + if ((anchor & AnchorType.SEMI_END_BUF) !=0 ) s.append("semi-end-buf "); + if ((anchor & AnchorType.END_LINE) !=0 ) s.append("end-line "); + if ((anchor & AnchorType.ANYCHAR_STAR) !=0 ) s.append("anychar-star "); + if ((anchor & AnchorType.ANYCHAR_STAR_ML) !=0 ) s.append("anychar-star-pl "); + s.append("]"); + + return s.toString(); + } +} diff --git a/third_party/joni/src/org/joni/OptEnvironment.java b/third_party/joni/src/org/joni/OptEnvironment.java new file mode 100644 index 000000000..bc627508d --- /dev/null +++ b/third_party/joni/src/org/joni/OptEnvironment.java @@ -0,0 +1,39 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +// remove this one in future and pass mmd directly +final class OptEnvironment { + final MinMaxLen mmd = new MinMaxLen(); + Encoding enc; + int options; + int caseFoldFlag; + ScanEnvironment scanEnv; + + void copy(OptEnvironment other) { + mmd.copy(other.mmd); + enc = other.enc; + options = other.options; + caseFoldFlag = other.caseFoldFlag; + scanEnv = other.scanEnv; + } +} diff --git a/third_party/joni/src/org/joni/OptExactInfo.java b/third_party/joni/src/org/joni/OptExactInfo.java new file mode 100644 index 000000000..ab975bbbb --- /dev/null +++ b/third_party/joni/src/org/joni/OptExactInfo.java @@ -0,0 +1,170 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +final class OptExactInfo { + static final int OPT_EXACT_MAXLEN = 24; + + final MinMaxLen mmd = new MinMaxLen(); + final OptAnchorInfo anchor = new OptAnchorInfo(); + boolean reachEnd; + int ignoreCase; /* -1: unset, 0: case sensitive, 1: ignore case */ + final byte[] bytes = new byte[OPT_EXACT_MAXLEN]; + int length; + + boolean isFull() { + return length >= OPT_EXACT_MAXLEN; + } + + void clear() { + mmd.clear(); + anchor.clear(); + reachEnd = false; + ignoreCase = -1; + length = 0; + } + + void copy(OptExactInfo other) { + mmd.copy(other.mmd); + anchor.copy(other.anchor); + reachEnd = other.reachEnd; + ignoreCase = other.ignoreCase; + length = other.length; + + System.arraycopy(other.bytes, 0, bytes, 0, OPT_EXACT_MAXLEN); + } + + void concat(OptExactInfo other, Encoding enc) { + if (ignoreCase < 0) { + ignoreCase = other.ignoreCase; + } else if (ignoreCase != other.ignoreCase) { + return; + } + + int p = 0; // add->s; + int end = p + other.length; + + int i; + for (i = length; p < end;) { + int len = enc.length(other.bytes, p, end); + if (i + len > OPT_EXACT_MAXLEN) break; + for (int j = 0; j < len && p < end; j++) { + bytes[i++] = other.bytes[p++]; // arraycopy or even don't copy anything ?? + } + } + + length = i; + reachEnd = (p == end && other.reachEnd); + + OptAnchorInfo tmp = new OptAnchorInfo(); + tmp.concat(anchor, other.anchor, 1, 1); + if (!reachEnd) tmp.rightAnchor = 0; + anchor.copy(tmp); + } + + void concatStr(byte[]lbytes, int p, int end, boolean raw, Encoding enc) { + int i; + for (i = length; p < end && i < OPT_EXACT_MAXLEN;) { + int len = enc.length(lbytes, p, end); + if (i + len > OPT_EXACT_MAXLEN) break; + for (int j = 0; j < len && p < end; j++) { + bytes[i++] = lbytes[p++]; + } + } + length = i; + } + + void altMerge(OptExactInfo other, OptEnvironment env) { + if (other.length == 0 || length == 0) { + clear(); + return; + } + + if (!mmd.equal(other.mmd)) { + clear(); + return; + } + + int i; + for (i=0; i= 0) { + ignoreCase |= other.ignoreCase; + } + + anchor.altMerge(other.anchor); + + if (!reachEnd) anchor.rightAnchor = 0; + } + + + void select(OptExactInfo alt, Encoding enc) { + int v1 = length; + int v2 = alt.length; + + if (v2 == 0) { + return; + } else if (v1 == 0) { + copy(alt); + return; + } else if (v1 <= 2 && v2 <= 2) { + /* ByteValTable[x] is big value --> low price */ + v2 = OptMapInfo.positionValue(enc, bytes[0] & 0xff); + v1 = OptMapInfo.positionValue(enc, alt.bytes[0] & 0xff); + + if (length > 1) v1 += 5; + if (alt.length > 1) v2 += 5; + } + + if (ignoreCase <= 0) v1 *= 2; + if (alt.ignoreCase <= 0) v2 *= 2; + + if (mmd.compareDistanceValue(alt.mmd, v1, v2) > 0) copy(alt); + } + + // comp_opt_exact_or_map_info + private static final int COMP_EM_BASE = 20; + int compare(OptMapInfo m) { + if (m.value <= 0) return -1; + + int ve = COMP_EM_BASE * length * (ignoreCase > 0 ? 1 : 2); + int vm = COMP_EM_BASE * 5 * 2 / m.value; + + return mmd.compareDistanceValue(m.mmd, ve, vm); + } +} diff --git a/third_party/joni/src/org/joni/OptMapInfo.java b/third_party/joni/src/org/joni/OptMapInfo.java new file mode 100644 index 000000000..0aeaec9e1 --- /dev/null +++ b/third_party/joni/src/org/joni/OptMapInfo.java @@ -0,0 +1,126 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.Encoding; + +final class OptMapInfo { + final MinMaxLen mmd = new MinMaxLen(); /* info position */ + final OptAnchorInfo anchor = new OptAnchorInfo(); + int value; /* weighted value */ + final byte[] map = new byte[Config.CHAR_TABLE_SIZE]; + + void clear() { + mmd.clear(); + anchor.clear(); + value = 0; + for (int i=0; i 0) copy(alt); + } + + // alt_merge_opt_map_info + void altMerge(OptMapInfo other, Encoding enc) { + /* if (! is_equal_mml(&to->mmd, &add->mmd)) return ; */ + if (value == 0) return; + if (other.value == 0 || mmd.max < other.mmd.max) { + clear(); + return; + } + + mmd.altMerge(other.mmd); + + int val = 0; + for (int i=0; i 1) { + return 20; + } else { + return ByteValTable[i]; + } + } else { + return 4; /* Take it easy. */ + } + } + +} diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java new file mode 100644 index 000000000..1747738a7 --- /dev/null +++ b/third_party/joni/src/org/joni/Option.java @@ -0,0 +1,149 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class Option { + /* options */ + public static final int NONE = 0; + public static final int IGNORECASE = (1 << 0); + public static final int EXTEND = (1 << 1); + public static final int MULTILINE = (1 << 2); + public static final int SINGLELINE = (1 << 3); + public static final int FIND_LONGEST = (1 << 4); + public static final int FIND_NOT_EMPTY = (1 << 5); + public static final int NEGATE_SINGLELINE = (1 << 6); + public static final int DONT_CAPTURE_GROUP = (1 << 7); + public static final int CAPTURE_GROUP = (1 << 8); + + /* options (search time) */ + public static final int NOTBOL = (1 << 9); + public static final int NOTEOL = (1 << 10); + public static final int POSIX_REGION = (1 << 11); + + /* options (ctype range) */ + public static final int ASCII_RANGE = (1 << 12); + public static final int POSIX_BRACKET_ALL_RANGE = (1 << 13); + public static final int WORD_BOUND_ALL_RANGE = (1 << 14); + /* options (newline) */ + public static final int NEWLINE_CRLF = (1 << 15); + public static final int NOTBOS = (1 << 16); + public static final int NOTEOS = (1 << 17); + public static final int CR_7_BIT = (1 << 18); + + public static final int MAXBIT = (1 << 19); /* limit */ + + public static final int DEFAULT = NONE; + + public static String toString(int option) { + String options = ""; + if (isIgnoreCase(option)) options += "IGNORECASE"; + if (isExtend(option)) options += "EXTEND"; + if (isMultiline(option)) options += "MULTILINE"; + if (isSingleline(option)) options += "SINGLELINE"; + if (isFindLongest(option)) options += "FIND_LONGEST"; + if (isFindNotEmpty(option)) options += "FIND_NOT_EMPTY"; + if (isNegateSingleline(option)) options += "NEGATE_SINGLELINE"; + if (isDontCaptureGroup(option)) options += "DONT_CAPTURE_GROUP"; + if (isCaptureGroup(option)) options += "CAPTURE_GROUP"; + if (isNotBol(option)) options += "NOTBOL"; + if (isNotEol(option)) options += "NOTEOL"; + if (isPosixRegion(option)) options += "POSIX_REGION"; + if (isCR7Bit(option)) options += "CR_7_BIT"; + return options; + } + + public static boolean isIgnoreCase(int option) { + return (option & IGNORECASE) != 0; + } + + public static boolean isExtend(int option) { + return (option & EXTEND) != 0; + } + + public static boolean isSingleline(int option) { + return (option & SINGLELINE) != 0; + } + + public static boolean isMultiline(int option) { + return (option & MULTILINE) != 0; + } + + public static boolean isFindLongest(int option) { + return (option & FIND_LONGEST) != 0; + } + + public static boolean isFindNotEmpty(int option) { + return (option & FIND_NOT_EMPTY) != 0; + } + + public static boolean isFindCondition(int option) { + return (option & (FIND_LONGEST | FIND_NOT_EMPTY)) != 0; + } + + public static boolean isNegateSingleline(int option) { + return (option & NEGATE_SINGLELINE) != 0; + } + + public static boolean isDontCaptureGroup(int option) { + return (option & DONT_CAPTURE_GROUP) != 0; + } + + public static boolean isCaptureGroup(int option) { + return (option & CAPTURE_GROUP) != 0; + } + + public static boolean isNotBol(int option) { + return (option & NOTBOL) != 0; + } + + public static boolean isNotEol(int option) { + return (option & NOTEOL) != 0; + } + + public static boolean isPosixRegion(int option) { + return (option & POSIX_REGION) != 0; + } + + public static boolean isAsciiRange(int option) { + return (option & ASCII_RANGE) != 0; + } + + public static boolean isPosixBracketAllRange(int option) { + return (option & POSIX_BRACKET_ALL_RANGE) != 0; + } + + public static boolean isWordBoundAllRange(int option) { + return (option & WORD_BOUND_ALL_RANGE) != 0; + } + + public static boolean isNewlineCRLF(int option) { + return (option & NEWLINE_CRLF) != 0; + } + + public static boolean isCR7Bit(int option) { + return (option & CR_7_BIT) != 0; + } + + public static boolean isDynamic(int option) { + // ignore-case and multibyte status are included in compiled code + // return (option & (MULTILINE | IGNORECASE)) != 0; + return false; + } +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java new file mode 100644 index 000000000..6077ae08d --- /dev/null +++ b/third_party/joni/src/org/joni/Parser.java @@ -0,0 +1,1433 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsOnAtSimple; +import static org.joni.BitStatus.bsOnOff; +import static org.joni.Option.isAsciiRange; +import static org.joni.Option.isDontCaptureGroup; +import static org.joni.Option.isIgnoreCase; +import static org.joni.Option.isPosixBracketAllRange; + +import org.jcodings.Encoding; +import org.jcodings.ObjPtr; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.jcodings.constants.PosixBracket; +import org.jcodings.exception.InternalException; +import org.jcodings.unicode.UnicodeCodeRange; +import org.joni.ast.AnchorNode; +import org.joni.ast.AnyCharNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CClassNode.CCSTATE; +import org.joni.ast.CClassNode.CCStateArg; +import org.joni.ast.CClassNode.CCVALTYPE; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.ListNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.EncloseType; +import org.joni.constants.internal.NodeType; +import org.joni.constants.internal.TokenType; +import org.joni.exception.ErrorMessages; + +class Parser extends Lexer { + protected int returnCode; // return code used by parser methods (they itself return parsed nodes) + // this approach will not affect recursive calls + + protected Parser(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex, syntax, bytes, p, end, warnings); + } + + private static final int POSIX_BRACKET_NAME_MIN_LEN = 4; + private static final int POSIX_BRACKET_CHECK_LIMIT_LENGTH = 20; + private static final byte[] BRACKET_END = ":]".getBytes(); + private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc) { + mark(); + + boolean not; + if (peekIs('^')) { + inc(); + not = true; + } else { + not = false; + } + if (enc.strLength(bytes, p, stop) >= POSIX_BRACKET_NAME_MIN_LEN + 3) { // else goto not_posix_bracket + boolean asciiRange = isAsciiRange(env.option) && !isPosixBracketAllRange(env.option); + + for (int i=0; i POSIX_BRACKET_CHECK_LIMIT_LENGTH) break; + } + + if (c == ':' && left()) { + inc(); + if (left()) { + fetch(); + if (c == ']') newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + } + } + restore(); + return true; /* 1: is not POSIX bracket, but no error. */ + } + + private boolean codeExistCheck(int code, boolean ignoreEscaped) { + mark(); + + boolean inEsc = false; + while (left()) { + if (ignoreEscaped && inEsc) { + inEsc = false; + } else { + fetch(); + if (c == code) { + restore(); + return true; + } + if (c == syntax.metaCharTable.esc) inEsc = true; + } + } + + restore(); + return false; + } + + private CClassNode parseCharClass(ObjPtr ascNode) { + final boolean neg; + CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null; + CCStateArg arg = new CCStateArg(); + + fetchTokenInCC(); + if (token.type == TokenType.CHAR && token.getC() == '^' && !token.escaped) { + neg = true; + fetchTokenInCC(); + } else { + neg = false; + } + + if (token.type == TokenType.CC_CLOSE && !syntax.op3OptionECMAScript()) { + if (!codeExistCheck(']', true)) newSyntaxException(EMPTY_CHAR_CLASS); + env.ccEscWarn("]"); + token.type = TokenType.CHAR; /* allow []...] */ + } + + cc = new CClassNode(); + if (isIgnoreCase(env.option)) { + ascCc = ascNode.p = new CClassNode(); + } + + boolean andStart = false; + arg.state = CCSTATE.START; + while (token.type != TokenType.CC_CLOSE) { + boolean fetched = false; + + switch (token.type) { + case CHAR: + final int len; + if (token.getCode() >= BitSet.SINGLE_BYTE_SIZE || (len = enc.codeToMbcLength(token.getC())) > 1) { + arg.inType = CCVALTYPE.CODE_POINT; + } else { + arg.inType = CCVALTYPE.SB; // sb_char: + } + arg.to = token.getC(); + arg.toIsRaw = false; + parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + break; + + case RAW_BYTE: + if (!enc.isSingleByte() && token.base != 0) { /* tok->base != 0 : octal or hexadec. */ + byte[]buf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN]; + int psave = p; + int base = token.base; + buf[0] = (byte)token.getC(); + int i; + for (i=1; i len) { /* fetch back */ + p = psave; + for (i=1; i ascPtr = new ObjPtr<>(); + CClassNode acc = parseCharClass(ascPtr); + cc.or(acc, env); + if (ascPtr.p != null) { + ascCc.or(ascPtr.p, env); + } + break; + + case CC_AND: /* && */ + if (arg.state == CCSTATE.VALUE) { + arg.to = 0; + arg.toIsRaw = false; + cc.nextStateValue(arg, ascCc, env); + } + /* initialize local variables */ + andStart = true; + arg.state = CCSTATE.START; + if (prevCc != null) { + prevCc.and(cc, env); + if (ascCc != null) { + ascPrevCc.and(ascCc, env); + } + } else { + prevCc = cc; + if (workCc == null) workCc = new CClassNode(); + cc = workCc; + if (ascCc != null) { + ascPrevCc = ascCc; + if (ascWorkCc == null) ascWorkCc = new CClassNode(); + ascCc = ascWorkCc; + } + } + cc.clear(); + if (ascCc != null) ascCc.clear(); + break; + + case EOT: + newSyntaxException(PREMATURE_END_OF_CHAR_CLASS); + + default: + newInternalException(PARSER_BUG); + } // switch + + if (!fetched) fetchTokenInCC(); + + } // while + + if (arg.state == CCSTATE.VALUE) { + arg.to = 0; + arg.toIsRaw = false; + cc.nextStateValue(arg, ascCc, env); + } + + if (prevCc != null) { + prevCc.and(cc, env); + cc = prevCc; + if (ascCc != null) { + ascPrevCc.and(ascCc, env); + ascCc = ascPrevCc; + } + } + + if (neg) { + cc.setNot(); + if (ascCc != null) ascCc.setNot(); + } else { + cc.clearNot(); + if (ascCc != null) ascCc.clearNot(); + } + + if (cc.isNot() && syntax.notNewlineInNegativeCC()) { + if (!cc.isEmpty()) { // ??? + final int NEW_LINE = 0x0a; + if (enc.isNewLine(NEW_LINE)) { + if (enc.codeToMbcLength(NEW_LINE) == 1) { + cc.bs.set(env, NEW_LINE); + } else { + cc.addCodeRange(env, NEW_LINE, NEW_LINE); + } + } + } + } + + return cc; + } + + private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + arg.inType = CCVALTYPE.SB; + arg.to = token.getC(); + arg.toIsRaw = false; + parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + } + + private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + arg.to = '-'; + arg.toIsRaw = false; + parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + } + + private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + int len = enc.codeToMbcLength(arg.to); + arg.inType = len == 1 ? CCVALTYPE.SB : CCVALTYPE.CODE_POINT; + parseCharClassValEntry2(cc, ascCc, arg); // val_entry2: + } + + private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + cc.nextStateValue(arg, ascCc, env); + } + + private Node parseEnclose(TokenType term) { + Node node = null; + + if (!left()) newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + + int option = env.option; + + if (peekIs('?') && syntax.op2QMarkGroupEffect()) { + inc(); + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + + boolean listCapture = false; + + fetch(); + switch(c) { + case '{': + return parseInternalCallout(); + case ':': /* (?:...) grouping only */ + fetchToken(); // group: + node = parseSubExp(term); + returnCode = 1; /* group */ + return node; + case '=': + node = new AnchorNode(AnchorType.PREC_READ); + break; + case '!': /* preceding read */ + node = new AnchorNode(AnchorType.PREC_READ_NOT); + if (syntax.op3OptionECMAScript()) { + env.pushPrecReadNotNode(node); + } + break; + case '>': /* (?>...) stop backtrack */ + node = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + break; + case '~': /* (?~...) absent operator */ + if (syntax.op2QMarkTildeAbsent()) { + node = new EncloseNode(EncloseType.ABSENT); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + case '\'': + if (Config.USE_NAMED_GROUP) { + if (syntax.op2QMarkLtNamedGroup()) { + listCapture = false; // goto named_group1 + node = parseEncloseNamedGroup2(listCapture); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + } // USE_NAMED_GROUP + break; + case '<': /* look behind (?<=...), (?...) */ + } + unfetch(); + } + } // USE_NAMED_GROUP + EncloseNode en = EncloseNode.newMemory(env.option, false); + int num = env.addMemEntry(); + if (num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); + en.regNum = num; + node = en; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case '(': /* conditional expression: (?(cond)yes), (?(cond)yes|no) */ + if (left() && syntax.op2QMarkLParenCondition()) { + int num = -1; + int name = -1; + fetch(); + if (enc.isDigit(c)) { /* (n) */ + unfetch(); + num = fetchName('(', true); + if (syntax.strictCheckBackref()) { + if (num > env.numMem || env.memNodes == null || env.memNodes[num] == null) newValueException(INVALID_BACKREF); + } + } else { + if (Config.USE_NAMED_GROUP) { + if (c == '<' || c == '\'') { /* (), ('name') */ + name = p; + fetchNamedBackrefToken(); + inc(); + num = token.getBackrefNum() > 1 ? token.getBackrefRefs()[0] : token.getBackrefRef1(); + } + } else { // USE_NAMED_GROUP + newSyntaxException(INVALID_CONDITION_PATTERN); + } + } + EncloseNode en = new EncloseNode(EncloseType.CONDITION); + en.regNum = num; + if (name != -1) en.setNameRef(); + node = en; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case '^': /* loads default options */ + if (left() && syntax.op2OptionPerl()) { + /* d-imsx */ + option = bsOnOff(option, Option.ASCII_RANGE, true); + option = bsOnOff(option, Option.IGNORECASE, true); + option = bsOnOff(option, Option.SINGLELINE, false); + option = bsOnOff(option, Option.MULTILINE, true); + option = bsOnOff(option, Option.EXTEND, true); + fetch(); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + // case 'p': #ifdef USE_POSIXLINE_OPTION + case '-': + case 'i': + case 'm': + case 's': + case 'x': + case 'a': + case 'd': + case 'l': + case 'u': + boolean neg = false; + while (true) { + switch(c) { + case ':': + case ')': + break; + case '-': + neg = true; + break; + case 'x': + option = bsOnOff(option, Option.EXTEND, neg); + break; + case 'i': + option = bsOnOff(option, Option.IGNORECASE, neg); + break; + case 's': + if (syntax.op2OptionPerl()) { + option = bsOnOff(option, Option.MULTILINE, neg); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + case 'm': + if (syntax.op2OptionPerl()) { + option = bsOnOff(option, Option.SINGLELINE, !neg); + } else if (syntax.op2OptionRuby()) { + option = bsOnOff(option, Option.MULTILINE, neg); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + // case 'p': #ifdef USE_POSIXLINE_OPTION // not defined + // option = bsOnOff(option, Option.MULTILINE|Option.SINGLELINE, neg); + // break; + + case 'a': /* limits \d, \s, \w and POSIX brackets to ASCII range */ + if ((syntax.op2OptionPerl() || syntax.op2OptionRuby()) && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, false); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, true); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, true); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + case 'u': + if ((syntax.op2OptionPerl() || syntax.op2OptionRuby()) && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, true); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, true); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + case 'd': + if (syntax.op2OptionPerl() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + } else if (syntax.op2OptionRuby() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, false); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, false); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, false); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case 'l': + if (syntax.op2OptionPerl() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + default: + newSyntaxException(UNDEFINED_GROUP_OPTION); + } // switch + + if (c == ')') { + node = EncloseNode.newOption(option); + returnCode = 2; /* option only */ + return node; + } else if (c == ':') { + int prev = env.option; + env.option = option; + fetchToken(); + Node target = parseSubExp(term); + env.option = prev; + EncloseNode en = EncloseNode.newOption(option); + en.setTarget(target); + node = en; + returnCode = 0; + return node; + } + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + } // while + + default: + newSyntaxException(UNDEFINED_GROUP_OPTION); + } // switch + + } else { + if (isDontCaptureGroup(env.option)) { + fetchToken(); // goto group + node = parseSubExp(term); + returnCode = 1; /* group */ + return node; + } + EncloseNode en = EncloseNode.newMemory(env.option, false); + en.regNum = env.addMemEntry(); + node = en; + } + + fetchToken(); + Node target = parseSubExp(term); + + if (node.getType() == NodeType.ANCHOR) { + AnchorNode an = (AnchorNode)node; + an.setTarget(target); + if (syntax.op3OptionECMAScript() && an.type == AnchorType.PREC_READ_NOT) { + env.popPrecReadNotNode(an); + } + } else { + EncloseNode en = (EncloseNode)node; + en.setTarget(target); + if (en.type == EncloseType.MEMORY) { + if (syntax.op3OptionECMAScript()) { + en.containingAnchor = env.currentPrecReadNotNode(); + } + /* Don't move this to previous of parse_subexp() */ + env.setMemNode(en.regNum, en); + } else if (en.type == EncloseType.CONDITION) { + if (target.getType() != NodeType.ALT) { /* convert (?(cond)yes) to (?(cond)yes|empty) */ + en.setTarget(ListNode.newAlt(target, ListNode.newAlt(StringNode.EMPTY, null))); + } + } + } + returnCode = 0; + return node; // ?? + } + + private Node parseInternalCallout() { + final String prefix = "=CALL:"; + for (int i = 0; i < prefix.length(); i++) { + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c != prefix.charAt(i)) newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + if (!left() || !enc.isDigit(peek())) newSyntaxException(UNDEFINED_GROUP_OPTION); + fetch(); + int calloutId = Encoding.digitVal(c); + while (left() && enc.isDigit(peek())) { + fetch(); + int digit = Encoding.digitVal(c); + if (calloutId > (Integer.MAX_VALUE - digit) / 10) { + newValueException("callout id is too large"); + } + calloutId = calloutId * 10 + digit; + } + + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c != '}') newSyntaxException(UNDEFINED_GROUP_OPTION); + if (!left()) newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + fetch(); + if (c != ')') newSyntaxException(UNMATCHED_CLOSE_PARENTHESIS); + + returnCode = 0; + return new CalloutNode(calloutId); + } + + private Node parseEncloseNamedGroup2(boolean listCapture) { + int nm = p; + fetchName(c, false); + int nameEnd = value; + int num = env.addMemEntry(); + if (listCapture && num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); + + regex.nameAdd(bytes, nm, nameEnd, num, syntax); + EncloseNode en = EncloseNode.newMemory(env.option, true); + en.regNum = num; + + if (listCapture) env.captureHistory = bsOnAtSimple(env.captureHistory, num); + env.numNamed++; + return en; + } + + private int findStrPosition(int[]s, int n, int from, int to, Ptr nextChar) { + int x; + int q; + int p = from; + int i; + while (p < to) { + x = enc.mbcToCode(bytes, p, to); + q = p + enc.length(bytes, p, to); + if (x == s[0]) { + for (i=1; i= n) { + if (bytes[nextChar.p] != 0) nextChar.p = q; // we may need zero term semantics... + return p; + } + } + p = q; + } + return -1; + } + + private Node parseExp(TokenType term) { + if (token.type == term) return StringNode.EMPTY; // goto end_of_token + Node node = null; + boolean group = false; + + switch(token.type) { + case ALT: + case EOT: + return StringNode.EMPTY; // end_of_token:, node_new_empty + + case SUBEXP_OPEN: + node = parseEnclose(TokenType.SUBEXP_CLOSE); + if (returnCode == 1) { + group = true; + } else if (returnCode == 2) { /* option only */ + int prev = env.option; + EncloseNode en = (EncloseNode)node; + env.option = en.option; + fetchToken(); + Node target = parseSubExp(term); + env.option = prev; + en.setTarget(target); + return node; + } + break; + case SUBEXP_CLOSE: + if (!syntax.allowUnmatchedCloseSubexp()) newSyntaxException(UNMATCHED_CLOSE_PARENTHESIS); + if (token.escaped) { + return parseExpTkRawByte(group); // goto tk_raw_byte + } else { + return parseExpTkByte(group); // goto tk_byte + } + case LINEBREAK: + node = parseLineBreak(); + break; + + case EXTENDED_GRAPHEME_CLUSTER: + node = parseExtendedGraphemeCluster(); + break; + + case KEEP: + node = new AnchorNode(AnchorType.KEEP); + break; + + case STRING: + return parseExpTkByte(group); // tk_byte: + + case RAW_BYTE: + return parseExpTkRawByte(group); // tk_raw_byte: + + case CODE_POINT: + return parseStringLoop(StringNode.fromCodePoint(token.getCode(), enc), group); + + case QUOTE_OPEN: + node = parseQuoteOpen(); + break; + + case CHAR_TYPE: + node = parseCharType(node); + break; + + case CHAR_PROPERTY: + node = parseCharProperty(); + break; + + case CC_OPEN: { + ObjPtr ascPtr = new ObjPtr<>(); + CClassNode cc = parseCharClass(ascPtr); + int code = cc.isOneChar(); + if (code != -1) return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + + node = cc; + if (isIgnoreCase(env.option)) node = cClassCaseFold(node, cc, ascPtr.p); + break; + } + + case ANYCHAR: + node = new AnyCharNode(); + break; + + case ANYCHAR_ANYTIME: + node = parseAnycharAnytime(); + break; + + case BACKREF: + node = parseBackref(); + break; + + case CALL: + if (Config.USE_SUBEXP_CALL) node = parseCall(); + break; + + case ANCHOR: + node = new AnchorNode(token.getAnchorSubtype(), token.getAnchorASCIIRange()); + break; + + case OP_REPEAT: + case INTERVAL: + if (syntax.contextIndepRepeatOps()) { + if (syntax.contextInvalidRepeatOps()) { + newSyntaxException(TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + } else { + node = StringNode.EMPTY; // node_new_empty + } + } else { + return parseExpTkByte(group); // goto tk_byte + } + break; + + default: + newInternalException(PARSER_BUG); + } //switch + + //targetp = node; + + fetchToken(); // re_entry: + + return parseExpRepeat(node, group); // repeat: + } + + private Node parseLineBreak() { + byte[]buflb = new byte[Config.ENC_CODE_TO_MBC_MAXLEN * 2]; + int len1 = enc.codeToMbc(0x0D, buflb, 0); + int len2 = enc.codeToMbc(0x0A, buflb, len1); + StringNode left = new StringNode(buflb, 0, len1 + len2); + left.setRaw(); + /* [\x0A-\x0D] or [\x0A-\x0D\x{85}\x{2028}\x{2029}] */ + CClassNode right = new CClassNode(); + if (enc.minLength() > 1) { + right.addCodeRange(env, 0x0A, 0x0D); + } else { + right.bs.setRange(env, 0x0A, 0x0D); + } + + if (enc.isUnicode()) { + /* UTF-8, UTF-16BE/LE, UTF-32BE/LE */ + right.addCodeRange(env, 0x85, 0x85); + right.addCodeRange(env, 0x2028, 0x2029); + } + /* (?>...) */ + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); + en.setTarget(ListNode.newAlt(left, ListNode.newAlt(right, null))); + return en; + } + + private void addPropertyToCC(CClassNode cc, UnicodeCodeRange range, boolean not) { + cc.addCType(range.getCType(), not, false, env, this); + } + + private void createPropertyNode(Node[]nodes, int np, UnicodeCodeRange range) { + CClassNode cc = new CClassNode(); + addPropertyToCC(cc, range, false); + nodes[np] = cc; + } + + private void quantifierNode(Node[]nodes, int np, int lower, int upper) { + QuantifierNode qnf = new QuantifierNode(lower, upper, false); + qnf.setTarget(nodes[np]); + nodes[np] = qnf; + } + + private void quantifierPropertyNode(Node[]nodes, int np, UnicodeCodeRange range, char repetitions) { + int lower = 0; + int upper = QuantifierNode.REPEAT_INFINITE; + + createPropertyNode(nodes, np, range); + switch (repetitions) { + case '?': upper = 1; break; + case '+': lower = 1; break; + case '*': break; + case '2': lower = upper = 2; break; + default : throw new InternalException(ErrorMessages.PARSER_BUG); + } + + quantifierNode(nodes, np, lower, upper); + } + + private void createNodeFromArray(boolean list, Node[] nodes, int np, int nodeArray) { + int i = 0; + ListNode tmp = null; + while (nodes[nodeArray + i] != null) i++; + while (--i >= 0) { + nodes[np] = list ? ListNode.newList(nodes[nodeArray + i], tmp) : ListNode.newAlt(nodes[nodeArray + i], tmp); + nodes[nodeArray + i] = null; + tmp = (ListNode)nodes[np]; + } + } + + private ListNode createNodeFromArray(Node[]nodes, int nodeArray) { + int i = 0; + ListNode np = null, tmp = null; + while (nodes[nodeArray + i] != null) i++; + while (--i >= 0) { + np = ListNode.newAlt(nodes[nodeArray + i], tmp); + nodes[nodeArray + i] = null; + tmp = np; + } + return np; + } + + private static final int NODE_COMMON_SIZE = 16; + private Node parseExtendedGraphemeCluster() { + final Node[] nodes = new Node[NODE_COMMON_SIZE]; + final int anyTargetPosition; + int alts = 0; + + StringNode strNode = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN * 2); + strNode.setRaw(); + strNode.catCode(0x0D, enc); + strNode.catCode(0x0A, enc); + nodes[alts] = strNode; + + if (Config.USE_UNICODE_PROPERTIES && enc.isUnicode()) { + CClassNode cc; + cc = new CClassNode(); + nodes[alts + 1] = cc; + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, false); + if (enc.minLength() > 1) { + cc.addCodeRange(env, 0x000A, 0x000A); + cc.addCodeRange(env, 0x000D, 0x000D); + } else { + cc.bs.set(0x0A); + cc.bs.set(0x0D); + } + + { + int list = alts + 3; + quantifierPropertyNode(nodes, list + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_PREPEND, '*'); + { + int coreAlts = list + 2; + { + int HList = coreAlts + 1; + quantifierPropertyNode(nodes, HList + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_L, '*'); + { + int HAlt2 = HList + 2; + quantifierPropertyNode(nodes, HAlt2 + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_V, '+'); + { + int HList2 = HAlt2 + 2; + createPropertyNode(nodes, HList2 + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_LV); + quantifierPropertyNode(nodes, HList2 + 1, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_V, '*'); + createNodeFromArray(true, nodes, HAlt2 + 1, HList2); + } + createPropertyNode(nodes, HAlt2 + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_LVT); + createNodeFromArray(false, nodes, HList + 1, HAlt2); + } + quantifierPropertyNode(nodes, HList + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_T, '*'); + createNodeFromArray(true, nodes, coreAlts + 0, HList); + } + quantifierPropertyNode(nodes, coreAlts + 1, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_L, '+'); + quantifierPropertyNode(nodes, coreAlts + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_T, '+'); + quantifierPropertyNode(nodes, coreAlts + 3, UnicodeCodeRange.REGIONALINDICATOR, '2'); + { + int XPList = coreAlts + 5; + createPropertyNode(nodes, XPList + 0, UnicodeCodeRange.EXTENDEDPICTOGRAPHIC); + { + int ExList = XPList + 2; + quantifierPropertyNode(nodes, ExList + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_EXTEND, '*'); + strNode = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN); + strNode.setRaw(); + strNode.catCode(0x200D, enc); + nodes[ExList + 1] = strNode; + createPropertyNode(nodes, ExList + 2, UnicodeCodeRange.EXTENDEDPICTOGRAPHIC); + createNodeFromArray(true, nodes, XPList + 1, ExList); + } + quantifierNode(nodes, XPList + 1, 0, QuantifierNode.REPEAT_INFINITE); + createNodeFromArray(true, nodes, coreAlts + 4, XPList); + } + cc = new CClassNode(); + nodes[coreAlts + 5] = cc; + if (enc.minLength() > 1) { + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, false); + cc.addCodeRange(env, 0x000A, 0x000A); + cc.addCodeRange(env, 0x000D, 0x000D); + cc.mbuf = CodeRangeBuffer.notCodeRangeBuff(env, cc.mbuf); + } else { + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, true); + cc.bs.clear(0x0A); + cc.bs.clear(0x0D); + } + createNodeFromArray(false, nodes, list + 1, coreAlts); + } + createPropertyNode(nodes, list + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_EXTEND); + cc = (CClassNode)nodes[list + 2]; + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_SPACINGMARK, false); + cc.addCodeRange(env, 0x200D, 0x200D); + quantifierNode(nodes, list + 2, 0, QuantifierNode.REPEAT_INFINITE); + createNodeFromArray(true, nodes, alts + 2, list); + + } + anyTargetPosition = 3; + } else { // enc.isUnicode() + anyTargetPosition = 1; + } + + Node any = new AnyCharNode(); + EncloseNode option = EncloseNode.newOption(bsOnOff(env.option, Option.MULTILINE, false)); + option.setTarget(any); + nodes[anyTargetPosition] = option; + + Node topAlt = createNodeFromArray(nodes, alts); + EncloseNode enclose = new EncloseNode(EncloseType.STOP_BACKTRACK); + enclose.setTarget(topAlt); + + if (Config.USE_UNICODE_PROPERTIES && enc.isUnicode()) { + option = EncloseNode.newOption(bsOnOff(env.option, Option.IGNORECASE, true)); + option.setTarget(enclose); + return option; + } else { + return enclose; + } + } + + private Node parseExpTkByte(boolean group) { + StringNode node = new StringNode(bytes, token.backP, p); // tk_byte: + return parseStringLoop(node, group); + } + + private Node parseStringLoop(StringNode node, boolean group) { + while (true) { + fetchToken(); + if (token.type == TokenType.STRING) { + if (token.backP == node.end) { + node.end = p; // non escaped character, remain shared, just increase shared range + } else { + node.catBytes(bytes, token.backP, p); // non continuous string stream, need to COW + } + } else if (token.type == TokenType.CODE_POINT) { + node.catCode(token.getCode(), enc); + } else { + break; + } + } + // targetp = node; + return parseExpRepeat(node, group); // string_end:, goto repeat + } + + private Node parseExpTkRawByte(boolean group) { + // tk_raw_byte: + StringNode node = new StringNode(); + node.setRaw(); + node.catByte((byte)token.getC()); + + int len = 1; + while (true) { + if (len >= enc.minLength()) { + if (len == enc.length(node.bytes, node.p, node.end)) { + fetchToken(); + node.clearRaw(); + // !goto string_end;! + return parseExpRepeat(node, group); + } + } + + fetchToken(); + if (token.type != TokenType.RAW_BYTE) { + /* Don't use this, it is wrong for little endian encodings. */ + // USE_PAD_TO_SHORT_BYTE_CHAR ... + newValueException(TOO_SHORT_MULTI_BYTE_STRING); + } + node.catByte((byte)token.getC()); + len++; + } // while + } + + private Node parseExpRepeat(Node target, boolean group) { + while (token.type == TokenType.OP_REPEAT || token.type == TokenType.INTERVAL) { // repeat: + if (isInvalidQuantifier(target)) newSyntaxException(TARGET_OF_REPEAT_OPERATOR_INVALID); + + if (!group && syntax.op3OptionECMAScript() && target.getType() == NodeType.QTFR) { + newSyntaxException(NESTED_REPEAT_NOT_ALLOWED); + } + QuantifierNode qtfr = new QuantifierNode(token.getRepeatLower(), + token.getRepeatUpper(), + token.type == TokenType.INTERVAL); + + qtfr.greedy = token.getRepeatGreedy(); + int ret = qtfr.setQuantifier(target, group, env, bytes, getBegin(), getEnd()); + Node qn = qtfr; + + if (token.getRepeatPossessive()) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + en.setTarget(qn); + qn = en; + } + + if (ret == 0 || (syntax.op3OptionECMAScript() && ret == 1)) { + target = qn; + } else if (ret == 2) { /* split case: /abc+/ */ + target = ListNode.newList(target, null); + ListNode tmp = ListNode.newList(qn, null); + ((ListNode)target).setTail(tmp); + + fetchToken(); + return parseExpRepeatForCar(target, tmp, group); + } + fetchToken(); // goto re_entry + } + return target; + } + + private Node parseExpRepeatForCar(Node top, ListNode target, boolean group) { + while (token.type == TokenType.OP_REPEAT || token.type == TokenType.INTERVAL) { // repeat: + if (isInvalidQuantifier(target.value)) newSyntaxException(TARGET_OF_REPEAT_OPERATOR_INVALID); + + QuantifierNode qtfr = new QuantifierNode(token.getRepeatLower(), + token.getRepeatUpper(), + token.type == TokenType.INTERVAL); + + qtfr.greedy = token.getRepeatGreedy(); + int ret = qtfr.setQuantifier(target.value, group, env, bytes, getBegin(), getEnd()); + Node qn = qtfr; + + if (token.getRepeatPossessive()) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + en.setTarget(qn); + qn = en; + } + + if (ret == 0) { + target.setValue(qn); + } else if (ret == 2) { /* split case: /abc+/ */ + assert false; + } + fetchToken(); // goto re_entry + } + return top; + } + + private boolean isInvalidQuantifier(Node node) { + if (Config.USE_NO_INVALID_QUANTIFIER) return false; + + ListNode consAlt; + switch(node.getType()) { + case NodeType.ANCHOR: + return true; + + case NodeType.ENCLOSE: + /* allow enclosed elements */ + /* return is_invalid_quantifier_target(NENCLOSE(node)->target); */ + break; + + case NodeType.LIST: + consAlt = (ListNode)node; + do { + if (!isInvalidQuantifier(consAlt.value)) return false; + } while ((consAlt = consAlt.tail) != null); + return false; + + case NodeType.ALT: + consAlt = (ListNode)node; + do { + if (isInvalidQuantifier(consAlt.value)) return true; + } while ((consAlt = consAlt.tail) != null); + break; + + default: + break; + } + return false; + } + + private Node parseQuoteOpen() { + int[]endOp = new int[]{syntax.metaCharTable.esc, 'E'}; + int qstart = p; + Ptr nextChar = new Ptr(); + int qend = findStrPosition(endOp, endOp.length, qstart, stop, nextChar); + if (qend == -1) nextChar.p = qend = stop; + Node node = new StringNode(bytes, qstart, qend); + p = nextChar.p; + return node; + } + + private Node parseCharType(Node node) { + switch(token.getPropCType()) { + case CharacterType.WORD: + node = new CTypeNode(token.getPropCType(), token.getPropNot(), isAsciiRange(env.option)); + break; + + case CharacterType.SPACE: + case CharacterType.DIGIT: + case CharacterType.XDIGIT: + CClassNode ccn = new CClassNode(); + ccn.addCType(token.getPropCType(), false, isAsciiRange(env.option), env, this); + if (token.getPropNot()) ccn.setNot(); + node = ccn; + break; + + default: + newInternalException(PARSER_BUG); + } // inner switch + return node; + } + + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc); + enc.applyAllCaseFold(env.caseFoldFlag, ApplyCaseFold.INSTANCE, arg); + if (arg.altRoot != null) { + node = ListNode.newAlt(node, arg.altRoot); + } + return node; + } + + private Node parseCharProperty() { + int ctype = fetchCharPropertyToCType(); + CClassNode cc = new CClassNode(); + Node node = cc; + cc.addCType(ctype, false, false, env, this); + if (token.getPropNot()) cc.setNot(); + + if (isIgnoreCase(env.option)) { + if (ctype != CharacterType.ASCII) { + node = cClassCaseFold(node, cc, cc); + } + } + return node; + } + + private Node parseAnycharAnytime() { + Node node = new AnyCharNode(); + QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false); + qn.setTarget(node); + return qn; + } + + private Node parseBackref() { + final Node node; + if (syntax.op3OptionECMAScript() && token.getBackrefNum() == 1 && env.memNodes != null) { + EncloseNode encloseNode = env.memNodes[token.getBackrefRef1()]; + boolean shouldIgnore = false; + if (encloseNode != null && encloseNode.containingAnchor != null) { + shouldIgnore = true; + for (Node anchorNode : env.precReadNotNodes) { + if (anchorNode == encloseNode.containingAnchor) { + shouldIgnore = false; + break; + } + } + } + if (shouldIgnore) { + node = StringNode.EMPTY; + } else { + node = newBackRef(new int[]{token.getBackrefRef1()}); + } + } else { + node = newBackRef(token.getBackrefNum() > 1 ? token.getBackrefRefs() : new int[]{token.getBackrefRef1()}); + } + return node; + } + + private BackRefNode newBackRef(int[]backRefs) { + return new BackRefNode(token.getBackrefNum(), + backRefs, + token.getBackrefByName(), + token.getBackrefExistLevel(), + token.getBackrefLevel(), + env); + } + + private Node parseCall() { + int gNum = token.getCallGNum(); + if (gNum < 0 || token.getCallRel()) { + if (gNum > 0) gNum--; + gNum = backrefRelToAbs(gNum); + if (gNum <= 0) newValueException(INVALID_BACKREF); + } + Node node = new CallNode(bytes, token.getCallNameP(), token.getCallNameEnd(), gNum); + env.numCall++; + return node; + } + + private Node parseBranch(TokenType term) { + Node node = parseExp(term); + + if (token.type == TokenType.EOT || token.type == term || token.type == TokenType.ALT) { + return node; + } else { + ListNode top = ListNode.newList(node, null); + ListNode t = top; + + while (token.type != TokenType.EOT && token.type != term && token.type != TokenType.ALT) { + node = parseExp(term); + if (node.getType() == NodeType.LIST) { + t.setTail((ListNode)node); + while (((ListNode)node).tail != null ) node = ((ListNode)node).tail; + + t = ((ListNode)node); + } else { + t.setTail(ListNode.newList(node, null)); + t = t.tail; + } + } + return top; + } + } + + /* term_tok: TK_EOT or TK_SUBEXP_CLOSE */ + private Node parseSubExp(TokenType term) { + Node node = parseBranch(term); + + if (token.type == term) { + return node; + } else if (token.type == TokenType.ALT) { + ListNode top = ListNode.newAlt(node, null); + ListNode t = top; + while (token.type == TokenType.ALT) { + fetchToken(); + node = parseBranch(term); + + t.setTail(ListNode.newAlt(node, null)); + t = t.tail; + } + + if (token.type != term) parseSubExpError(term); + return top; + } else { + parseSubExpError(term); + return null; //not reached + } + } + + private void parseSubExpError(TokenType term) { + if (term == TokenType.SUBEXP_CLOSE) { + newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + } else { + newInternalException(PARSER_BUG); + } + } + + protected final Node parseRegexp() { + fetchToken(); + Node top = parseSubExp(TokenType.EOT); + if (Config.USE_SUBEXP_CALL) { + if (env.numCall > 0) { + /* Capture the pattern itself. It is used for (?R), (?0) and \g<0>. */ + EncloseNode np = EncloseNode.newMemory(env.option, false); + np.regNum = 0; + np.setTarget(top); + if (env.memNodes == null) env.memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + env.memNodes[0] = np; + top = np; + } + } + return top; + } +} diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java new file mode 100644 index 000000000..46db17e51 --- /dev/null +++ b/third_party/joni/src/org/joni/Regex.java @@ -0,0 +1,521 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAt; +import static org.joni.Config.USE_SUNDAY_QUICK_SEARCH; +import static org.joni.Option.isCaptureGroup; +import static org.joni.Option.isDontCaptureGroup; + +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.Iterator; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.Encoding; +import org.jcodings.EncodingDB; +import org.jcodings.specific.ASCIIEncoding; +import org.jcodings.specific.UTF8Encoding; +import org.jcodings.util.BytesHash; +import org.joni.constants.internal.AnchorType; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.ValueException; + +public final class Regex { + int[] code; /* compiled pattern */ + int codeLength; + boolean requireStack; + + int numMem; /* used memory(...) num counted from 1 */ + int numRepeat; /* OP_REPEAT/OP_REPEAT_NG id-counter */ + int numNullCheck; /* OP_NULL_CHECK_START/END id counter */ + int numCombExpCheck; /* combination explosion check */ + int numCall; /* number of subexp call */ + int captureHistory; /* (?@...) flag (1-31) */ + int btMemStart; /* need backtrack flag */ + int btMemEnd; /* need backtrack flag */ + + int stackPopLevel; + + int[]repeatRangeLo; + int[]repeatRangeHi; + + MatcherFactory factory; + + final Encoding enc; + int options; + int userOptions; + Object userObject; + final int caseFoldFlag; + + private BytesHash nameTable; // named entries + + /* optimization info (string search, char-map and anchors) */ + Search.Forward forward; /* optimize flag */ + Search.Backward backward; + int thresholdLength; /* search str-length for apply optimize */ + int anchor; /* BEGIN_BUF, BEGIN_POS, (SEMI_)END_BUF */ + int anchorDmin; /* (SEMI_)END_BUF anchor distance */ + int anchorDmax; /* (SEMI_)END_BUF anchor distance */ + int subAnchor; /* start-anchor for exact or map */ + + byte[]exact; + int exactP; + int exactEnd; + + byte[]map; /* used as BM skip or char-map */ + int[]intMap; /* BM skip for exact_len > 255 */ + int[]intMapBackward; /* BM skip for backward search */ + int dMin; /* min-distance of exact or map */ + int dMax; /* max-distance of exact or map */ + + byte[][]templates; /* fixed pattern strings not embedded in bytecode */ + int templateNum; + + private static final Encoding DEFAULT_ENCODING; + static { + Encoding defaultEncoding; + EncodingDB.Entry entry = EncodingDB.getEncodings().get(Charset.defaultCharset().name().getBytes()); + if (entry == null) { + defaultEncoding = UTF8Encoding.INSTANCE; + } else { + defaultEncoding = entry.getEncoding(); + } + DEFAULT_ENCODING = defaultEncoding; + } + + public Regex(CharSequence cs) { + this(cs.toString()); + } + + public Regex(CharSequence cs, Encoding enc) { + this(cs.toString(), enc); + } + + public Regex(String str) { + this(str.getBytes(), 0, str.length(), 0, UTF8Encoding.INSTANCE); + } + + public Regex(String str, Encoding enc) { + this(str.getBytes(), 0, str.length(), 0, enc); + } + + public Regex(byte[] bytes) { + this(bytes, 0, bytes.length, 0, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[] bytes, int p, int end) { + this(bytes, p, end, 0, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[] bytes, int p, int end, int option) { + this(bytes, p, end, option, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[]bytes, int p, int end, int option, Encoding enc) { + this(bytes, p, end, option, enc, Syntax.RUBY, WarnCallback.DEFAULT); + } + + // onig_new + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, Syntax syntax) { + this(bytes, p, end, option, Config.ENC_CASE_FOLD_DEFAULT, enc, syntax, WarnCallback.DEFAULT); + } + + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, WarnCallback warnings) { + this(bytes, p, end, option, enc, Syntax.RUBY, warnings); + } + + // onig_new + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, Syntax syntax, WarnCallback warnings) { + this(bytes, p, end, option, Config.ENC_CASE_FOLD_DEFAULT, enc, syntax, warnings); + } + + // onig_alloc_init + public Regex(byte[]bytes, int p, int end, int option, int caseFoldFlag, Encoding enc, Syntax syntax, WarnCallback warnings) { + if (Config.REGEX_MAX_LENGTH > 0 && (end - p) > Config.REGEX_MAX_LENGTH) { + throw new ValueException(ErrorMessages.REGEX_TOO_LONG); + } + + if ((option & (Option.DONT_CAPTURE_GROUP | Option.CAPTURE_GROUP)) == + (Option.DONT_CAPTURE_GROUP | Option.CAPTURE_GROUP)) { + throw new ValueException(ErrorMessages.INVALID_COMBINATION_OF_OPTIONS); + } + + if ((option & Option.NEGATE_SINGLELINE) != 0) { + option |= syntax.options; + option &= ~Option.SINGLELINE; + } else { + option |= syntax.options; + } + + this.enc = enc; + this.options = option; + this.caseFoldFlag = caseFoldFlag; + new Analyser(this, syntax, bytes, p, end, warnings).compile(); + } + + public Matcher matcher(byte[]bytes) { + return matcher(bytes, 0, bytes.length); + } + + public Matcher matcherNoRegion(byte[]bytes) { + return matcherNoRegion(bytes, 0, bytes.length); + } + + public Matcher matcher(byte[]bytes, int p, int end) { + return factory.create(this, numMem == 0 ? null : Region.newRegion(numMem + 1), bytes, p, end); + } + + public Matcher matcher(byte[]bytes, int p, int end, long timeout) { + return factory.create(this, numMem == 0 ? null : Region.newRegion(numMem + 1), bytes, p, end, timeout); + } + + public Matcher matcherNoRegion(byte[]bytes, int p, int end) { + return factory.create(this, null, bytes, p, end); + } + + public Matcher matcherNoRegion(byte[]bytes, int p, int end, long timeout) { + return factory.create(this, null, bytes, p, end, timeout); + } + + public int numberOfCaptures() { + return numMem; + } + + public int numberOfCaptureHistories() { + if (Config.USE_CAPTURE_HISTORY) { + int n = 0; + for (int i=0; i<=Config.MAX_CAPTURE_HISTORY_GROUP; i++) { + if (bsAt(captureHistory, i)) n++; + } + return n; + } else { + return 0; + } + } + + private NameEntry nameFind(byte[]name, int nameP, int nameEnd) { + if (nameTable != null) return nameTable.get(name, nameP, nameEnd); + return null; + } + + void renumberNameTable(int[]map) { + if (nameTable != null) { + for (NameEntry e : nameTable) { + if (e.backNum > 1) { + for (int i=0; i(); // 13, oni defaults to 5 + } else { + e = nameFind(name, nameP, nameEnd); + } + + if (e == null) { + // dup the name here as oni does ?, what for ? (it has to manage it, we don't) + e = new NameEntry(name, nameP, nameEnd); + nameTable.putDirect(name, nameP, nameEnd, e); + } else if (e.backNum >= 1 && !syntax.allowMultiplexDefinitionName()) { + throw new ValueException(ErrorMessages.MULTIPLEX_DEFINED_NAME, new String(name, nameP, nameEnd - nameP)); + } + + e.addBackref(backRef); + } + + NameEntry nameToGroupNumbers(byte[]name, int nameP, int nameEnd) { + return nameFind(name, nameP, nameEnd); + } + + public int nameToBackrefNumber(byte[]name, int nameP, int nameEnd, Region region) { + return nameToBackrefNumber(name, nameP, nameEnd, DEFAULT_ENCODING, region); + } + + public int nameToBackrefNumber(byte[]name, int nameP, int nameEnd, Encoding nameEncoding, Region region) { + NameEntry e = nameToGroupNumbers(name, nameP, nameEnd); + if (e == null) throw new ValueException(ErrorMessages.UNDEFINED_NAME_REFERENCE, + new String(name, nameP, nameEnd - nameP, nameEncoding.getCharset())); + + switch(e.backNum) { + case 0: + throw new InternalException(ErrorMessages.PARSER_BUG); + case 1: + return e.backRef1; + default: + if (region != null) { + for (int i = e.backNum - 1; i >= 0; i--) { + if (region.getBeg(e.backRefs[i]) != Region.REGION_NOTPOS) return e.backRefs[i]; + } + } + return e.backRefs[e.backNum - 1]; + } + } + + String nameTableToString() { + StringBuilder sb = new StringBuilder(); + + if (nameTable != null) { + sb.append("name table\n"); + for (NameEntry ne : nameTable) { + sb.append(" ").append(ne).append("\n"); + } + sb.append("\n"); + } + return sb.toString(); + } + + public Iterator namedBackrefIterator() { + return nameTable == null ? Collections.emptyIterator() : nameTable.iterator(); + } + + public int numberOfNames() { + return nameTable == null ? 0 : nameTable.size(); + } + + public boolean noNameGroupIsActive(Syntax syntax) { + if (isDontCaptureGroup(options)) return false; + + if (Config.USE_NAMED_GROUP) { + if (numberOfNames() > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(options)) return false; + } + return true; + } + + /* set skip map for Boyer-Moor search */ + boolean setupBMSkipMap(boolean ignoreCase) { + byte[]bytes = exact; + int s = exactP; + int end = exactEnd; + int len = end - s; + int clen; + CaseFoldCodeItem[]items = CaseFoldCodeItem.EMPTY_FOLD_CODES; + byte[]buf = new byte[Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM * Config.ENC_MBC_CASE_FOLD_MAXLEN]; + + final int ilen = USE_SUNDAY_QUICK_SEARCH ? len : len - 1; + if (Config.USE_BYTE_MAP || len < Config.CHAR_TABLE_SIZE) { + if (map == null) map = new byte[Config.CHAR_TABLE_SIZE]; // map/skip + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) map[i] = (byte)(USE_SUNDAY_QUICK_SEARCH ? len + 1 : len); + + for (int i = 0; i < ilen; i += clen) { + if (ignoreCase) items = enc.caseFoldCodesByString(caseFoldFlag, bytes, s + i, end); + clen = setupBMSkipMapCheck(bytes, s + i, end, items, buf); + if (clen == 0) return true; + + for (int j = 0; j < clen; j++) { + map[bytes[s + i + j] & 0xff] = (byte)(ilen - i - j); + for (int k = 0; k < items.length; k++) { + map[buf[k * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM + j] & 0xff] = (byte)(ilen - i - j); + } + } + } + } else { + if (intMap == null) intMap = new int[Config.CHAR_TABLE_SIZE]; + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) intMap[i] = (USE_SUNDAY_QUICK_SEARCH ? len + 1 : len); + + for (int i = 0; i < ilen; i += clen) { + if (ignoreCase) items = enc.caseFoldCodesByString(caseFoldFlag, bytes, s + i, end); + clen = setupBMSkipMapCheck(bytes, s + i, end, items, buf); + if (clen == 0) return true; + + for (int j = 0; j < clen; j++) { + intMap[bytes[s + i + j] & 0xff] = ilen - i - j; + for (int k = 0; k < items.length; k++) { + intMap[buf[k * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM + j] & 0xff] = ilen - i - j; + } + } + } + } + return false; + } + + private int setupBMSkipMapCheck(byte[]bytes, int p, int end, CaseFoldCodeItem[]items, byte[]buf) { + int clen = enc.length(bytes, p, end); + if (p + clen > end) clen = end - p; + for (int j = 0; j < items.length; j++) { + if (items[j].code.length != 1 || items[j].byteLen != clen) return 0; + int flen = enc.codeToMbc(items[j].code[0], buf, j * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM); + if (flen != clen) return 0; + } + return clen; + } + + void setOptimizeExactInfo(OptExactInfo e) { + if (e.length == 0) return; + + // shall we copy that ? + exact = e.bytes; + exactP = 0; + exactEnd = e.length; + boolean allowReverse = enc.isReverseMatchAllowed(exact, exactP, exactEnd); + + if (e.ignoreCase > 0) { + if (e.length >= 3 || (e.length >= 2 && allowReverse)) { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + if (!setupBMSkipMap(true)) { + forward = allowReverse ? (enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD) : Search.BM_NOT_REV_IC_FORWARD; + // FIXME: put above line in place to work around some failures. Either BM_IC_FORWARD is broken here or we are choosing it when we shouldn't. + //forward = allowReverse ? Search.BM_IC_FORWARD : Search.BM_NOT_REV_IC_FORWARD; + } else { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + } + } else { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + } + backward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_BACKWARD : Search.SLOW_IC_BACKWARD; + } else { + if (e.length >= 3 || (e.length >= 2 && allowReverse)) { + if (!setupBMSkipMap(false)) { + forward = allowReverse ? Search.BM_FORWARD : Search.BM_NOT_REV_FORWARD; + } else { + forward = enc.isSingleByte() ? Search.SLOW_SB_FORWARD : Search.SLOW_FORWARD; + } + } else { + forward = enc.isSingleByte() ? Search.SLOW_SB_FORWARD : Search.SLOW_FORWARD; + } + backward = enc.isSingleByte() ? Search.SLOW_SB_BACKWARD : Search.SLOW_BACKWARD; + } + + dMin = e.mmd.min; + dMax = e.mmd.max; + + if (dMin != MinMaxLen.INFINITE_DISTANCE) { + thresholdLength = dMin + (exactEnd - exactP); + } + } + + void setOptimizeMapInfo(OptMapInfo m) { + map = m.map; + + if (enc.isSingleByte()) { + forward = Search.MAP_SB_FORWARD; + backward = Search.MAP_SB_BACKWARD; + } else { + forward = Search.MAP_FORWARD; + backward = Search.MAP_BACKWARD; + } + + dMin = m.mmd.min; + dMax = m.mmd.max; + + if (dMin != MinMaxLen.INFINITE_DISTANCE) { + thresholdLength = dMin + 1; + } + } + + void setSubAnchor(OptAnchorInfo anc) { + subAnchor |= anc.leftAnchor & AnchorType.BEGIN_LINE; + subAnchor |= anc.rightAnchor & AnchorType.END_LINE; + } + + void clearOptimizeInfo() { + forward = null; + backward = null; + anchor = 0; + anchorDmax = 0; + anchorDmin = 0; + subAnchor = 0; + + exact = null; + exactP = exactEnd = 0; + } + + public String optimizeInfoToString() { + String s = ""; + s += "optimize: " + (forward != null ? forward.getName() : "NONE") + "\n"; + s += " anchor: " + OptAnchorInfo.anchorToString(anchor); + + if ((anchor & AnchorType.END_BUF_MASK) != 0) { + s += MinMaxLen.distanceRangeToString(anchorDmin, anchorDmax); + } + + s += "\n"; + + if (forward != null) { + s += " sub anchor: " + OptAnchorInfo.anchorToString(subAnchor) + "\n"; + } + + s += "dmin: " + dMin + " dmax: " + dMax + "\n"; + s += "threshold length: " + thresholdLength + "\n"; + + if (exact != null) { + s += "exact: [" + new String(exact, exactP, exactEnd - exactP) + "]: length: " + (exactEnd - exactP) + "\n"; + } else if (forward == Search.MAP_FORWARD || forward == Search.MAP_SB_FORWARD) { + int n=0; + for (int i=0; i 0) { + int c=0; + s += "["; + for (int i=0; i 0) s += ", "; + c++; + if (enc.maxLength() == 1 && enc.isPrint(i)) s += ((char)i); + else s += i; + } + } + s += "]\n"; + } + } + + return s; + } + + public Encoding getEncoding() { + return enc; + } + + public int getOptions() { + return options; + } + + public void setUserOptions(int options) { + this.userOptions = options; + } + + public int getUserOptions() { + return userOptions; + } + + public void setUserObject(Object object) { + this.userObject = object; + } + + public Object getUserObject() { + return userObject; + } + + public boolean isLinear() { + return !requireStack; + } +} diff --git a/third_party/joni/src/org/joni/Region.java b/third_party/joni/src/org/joni/Region.java new file mode 100644 index 000000000..4507c4517 --- /dev/null +++ b/third_party/joni/src/org/joni/Region.java @@ -0,0 +1,66 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public abstract class Region { + static final int REGION_NOTPOS = -1; + + protected CaptureTreeNode historyRoot; + + public static Region newRegion(int num) { + if (num == 1) return new SingleRegion(num); + return new MultiRegion(num); + } + + public static Region newRegion(int begin, int end) { + return new SingleRegion(begin, end); + } + + @Override + public abstract Region clone(); + + public abstract int getNumRegs(); + + public abstract int getBeg(int index); + + public abstract int setBeg(int index, int value); + + public abstract int getEnd(int index); + + public abstract int setEnd(int index, int value); + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Region: \n"); + for (int i=0; i= Config.MAX_CAPTURE_GROUP_NUM) throw new InternalException(ErrorMessages.TOO_MANY_CAPTURE_GROUPS); + if (numMem++ == 0) { + memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + } else if (numMem >= memNodes.length) { + EncloseNode[]tmp = new EncloseNode[memNodes.length << 1]; + System.arraycopy(memNodes, 0, tmp, 0, memNodes.length); + memNodes = tmp; + } + + return numMem; + } + + void setMemNode(int num, EncloseNode node) { + if (numMem >= num) { + memNodes[num] = node; + } else { + throw new InternalException(ErrorMessages.PARSER_BUG); + } + } + + + void pushPrecReadNotNode(Node node) { + numPrecReadNotNodes++; + if (precReadNotNodes == null) { + precReadNotNodes = new Node[Config.SCANENV_MEMNODES_SIZE]; + } else if (numPrecReadNotNodes >= precReadNotNodes.length) { + Node[]tmp = new Node[precReadNotNodes.length << 1]; + System.arraycopy(precReadNotNodes, 0, tmp, 0, precReadNotNodes.length); + precReadNotNodes = tmp; + } + precReadNotNodes[numPrecReadNotNodes - 1] = node; + } + + void popPrecReadNotNode(Node node) { + if (precReadNotNodes != null && precReadNotNodes[numPrecReadNotNodes - 1] == node) { + precReadNotNodes[numPrecReadNotNodes - 1] = null; + numPrecReadNotNodes--; + } + } + + Node currentPrecReadNotNode() { + if (numPrecReadNotNodes > 0) { + return precReadNotNodes[numPrecReadNotNodes - 1]; + } + return null; + } + + int convertBackslashValue(int c) { + if (syntax.opEscControlChars()) { + switch (c) { + case 'n': return '\n'; + case 't': return '\t'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'a': return '\007'; + case 'b': return '\010'; + case 'e': return '\033'; + case 'v': + if (syntax.op2EscVVtab()) return 11; // '\v' + break; + default: + if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')) unknownEscWarn(String.valueOf((char)c)); + } + } + return c; + } + + void ccEscWarn(String s) { + if (warnings != WarnCallback.NONE) { + if (syntax.warnCCOpNotEscaped() && syntax.backSlashEscapeInCC()) { + warnings.warn("character class has '" + s + "' without escape"); + } + } + } + + void unknownEscWarn(String s) { + if (warnings != WarnCallback.NONE) { + warnings.warn("Unknown escape \\" + s + " is ignored"); + } + } + + void closeBracketWithoutEscapeWarn(String s) { + if (warnings != WarnCallback.NONE) { + if (syntax.warnCCOpNotEscaped()) { + warnings.warn("regular expression has '" + s + "' without escape"); + } + } + } + + void ccDuplicateWarn() { + if (syntax.warnCCDup() && (warningsFlag & SyntaxProperties.WARN_CC_DUP) == 0) { + warningsFlag |= SyntaxProperties.WARN_CC_DUP; + // FIXME: #34 points out problem and what it will take to uncomment this (we were getting erroneous versions of this) + // warnings.warn("character class has duplicated range"); + } + } +} diff --git a/third_party/joni/src/org/joni/ScannerSupport.java b/third_party/joni/src/org/joni/ScannerSupport.java new file mode 100644 index 000000000..9b9857992 --- /dev/null +++ b/third_party/joni/src/org/joni/ScannerSupport.java @@ -0,0 +1,182 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.SyntaxException; +import org.joni.exception.ValueException; + +abstract class ScannerSupport extends IntHolder implements ErrorMessages { + protected final Encoding enc; // fast access to encoding + protected final byte[]bytes; // pattern + protected int p; // current scanner position + protected int stop; // pattern end (mutable) + private int lastFetched; // last fetched value for unfetch support + protected int c; // current code point + + private final int begin; // pattern begin position for reset() support + private final int end; // pattern end position for reset() support + protected int _p; // used by mark()/restore() to mark positions + + protected ScannerSupport(Encoding enc, byte[]bytes, int p, int end) { + this.enc = enc; + this.bytes = bytes; + this.begin = p; + this.end = end; + } + + protected final int getBegin() { + return begin; + } + + protected final int getEnd() { + return end; + } + + private static final int INT_SIGN_BIT = 1 << 31; + protected final int scanUnsignedNumber() { + int last = c; + int num = 0; // long ??? + while(left()) { + fetch(); + if (enc.isDigit(c)) { + int onum = num; + num = num * 10 + Encoding.digitVal(c); + if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; + } else { + unfetch(); + break; + } + } + c = last; + return num; + } + + protected final int scanUnsignedHexadecimalNumber(int minLength, int maxLength) { + int last = c; + int num = 0; + int restLen = maxLength - minLength; + while(left() && maxLength-- != 0) { + fetch(); + if (enc.isXDigit(c)) { + int val = enc.xdigitVal(c); + if ((Integer.MAX_VALUE - val) / 16 < num) return -1; + num = (num << 4) + val; + } else { + unfetch(); + maxLength++; + break; + } + } + if (maxLength > restLen) return -2; + c = last; + return num; + } + + protected final int scanUnsignedOctalNumber(int maxLength) { + int last = c; + int num = 0; + while(left() && maxLength-- != 0) { + fetch(); + if (enc.isDigit(c) && c < '8') { + int onum = num; + int val = Encoding.odigitVal(c); + num = (num << 3) + val; + if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; + } else { + unfetch(); + break; + } + } + c = last; + return num; + } + + protected final void reset() { + p = begin; + stop = end; + } + + protected final void mark() { + _p = p; + } + + protected final void restore() { + p = _p; + } + + protected final void inc() { + lastFetched = p; + p += enc.length(bytes, p, stop); + } + + protected final void fetch() { + c = enc.mbcToCode(bytes, p, stop); + lastFetched = p; + p += enc.length(bytes, p, stop); + } + + protected int fetchTo() { + int to = enc.mbcToCode(bytes, p, stop); + lastFetched = p; + p += enc.length(bytes, p, stop); + return to; + } + + protected final void unfetch() { + p = lastFetched; + } + + protected final int peek() { + return p < stop ? enc.mbcToCode(bytes, p, stop) : 0; + } + + protected final boolean peekIs(int c) { + return peek() == c; + } + + protected final boolean left() { + return p < stop; + } + + protected void newSyntaxException(String message) { + throw new SyntaxException(message); + } + + protected void newValueException(String message) { + throw new ValueException(message); + } + + protected void newValueException(String message, String str) { + throw new ValueException(message, str); + } + + protected void newValueException(String message, int p, int end) { + throw new ValueException(message, new String(bytes, p, end - p)); + } + + protected void newInternalException(String message) { + throw new InternalException(message); + } + +} diff --git a/third_party/joni/src/org/joni/Search.java b/third_party/joni/src/org/joni/Search.java new file mode 100644 index 000000000..9616d5a80 --- /dev/null +++ b/third_party/joni/src/org/joni/Search.java @@ -0,0 +1,633 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.Config.USE_SUNDAY_QUICK_SEARCH; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; + +final class Search { + + static abstract class Forward { + abstract String getName(); + abstract int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange); + } + + static abstract class Backward { + abstract int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_); + } + + private static boolean lowerCaseMatch(byte[] t, int tP, int tEnd, byte[] bytes, int p, int end, Encoding enc, byte[] buf, int caseFoldFlag) { + final IntHolder holder = new IntHolder(); + holder.value = p; + while (tP < tEnd) { + int lowlen = enc.mbcCaseFold(caseFoldFlag, bytes, holder, end, buf); + if (lowlen == 1) { + if (t[tP++] != buf[0]) + return false; + } else { + int q = 0; + while (lowlen > 0) { + if (t[tP++] != buf[q++]) + return false; + lowlen--; + } + } + } + return true; + } + + static final Forward SLOW_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + + if (t == targetEnd) return s; + } + s += enc.length(text, s, textEnd); + } + return -1; + } + }; + + static final Backward SLOW_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + s = (s > textStart) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + + while (s >= textP) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + if (t == targetEnd) return s; + } + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward SLOW_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + + if (t == targetEnd) return s; + } + s++; + } + return -1; + } + }; + + static final Backward SLOW_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + if (s > textStart) s = textStart; + + while (s >= textP) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + if (t == targetEnd) return s; + } + //s = s <= adjustText ? -1 : s - 1; + s--; + } + return -1; + } + }; + + static final Forward SLOW_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + byte[]buf = matcher.icbuf(); + + while (s < end) { + if (lowerCaseMatch(target, targetP, targetEnd, text, s, textEnd, enc, buf, regex.caseFoldFlag)) return s; + s += enc.length(text, s, textEnd); + } + return -1; + } + + }; + + static final Backward SLOW_IC_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + s = (s > textStart) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + byte[]buf = matcher.icbuf(); + + while (s >= textP) { + if (lowerCaseMatch(target, targetP, targetEnd, text, s, textEnd, enc, buf, regex.caseFoldFlag)) return s; + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward SLOW_IC_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_IC_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + final byte[]toLowerTable = regex.enc.toLowerCaseTable(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (target[targetP] == toLowerTable[text[s] & 0xff]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != toLowerTable[text[p++] & 0xff]) break; + t++; + } + + if (t == targetEnd) return s; + } + s++; + } + return -1; + } + + }; + + static final Backward SLOW_IC_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + final byte[]toLowerTable = regex.enc.toLowerCaseTable(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + + if (s > textStart) s = textStart; + + while (s >= textP) { + if (target[targetP] == toLowerTable[text[s] & 0xff]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != toLowerTable[text[p++] & 0xff]) break; + t++; + } + if (t == targetEnd) return s; + } + //s = s <= adjustText ? -1 : s - 1; + s--; + } + return -1; + } + }; + + static final Forward BM_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end, s; + int tail = targetEnd - 1; + if (USE_SUNDAY_QUICK_SEARCH) { + int tlen1 = tail - targetP; + end = textRange + tlen1; + s = textP + tlen1; + } else { + end = textRange + (targetEnd - targetP) - 1; + s = textP + (targetEnd - targetP) - 1; + } + if (end > textEnd) end = textEnd; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int p = s; + int t = tail; + + while (text[p] == target[t]) { + if (t == targetP) return p; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.map[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } else { /* see int_map[] */ + while (s < end) { + int p = s; + int t = tail; + + while (text[p] == target[t]) { + if (t == targetP) return p; + p--; t--; + } + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } + return -1; + } + }; + + static final Backward BM_BACKWARD = new Backward() { + private static final int BM_BACKWARD_SEARCH_LENGTH_THRESHOLD = 100; + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + if (Config.USE_INT_MAP_BACKWARD) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + if (regex.intMapBackward == null) { + if (s_ - range_ < BM_BACKWARD_SEARCH_LENGTH_THRESHOLD) { + return SLOW_BACKWARD.search(matcher, text, textP, adjustText, textEnd, textStart, s_, range_); // goto exact_method; + } + setBmBackwardSkip(regex, target, targetP, targetEnd); + } + + int s = textEnd - (targetEnd - targetP); + s = (textStart < s) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + + while (s >= textP) { + int p = s; + int t = targetP; + while (t < targetEnd && text[p] == target[t]) { + p++; t++; + } + if (t == targetEnd) return s; + + s -= regex.intMapBackward[text[s] & 0xff]; + s = enc.leftAdjustCharHead(text, adjustText, s, textEnd); + } + return -1; + } else { + return SLOW_BACKWARD.search(matcher, text, textP, adjustText, textEnd, textStart, s_, range_); // goto exact_method; + } + } + + private void setBmBackwardSkip(Regex regex, byte[]bytes, int p, int end) { + final int[] skip; + if (regex.intMapBackward == null) { + regex.intMapBackward = skip = new int[Config.CHAR_TABLE_SIZE]; + } else { + skip = regex.intMapBackward; + } + + int len = end - p; + + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) skip[i] = len; + for (int i = len - 1; i > 0; i--) skip[bytes[i] & 0xff] = i; + } + }; + + static final Forward BM_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]buf = matcher.icbuf(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end, s; + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + if (USE_SUNDAY_QUICK_SEARCH) { + end = textRange + tlen1; + s = textP + tlen1; + } else { + end = textRange + (targetEnd - targetP) - 1; + s = textP + (targetEnd - targetP) - 1; + } + if (end > textEnd) end = textEnd; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int p = USE_SUNDAY_QUICK_SEARCH ? s - tlen1 : s - (targetEnd - targetP) + 1; + if (lowerCaseMatch(target, targetP, targetEnd, text, p, s + 1, enc, buf, regex.caseFoldFlag)) return p; + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.map[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } else { /* see int_map[] */ + while (s < end) { + int p = USE_SUNDAY_QUICK_SEARCH ? s - tlen1 : s - (targetEnd - targetP) + 1; + if (lowerCaseMatch(target, targetP, targetEnd, text, p, s + 1, enc, buf, regex.caseFoldFlag)) return p; + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } + return -1; + } + }; + + static final Forward BM_NOT_REV_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_NOT_REV_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + int end = textRange; + + if (end + tlen1 > textEnd) end = textEnd - tlen1; + int s = textP, p, se; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + p = se = s + tlen1; + int t = tail; + while (text[p] == target[t]) { + if (t == targetP) return s; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.map[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } else { + while (s < end) { + p = se = s + tlen1; + int t = tail; + while (text[p] == target[t]) { + if (t == targetP) return s; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + + } + } + return -1; + } + }; + + static final Forward BM_NOT_REV_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_NOT_REV_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]buf = matcher.icbuf(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + int end = textRange; + + if (end + tlen1 > textEnd) end = textEnd - tlen1; + int s = textP; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int se = s + tlen1; + if (lowerCaseMatch(target, targetP, targetEnd, text, s, se + 1, enc, buf, regex.caseFoldFlag)) return s; + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.map[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + int t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } else { + while (s < end) { + int se = s + tlen1; + if (lowerCaseMatch(target, targetP, targetEnd, text, s, se + 1, enc, buf, regex.caseFoldFlag)) return s; + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + int t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } + return -1; + } + }; + + static final Forward MAP_FORWARD = new Forward() { + @Override + final String getName() { + return "MAP_FORWARD"; + } + + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]map = regex.map; + int s = textP; + + while (s < textRange) { + if (map[text[s] & 0xff] != 0) return s; + s += enc.length(text, s, textEnd); + } + return -1; + } + }; + + static final Backward MAP_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]map = regex.map; + int s = textStart; + + if (s >= textEnd) s = textEnd - 1; // multibyte safe ? + while (s >= textP) { + if (map[text[s] & 0xff] != 0) return s; + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward MAP_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "MAP_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]map = regex.map; + int s = textP; + + while (s < textRange) { + if (map[text[s] & 0xff] != 0) return s; + s++; + } + return -1; + } + }; + + static final Backward MAP_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + byte[]map = regex.map; + int s = textStart; + + if (s >= textEnd) s = textEnd - 1; + while (s >= textP) { + if (map[text[s] & 0xff] != 0) return s; + s--; + } + return -1; + } + }; +} diff --git a/third_party/joni/src/org/joni/SingleRegion.java b/third_party/joni/src/org/joni/SingleRegion.java new file mode 100644 index 000000000..3064264cc --- /dev/null +++ b/third_party/joni/src/org/joni/SingleRegion.java @@ -0,0 +1,75 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public class SingleRegion extends Region { + int beg; + int end; + + public SingleRegion(int num) { + if (num != 1) throw new IndexOutOfBoundsException(""+num); + } + + public SingleRegion(int begin, int end) { + this.beg = begin; + this.end = end; + } + + @Override + public int getNumRegs() { + return 1; + } + + @Override + public SingleRegion clone() { + SingleRegion region = new SingleRegion(beg, end); + if (getCaptureTree() != null) region.setCaptureTree(getCaptureTree().cloneTree()); + return region; + } + + @Override + public int getBeg(int index) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return beg; + } + + @Override + public int setBeg(int index, int value) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return beg = value; + } + + @Override + public int getEnd(int index) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return end; + } + + @Override + public int setEnd(int index, int value) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return end = value; + } + + @Override + void clear() { + beg = end = REGION_NOTPOS; + } +} diff --git a/third_party/joni/src/org/joni/StackEntry.java b/third_party/joni/src/org/joni/StackEntry.java new file mode 100644 index 000000000..2239e3260 --- /dev/null +++ b/third_party/joni/src/org/joni/StackEntry.java @@ -0,0 +1,202 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +class StackEntry { + int type; + private int E1, E2, E3, E4; + private Object calloutToken; + + // first union member + /* byte code position */ + void setStatePCode(int pcode) { + E1 = pcode; + } + int getStatePCode() { + return E1; + } + /* string position */ + void setStatePStr(int pstr) { + E2 = pstr; + } + int getStatePStr() { + return E2; + } + /* previous char position of pstr */ + void setStatePStrPrev(int pstrPrev) { + E3 = pstrPrev; + } + int getStatePStrPrev() { + return E3; + } + + void setPKeep(int pkeep) { + E4 = pkeep; + } + int getPKeep() { + return E4; + } + + // second union member + /* for OP_REPEAT_INC, OP_REPEAT_INC_NG */ + void setRepeatCount(int count) { + E1 = count; + } + int getRepeatCount() { + return E1; + } + void decreaseRepeatCount() { + E1--; + } + void increaseRepeatCount() { + E1++; + } + /* byte code position (head of repeated target) */ + void setRepeatPCode(int pcode) { + E2 = pcode; + } + int getRepeatPCode() { + return E2; + } + /* repeat id */ + void setRepeatNum(int num) { + E3 = num; + } + int getRepeatNum() { + return E3; + } + + // third union member + /* index of stack */ /*int repeat_inc struct*/ + void setSi(int si) { + E1 = si; + } + int getSi() { + return E1; + } + + // fourth union member + /* memory num */ + void setMemNum(int num) { + E1 = num; + } + int getMemNum() { + return E1; + } + /* start/end position */ + void setMemPstr(int pstr) { + E2 = pstr; + } + int getMemPStr() { + return E2; + } + + /* Following information is set, if this stack type is MEM-START */ + /* prev. info (for backtrack "(...)*" ) */ + void setMemStart(int start) { + E3 = start; + } + int getMemStart() { + return E3; + } + /* prev. info (for backtrack "(...)*" ) */ + void setMemEnd(int end) { + E4 = end; + } + int getMemEnd() { + return E4; + } + + // fifth union member + /* null check id */ + void setNullCheckNum(int num) { + E1 = num; + } + int getNullCheckNum() { + return E1; + } + /* start position */ + void setNullCheckPStr(int pstr) { + E2 = pstr; + } + int getNullCheckPStr() { + return E2; + } + + // sixth union member + /* byte code position */ + void setCallFrameRetAddr(int addr) { + E1 = addr; + } + int getCallFrameRetAddr() { + return E1; + } + /* null check id */ + void setCallFrameNum(int num) { + E2 = num; + } + int getCallFrameNum() { + return E2; + } + /* string position */ + void setCallFramePStr(int pstr) { + E3 = pstr; + } + int getCallFramePStr() { + return E3; + } + + /* absent position */ + void setAbsentStr(int pos) { + E1 = pos; + } + int getAbsentStr() { + return E1; + } + + void setAbsentEndStr(int pos) { + E2 = pos; + } + int getAbsentEndStr() { + return E2; + } + + void setCalloutToken(Object token) { + calloutToken = token; + } + + Object takeCalloutToken() { + Object token = calloutToken; + calloutToken = null; + return token; + } +} + +final class SCStackEntry extends StackEntry { + private int E5; + + void setStateCheck(int check) { + E5 = check; + } + + int getStateCheck() { + return E5; + } +} diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java new file mode 100644 index 000000000..18a9f7657 --- /dev/null +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -0,0 +1,635 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAt; +import static org.joni.Config.USE_CEC; + +import java.lang.ref.WeakReference; +import java.util.Arrays; + +import org.joni.constants.internal.StackPopLevel; +import org.joni.constants.internal.StackType; + +abstract class StackMachine extends Matcher implements StackType { + protected static final int INVALID_INDEX = -1; + + protected StackEntry[]stack; + protected int stk; // stkEnd + protected final int[]repeatStk; + protected final int memStartStk, memEndStk; + protected byte[] stateCheckBuff; // CEC, move to int[] ? + protected int stateCheckBuffSize; + + protected StackMachine(Regex regex, Region region, byte[]bytes, int p , int end) { + super(regex, region, bytes, p, end); + stack = regex.requireStack ? fetchStack() : null; + final int n; + if (Config.USE_SUBEXP_CALL) { + n = regex.numRepeat + ((regex.numMem + 1) << 1); + memStartStk = regex.numRepeat; + memEndStk = memStartStk + regex.numMem + 1; + } else { + n = regex.numRepeat + (regex.numMem << 1); + memStartStk = regex.numRepeat - 1; + memEndStk = memStartStk + regex.numMem; + /* for index start from 1, mem_start_stk[1]..mem_start_stk[num_mem] */ + /* for index start from 1, mem_end_stk[1]..mem_end_stk[num_mem] */ + } + repeatStk = n > 0 ? new int[n] : null; + } + + protected final void stackInit() { + if (stack != null) pushEnsured(ALT, regex.codeLength - 1); /* bottom stack */ + if (repeatStk != null) { + for (int i = (Config.USE_SUBEXP_CALL ? 0 : 1); i <= regex.numMem; i++) { + repeatStk[i + memStartStk] = repeatStk[i + memEndStk] = INVALID_INDEX; + } + } + } + + private static StackEntry[] allocateStack() { + StackEntry[]stack = new StackEntry[Config.INIT_MATCH_STACK_SIZE]; + stack[0] = USE_CEC ? new SCStackEntry() : new StackEntry(); + return stack; + } + + private void doubleStack() { + StackEntry[] newStack = new StackEntry[stack.length << 1]; + System.arraycopy(stack, 0, newStack, 0, stack.length); + stack = newStack; + } + + static final ThreadLocal> stacks + = new ThreadLocal<>(); + + private static StackEntry[] fetchStack() { + WeakReference ref = stacks.get(); + StackEntry[] stack; + if (ref == null) { + stacks.set( new WeakReference<>(stack = allocateStack()) ); + } + else { + stack = ref.get(); + if (stack == null) { + stacks.set( new WeakReference<>(stack = allocateStack()) ); + } + } + return stack; + } + + private final StackEntry ensure1() { + if (stk >= stack.length) doubleStack(); + StackEntry e = stack[stk]; + if (e == null) stack[stk] = e = USE_CEC ? new SCStackEntry() : new StackEntry(); + return e; + } + + private final void pushType(int type) { + ensure1().type = type; + stk++; + } + + // CEC + + // STATE_CHECK_POS + private int stateCheckPos(int s, int snum) { + return (s - str) * regex.numCombExpCheck + (snum - 1); + } + + // STATE_CHECK_VAL + protected final boolean stateCheckVal(int s, int snum) { + if (stateCheckBuff != null) { + int x = stateCheckPos(s, snum); + return (stateCheckBuff[x / 8] & (1 << (x % 8))) != 0; + } + return false; + } + + // ELSE_IF_STATE_CHECK_MARK + private void stateCheckMark() { + StackEntry e = stack[stk]; + int x = stateCheckPos(e.getStatePStr(), ((SCStackEntry)e).getStateCheck()); + stateCheckBuff[x / 8] |= (1 << (x % 8)); + } + + // STATE_CHECK_BUFF_INIT + private static final int STATE_CHECK_BUFF_MALLOC_THRESHOLD_SIZE = 16; + @Override + protected final void stateCheckBuffInit(int strLength, int offset, int stateNum) { + if (stateNum > 0 && strLength >= Config.CHECK_STRING_THRESHOLD_LEN) { + int size = ((strLength + 1) * stateNum + 7) >>> 3; + offset = (offset * stateNum) >>> 3; + + if (size > 0 && offset < size && size < Config.CHECK_BUFF_MAX_SIZE) { + if (size >= STATE_CHECK_BUFF_MALLOC_THRESHOLD_SIZE) { + stateCheckBuff = new byte[size]; + } else { + // same impl, reduce... + stateCheckBuff = new byte[size]; + } + Arrays.fill(stateCheckBuff, offset, (size - offset), (byte)0); + stateCheckBuffSize = size; + } else { + stateCheckBuff = null; // reduce + stateCheckBuffSize = 0; + } + } else { + stateCheckBuff = null; // reduce + stateCheckBuffSize = 0; + } + } + + @Override + protected final void stateCheckBuffClear() { + stateCheckBuff = null; + stateCheckBuffSize = 0; + } + + private void push(int type, int pat, int s, int prev, int pkeep) { + StackEntry e = ensure1(); + e.type = type; + e.setStatePCode(pat); + e.setStatePStr(s); + e.setStatePStrPrev(prev); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(0); + e.setPKeep(pkeep); + stk++; + } + + private final void pushEnsured(int type, int pat) { + StackEntry e = stack[stk]; + e.type = type; + e.setStatePCode(pat); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(0); + stk++; + } + + protected final void pushAltWithStateCheck(int pat, int s, int sprev, int snum, int pkeep) { + StackEntry e = ensure1(); + e.type = ALT; + e.setStatePCode(pat); + e.setStatePStr(s); + e.setStatePStrPrev(sprev); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(stateCheckBuff != null ? snum : 0); + e.setPKeep(pkeep); + stk++; + } + + protected final void pushStateCheck(int s, int snum) { + if (stateCheckBuff != null) { + StackEntry e = ensure1(); + e.type = STATE_CHECK_MARK; + e.setStatePStr(s); + ((SCStackEntry)e).setStateCheck(snum); + stk++; + } + } + + protected final void pushAlt(int pat, int s, int prev, int pkeep) { + push(ALT, pat, s, prev, pkeep); + } + + protected final void pushPos(int s, int prev, int pkeep) { + push(POS, -1 /*NULL_UCHARP*/, s, prev, pkeep); + } + + protected final void pushPosNot(int pat, int s, int prev, int pkeep) { + push(POS_NOT, pat, s, prev, pkeep); + } + + protected final void pushStopBT() { + pushType(STOP_BT); + } + + protected final void pushLookBehindNot(int pat, int s, int sprev, int pkeep) { + push(LOOK_BEHIND_NOT, pat, s, sprev, pkeep); + } + + protected final void pushRepeat(int id, int pat) { + StackEntry e = ensure1(); + e.type = REPEAT; + e.setRepeatNum(id); + e.setRepeatPCode(pat); + e.setRepeatCount(0); + stk++; + } + + protected final void pushRepeatInc(int sindex) { + StackEntry e = ensure1(); + e.type = REPEAT_INC; + e.setSi(sindex); + stk++; + } + + protected final void pushMemStart(int mnum, int s) { + StackEntry e = ensure1(); + e.type = MEM_START; + e.setMemNum(mnum); + e.setMemPstr(s); + e.setMemStart(repeatStk[memStartStk + mnum]); + e.setMemEnd(repeatStk[memEndStk + mnum]); + repeatStk[memStartStk + mnum] = stk; + repeatStk[memEndStk + mnum] = INVALID_INDEX; + stk++; + } + + protected final void pushMemEnd(int mnum, int s) { + StackEntry e = ensure1(); + e.type = MEM_END; + e.setMemNum(mnum); + e.setMemPstr(s); + e.setMemStart(repeatStk[memStartStk + mnum]); + e.setMemEnd(repeatStk[memEndStk + mnum]); + repeatStk[memEndStk + mnum] = stk; + stk++; + } + + protected final void pushMemEndMark(int mnum) { + StackEntry e = ensure1(); + e.type = MEM_END_MARK; + e.setMemNum(mnum); + stk++; + } + + protected final int getMemStart(int mnum) { + int level = 0; + int stkp = stk; + + while (stkp > 0) { + stkp--; + StackEntry e = stack[stkp]; + if ((e.type & MASK_MEM_END_OR_MARK) != 0 && e.getMemNum() == mnum) { + level++; + } else if (e.type == MEM_START && e.getMemNum() == mnum) { + if (level == 0) break; + level--; + } + } + return stkp; + } + + protected final void pushNullCheckStart(int cnum, int s) { + StackEntry e = ensure1(); + e.type = NULL_CHECK_START; + e.setNullCheckNum(cnum); + e.setNullCheckPStr(s); + stk++; + } + + protected final void pushNullCheckEnd(int cnum) { + StackEntry e = ensure1(); + e.type = NULL_CHECK_END; + e.setNullCheckNum(cnum); + stk++; + } + + protected final void pushCallFrame(int pat) { + StackEntry e = ensure1(); + e.type = CALL_FRAME; + e.setCallFrameRetAddr(pat); + stk++; + } + + protected final void pushReturn() { + StackEntry e = ensure1(); + e.type = RETURN; + stk++; + } + + protected final void pushAbsent() { + StackEntry e = ensure1(); + e.type = ABSENT; + stk++; + } + + protected final void pushAbsentPos(int start, int end) { + StackEntry e = ensure1(); + e.type = ABSENT_POS; + e.setAbsentStr(start); + e.setAbsentEndStr(end); + stk++; + } + + protected final void pushCallout(Object token) { + StackEntry e = ensure1(); + e.type = CALLOUT; + e.setCalloutToken(token); + stk++; + } + + protected final void popOne() { + stk--; + } + + protected final StackEntry pop() { + switch (regex.stackPopLevel) { + case StackPopLevel.FREE: + return popFree(); + case StackPopLevel.MEM_START: + return popMemStart(); + default: + return popDefault(); + } + } + + private StackEntry popFree() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) { + return e; + } else if (e.type == CALLOUT) { + unwindCallout(e); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + } + + private StackEntry popMemStart() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) { + return e; + } else if (e.type == CALLOUT) { + unwindCallout(e); + } else if (e.type == MEM_START) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + } + + private void popRewrite(StackEntry e) { + if (e.type == CALLOUT) { + unwindCallout(e); + } else if (e.type == MEM_START) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (e.type == REPEAT_INC) { + stack[e.getSi()].decreaseRepeatCount(); + } else if (e.type == MEM_END) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + + protected final void unwindActiveCallouts() { + if (stack == null) return; + for (int i = stk - 1; i >= 0; i--) { + if (stack[i].type == CALLOUT) unwindCallout(stack[i]); + } + } + + private void unwindCallout(StackEntry entry) { + Object token = entry.takeCalloutToken(); + if (token != null) getCalloutHandler().unwind(token); + } + + private StackEntry popDefault() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) return e; else popRewrite(e); + } + } + + protected final void popTilPosNot() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == POS_NOT) break; else popRewrite(e); + } + } + + protected final void popTilLookBehindNot() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == LOOK_BEHIND_NOT) break; else popRewrite(e); + } + } + + protected final void popTilAbsent() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == ABSENT) break; else popRewrite(e); + } + } + + protected final int posEnd() { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + if ((e.type & MASK_TO_VOID_TARGET) != 0) { + e.type = VOID; + } else if (e.type == POS) { + e.type = VOID; + break; + } + } + return k; + } + + protected final void stopBtEnd() { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if ((e.type & MASK_TO_VOID_TARGET) != 0) { + e.type = VOID; + } else if (e.type == STOP_BT) { + e.type = VOID; + break; + } + } + } + + // int for consistency with other null check routines + protected final int nullCheck(int id, int s) { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + return e.getNullCheckPStr() == s ? 1 : 0; + } + } + } + } + + protected final int nullCheckRec(int id, int s) { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (level == 0) { + return e.getNullCheckPStr() == s ? 1 : 0; + } else { + level--; + } + } + } else if (e.type == NULL_CHECK_END) { + level++; + } + } + } + + protected final int nullCheckMemSt(int id, int s) { + int k = stk; + int isNull; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (e.getNullCheckPStr() != s) { + isNull = 0; + break; + } else { + int endp; + isNull = 1; + while (k < stk) { + e = stack[k++]; + if (e.type == MEM_START) { + if (e.getMemEnd() == INVALID_INDEX) { + isNull = 0; + break; + } + if (bsAt(regex.btMemEnd, e.getMemNum())) { + endp = stack[e.getMemEnd()].getMemPStr(); + } else { + endp = e.getMemEnd(); + } + if (stack[e.getMemStart()].getMemPStr() != endp) { + isNull = 0; + break; + } else if (endp != s) { + isNull = -1; /* empty, but position changed */ + } + } + } + break; + } + } + } + } + return isNull; + } + + protected final int nullCheckMemStRec(int id, int s) { + int level = 0; + int k = stk; + int isNull; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (level == 0) { + if (e.getNullCheckPStr() != s) { + isNull = 0; + break; + } else { + int endp; + isNull = 1; + while (k < stk) { + if (e.type == MEM_START) { + if (e.getMemEnd() == INVALID_INDEX) { + isNull = 0; + break; + } + if (bsAt(regex.btMemEnd, e.getMemNum())) { + endp = stack[e.getMemEnd()].getMemPStr(); + } else { + endp = e.getMemEnd(); + } + if (stack[e.getMemStart()].getMemPStr() != endp) { + isNull = 0; + break; + } else if (endp != s) { + isNull = -1; /* empty, but position changed */ + } + } + k++; + e = stack[k]; + } + break; + } + } else { + level--; + } + } + } else if (e.type == NULL_CHECK_END) { + if (e.getNullCheckNum() == id) level++; + } + } + return isNull; + } + + protected final int getRepeat(int id) { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == REPEAT) { + if (level == 0) { + if (e.getRepeatNum() == id) return k; + } + } else if (e.type == CALL_FRAME) { + level--; + } else if (e.type == RETURN) { + level++; + } + } + } + + protected final int sreturn() { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == CALL_FRAME) { + if (level == 0) { + return e.getCallFrameRetAddr(); + } else { + level--; + } + } else if (e.type == RETURN) { + level++; + } + } + } +} diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java new file mode 100644 index 000000000..c24db21a6 --- /dev/null +++ b/third_party/joni/src/org/joni/Syntax.java @@ -0,0 +1,753 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.constants.MetaChar.INEFFECTIVE_META_CHAR; + +import org.joni.constants.SyntaxProperties; + +public final class Syntax implements SyntaxProperties { + public final String name; + public final int op; + public final int op2; + public final int op3; + public final int behavior; + public final int options; + public final MetaCharTable metaCharTable; + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable) { + this.name = name; + this.op = op; + this.op2 = op2; + this.op3 = op3; + this.behavior = behavior; + this.options = options; + this.metaCharTable = metaCharTable; + } + + public static class MetaCharTable { + public final int esc; + public final int anyChar; + public final int anyTime; + public final int zeroOrOneTime; + public final int oneOrMoreTime; + public final int anyCharAnyTime; + + public MetaCharTable(int esc, int anyChar, int anyTime, + int zeroOrOneTime, int oneOrMoreTime, int anyCharAnyTime) { + this.esc = esc; + this.anyChar = anyChar; + this.anyTime = anyTime; + this.zeroOrOneTime = zeroOrOneTime; + this.oneOrMoreTime = oneOrMoreTime; + this.anyCharAnyTime = anyCharAnyTime; + } + } + + /** + * OP + * + */ + protected boolean isOp(int opm) { + return (op & opm) != 0; + } + + public boolean opVariableMetaCharacters() { + return isOp(OP_VARIABLE_META_CHARACTERS); + } + + public boolean opDotAnyChar() { + return isOp(OP_DOT_ANYCHAR); + } + + public boolean opAsteriskZeroInf() { + return isOp(OP_ASTERISK_ZERO_INF); + } + + public boolean opEscAsteriskZeroInf() { + return isOp(OP_ESC_ASTERISK_ZERO_INF); + } + + public boolean opPlusOneInf() { + return isOp(OP_PLUS_ONE_INF); + } + + public boolean opEscPlusOneInf() { + return isOp(OP_ESC_PLUS_ONE_INF); + } + + public boolean opQMarkZeroOne() { + return isOp(OP_QMARK_ZERO_ONE); + } + + public boolean opEscQMarkZeroOne() { + return isOp(OP_ESC_QMARK_ZERO_ONE); + } + + public boolean opBraceInterval() { + return isOp(OP_BRACE_INTERVAL); + } + + public boolean opEscBraceInterval() { + return isOp(OP_ESC_BRACE_INTERVAL); + } + + public boolean opVBarAlt() { + return isOp(OP_VBAR_ALT); + } + + public boolean opEscVBarAlt() { + return isOp(OP_ESC_VBAR_ALT); + } + + public boolean opLParenSubexp() { + return isOp(OP_LPAREN_SUBEXP); + } + + public boolean opEscLParenSubexp() { + return isOp(OP_ESC_LPAREN_SUBEXP); + } + + public boolean opEscAZBufAnchor() { + return isOp(OP_ESC_AZ_BUF_ANCHOR); + } + + public boolean opEscCapitalGBeginAnchor() { + return isOp(OP_ESC_CAPITAL_G_BEGIN_ANCHOR); + } + + public boolean opDecimalBackref() { + return isOp(OP_DECIMAL_BACKREF); + } + + public boolean opBracketCC() { + return isOp(OP_BRACKET_CC); + } + + public boolean opEscWWord() { + return isOp(OP_ESC_W_WORD); + } + + public boolean opEscLtGtWordBeginEnd() { + return isOp(OP_ESC_LTGT_WORD_BEGIN_END); + } + + public boolean opEscBWordBound() { + return isOp(OP_ESC_B_WORD_BOUND); + } + + public boolean opEscSWhiteSpace() { + return isOp(OP_ESC_S_WHITE_SPACE); + } + + public boolean opEscDDigit() { + return isOp(OP_ESC_D_DIGIT); + } + + public boolean opLineAnchor() { + return isOp(OP_LINE_ANCHOR); + } + + public boolean opPosixBracket() { + return isOp(OP_POSIX_BRACKET); + } + + public boolean opQMarkNonGreedy() { + return isOp(OP_QMARK_NON_GREEDY); + } + + public boolean opEscControlChars() { + return isOp(OP_ESC_CONTROL_CHARS); + } + + public boolean opEscCControl() { + return isOp(OP_ESC_C_CONTROL); + } + + public boolean opEscOctal3() { + return isOp(OP_ESC_OCTAL3); + } + + public boolean opEscXHex2() { + return isOp(OP_ESC_X_HEX2); + } + + public boolean opEscXBraceHex8() { + return isOp(OP_ESC_X_BRACE_HEX8); + } + + public boolean opEscOBraceOctal() { + return isOp(OP_ESC_O_BRACE_OCTAL); + } + + /** + * OP + * + */ + protected boolean isOp2(int opm) { + return (op2 & opm) != 0; + } + + public boolean op2EscCapitalQQuote() { + return isOp2(OP2_ESC_CAPITAL_Q_QUOTE); + } + + public boolean op2QMarkGroupEffect() { + return isOp2(OP2_QMARK_GROUP_EFFECT); + } + + public boolean op2OptionPerl() { + return isOp2(OP2_OPTION_PERL); + } + + public boolean op2OptionRuby() { + return isOp2(OP2_OPTION_RUBY); + } + + public boolean op2PlusPossessiveRepeat() { + return isOp2(OP2_PLUS_POSSESSIVE_REPEAT); + } + + public boolean op2PlusPossessiveInterval() { + return isOp2(OP2_PLUS_POSSESSIVE_INTERVAL); + } + + public boolean op2CClassSetOp() { + return isOp2(OP2_CCLASS_SET_OP); + } + + public boolean op2QMarkLtNamedGroup() { + return isOp2(OP2_QMARK_LT_NAMED_GROUP); + } + + public boolean op2EscKNamedBackref() { + return isOp2(OP2_ESC_K_NAMED_BACKREF); + } + + public boolean op2EscGSubexpCall() { + return isOp2(OP2_ESC_G_SUBEXP_CALL); + } + + public boolean op2AtMarkCaptureHistory() { + return isOp2(OP2_ATMARK_CAPTURE_HISTORY); + } + + public boolean op2EscCapitalCBarControl() { + return isOp2(OP2_ESC_CAPITAL_C_BAR_CONTROL); + } + + public boolean op2EscCapitalMBarMeta() { + return isOp2(OP2_ESC_CAPITAL_M_BAR_META); + } + + public boolean op2EscVVtab() { + return isOp2(OP2_ESC_V_VTAB); + } + + public boolean op2EscUHex4() { + return isOp2(OP2_ESC_U_HEX4); + } + + public boolean op2EscGnuBufAnchor() { + return isOp2(OP2_ESC_GNU_BUF_ANCHOR); + } + + public boolean op2EscPBraceCharProperty() { + return isOp2(OP2_ESC_P_BRACE_CHAR_PROPERTY); + } + + public boolean op2EscPBraceCircumflexNot() { + return isOp2(OP2_ESC_P_BRACE_CIRCUMFLEX_NOT); + } + + public boolean op2EscHXDigit() { + return isOp2(OP2_ESC_H_XDIGIT); + } + + public boolean op2IneffectiveEscape() { + return isOp2(OP2_INEFFECTIVE_ESCAPE); + } + + public boolean op2EscCapitalRLinebreak() { + return isOp2(OP2_ESC_CAPITAL_R_LINEBREAK); + } + + public boolean op2EscCapitalXExtendedGraphemeCluster() { + return isOp2(OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER); + } + + public boolean op2EscVVerticalWhiteSpace() { + return isOp2(OP2_ESC_V_VERTICAL_WHITESPACE); + } + + public boolean op2EscHHorizontalWhiteSpace() { + return isOp2(OP2_ESC_H_HORIZONTAL_WHITESPACE); + } + + public boolean op2EscCapitalKKeep() { + return isOp2(OP2_ESC_CAPITAL_K_KEEP); + } + + public boolean op2QMarkTildeAbsent() { + return isOp2(OP2_QMARK_TILDE_ABSENT); + } + + public boolean op2EscGBraceBackref() { + return isOp2(OP2_ESC_G_BRACE_BACKREF); + } + + public boolean op2QMarkSubexpCall() { + return isOp2(OP2_QMARK_SUBEXP_CALL); + } + + public boolean op2QMarkBarBranchReset() { + return isOp2(OP2_QMARK_BAR_BRANCH_RESET); + } + + public boolean op2QMarkLParenCondition() { + return isOp2(OP2_QMARK_LPAREN_CONDITION); + } + + public boolean op2QMarkCapitalPNamedGroup() { + return isOp2(OP2_QMARK_CAPITAL_P_NAMED_GROUP); + } + + protected boolean isOp3(int opm) { + return (op3 & opm) != 0; + } + + public boolean op3OptionJava() { + return isOp3(OP3_OPTION_JAVA); + } + + public boolean op3OptionECMAScript() { + return isOp3(OP3_OPTION_ECMASCRIPT); + } + + + /** + * BEHAVIOR + * + */ + protected boolean isBehavior(int bvm) { + return (behavior & bvm) != 0; + } + + public boolean contextIndepRepeatOps() { + return isBehavior(CONTEXT_INDEP_REPEAT_OPS); + } + + public boolean contextInvalidRepeatOps() { + return isBehavior(CONTEXT_INVALID_REPEAT_OPS); + } + + public boolean allowUnmatchedCloseSubexp() { + return isBehavior(ALLOW_UNMATCHED_CLOSE_SUBEXP); + } + + public boolean allowInvalidInterval() { + return isBehavior(ALLOW_INVALID_INTERVAL); + } + + public boolean allowIntervalLowAbbrev() { + return isBehavior(ALLOW_INTERVAL_LOW_ABBREV); + } + + public boolean strictCheckBackref() { + return isBehavior(STRICT_CHECK_BACKREF); + } + + public boolean differentLengthAltLookBehind() { + return isBehavior(DIFFERENT_LEN_ALT_LOOK_BEHIND); + } + + public boolean captureOnlyNamedGroup() { + return isBehavior(CAPTURE_ONLY_NAMED_GROUP); + } + + public boolean allowMultiplexDefinitionName() { + return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME); + } + + public boolean fixedIntervalIsGreedyOnly() { + return isBehavior(FIXED_INTERVAL_IS_GREEDY_ONLY); + } + + + public boolean notNewlineInNegativeCC() { + return isBehavior(NOT_NEWLINE_IN_NEGATIVE_CC); + } + + public boolean backSlashEscapeInCC() { + return isBehavior(BACKSLASH_ESCAPE_IN_CC); + } + + public boolean allowEmptyRangeInCC() { + return isBehavior(ALLOW_EMPTY_RANGE_IN_CC); + } + + public boolean allowDoubleRangeOpInCC() { + return isBehavior(ALLOW_DOUBLE_RANGE_OP_IN_CC); + } + + public boolean warnCCOpNotEscaped() { + return isBehavior(WARN_CC_OP_NOT_ESCAPED); + } + + public boolean warnCCDup() { + return isBehavior(WARN_CC_DUP); + } + + public boolean warnReduntantNestedRepeat() { + return isBehavior(WARN_REDUNDANT_NESTED_REPEAT); + } + + public static final Syntax RUBY = new Syntax( + "RUBY", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_QMARK_GROUP_EFFECT | + OP2_OPTION_RUBY | + OP2_QMARK_LT_NAMED_GROUP | OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_G_SUBEXP_CALL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_PLUS_POSSESSIVE_REPEAT | + OP2_CCLASS_SET_OP | OP2_ESC_CAPITAL_C_BAR_CONTROL | + OP2_ESC_CAPITAL_M_BAR_META | OP2_ESC_V_VTAB | + OP2_ESC_H_XDIGIT | + OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER | + OP2_QMARK_LPAREN_CONDITION | + OP2_ESC_CAPITAL_R_LINEBREAK | + OP2_ESC_CAPITAL_K_KEEP | + OP2_QMARK_TILDE_ABSENT + ), + + 0, + + ( GNU_REGEX_BV | + ALLOW_INTERVAL_LOW_ABBREV | + DIFFERENT_LEN_ALT_LOOK_BEHIND | + CAPTURE_ONLY_NAMED_GROUP | + ALLOW_MULTIPLEX_DEFINITION_NAME | + FIXED_INTERVAL_IS_GREEDY_ONLY | + WARN_CC_OP_NOT_ESCAPED | + WARN_CC_DUP | + WARN_REDUNDANT_NESTED_REPEAT ), + + (Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax DEFAULT = RUBY; + + public static final Syntax TEST = new Syntax("TEST", RUBY.op, RUBY.op2 | OP2_ESC_U_HEX4, RUBY.op3, RUBY.behavior, RUBY.options & ~ Option.ASCII_RANGE, RUBY.metaCharTable); + + public static final Syntax ASIS = new Syntax( + "ASIS", + 0, + + OP2_INEFFECTIVE_ESCAPE, + + 0, + + 0, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PosixBasic = new Syntax( + "PosixBasic", + (POSIX_COMMON_OP | OP_ESC_LPAREN_SUBEXP | + OP_ESC_BRACE_INTERVAL ), + + 0, + + 0, + + 0, + + ( Option.SINGLELINE | Option.MULTILINE ), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PosixExtended = new Syntax( + "PosixExtended", + ( POSIX_COMMON_OP | OP_LPAREN_SUBEXP | + OP_BRACE_INTERVAL | + OP_PLUS_ONE_INF | OP_QMARK_ZERO_ONE |OP_VBAR_ALT ), + + 0, + + 0, + + ( CONTEXT_INDEP_ANCHORS | + CONTEXT_INDEP_REPEAT_OPS | CONTEXT_INVALID_REPEAT_OPS | + ALLOW_UNMATCHED_CLOSE_SUBEXP | + ALLOW_DOUBLE_RANGE_OP_IN_CC ), + + ( Option.SINGLELINE | Option.MULTILINE ), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Emacs = new Syntax( + "Emacs", + ( OP_DOT_ANYCHAR | OP_BRACKET_CC | + OP_ESC_BRACE_INTERVAL | + OP_ESC_LPAREN_SUBEXP | OP_ESC_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_PLUS_ONE_INF | + OP_QMARK_ZERO_ONE | OP_DECIMAL_BACKREF | + OP_LINE_ANCHOR | OP_ESC_CONTROL_CHARS ), + + OP2_ESC_GNU_BUF_ANCHOR, + + 0, + + ALLOW_EMPTY_RANGE_IN_CC, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Grep = new Syntax( + "Grep", + ( OP_DOT_ANYCHAR | OP_BRACKET_CC | OP_POSIX_BRACKET | + OP_ESC_BRACE_INTERVAL | OP_ESC_LPAREN_SUBEXP | + OP_ESC_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_ESC_PLUS_ONE_INF | + OP_ESC_QMARK_ZERO_ONE | OP_LINE_ANCHOR | + OP_ESC_W_WORD | OP_ESC_B_WORD_BOUND | + OP_ESC_LTGT_WORD_BEGIN_END | OP_DECIMAL_BACKREF ), + + 0, + + 0, + + ( ALLOW_EMPTY_RANGE_IN_CC | NOT_NEWLINE_IN_NEGATIVE_CC ), + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax GnuRegex = new Syntax( + "GnuRegex", + GNU_REGEX_OP, + + 0, + + 0, + + GNU_REGEX_BV, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Java = new Syntax( + "Java", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_CONTROL_CHARS | OP_ESC_C_CONTROL | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | OP2_QMARK_GROUP_EFFECT | + OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_REPEAT | + OP2_PLUS_POSSESSIVE_INTERVAL | OP2_CCLASS_SET_OP | + OP2_QMARK_LT_NAMED_GROUP | OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_V_VTAB | OP2_ESC_U_HEX4 | + OP2_ESC_P_BRACE_CHAR_PROPERTY ), + + 0, + + ( GNU_REGEX_BV | DIFFERENT_LEN_ALT_LOOK_BEHIND ), + + (Option.SINGLELINE | Option.WORD_BOUND_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Perl = new Syntax( + "Perl", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT ), + + 0, + + GNU_REGEX_BV, + + Option.SINGLELINE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PerlNG = new Syntax( + "PerlNG", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_QMARK_LT_NAMED_GROUP | + OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_G_SUBEXP_CALL ), + + 0, + + ( GNU_REGEX_BV | + CAPTURE_ONLY_NAMED_GROUP | + ALLOW_MULTIPLEX_DEFINITION_NAME ), + + Option.SINGLELINE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax ECMAScript = new Syntax( + "ECMAScript", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_CONTROL_CHARS | OP_ESC_C_CONTROL | + OP_DECIMAL_BACKREF | OP_ESC_D_DIGIT | + OP_ESC_S_WHITE_SPACE | OP_ESC_W_WORD ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_ESC_U_HEX4 | OP2_ESC_V_VTAB), + + OP3_OPTION_ECMASCRIPT, + + ( CONTEXT_INDEP_ANCHORS | + CONTEXT_INDEP_REPEAT_OPS | + CONTEXT_INVALID_REPEAT_OPS | + ALLOW_INVALID_INTERVAL | + BACKSLASH_ESCAPE_IN_CC | + ALLOW_DOUBLE_RANGE_OP_IN_CC | + DIFFERENT_LEN_ALT_LOOK_BEHIND ), + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); +} diff --git a/third_party/joni/src/org/joni/Token.java b/third_party/joni/src/org/joni/Token.java new file mode 100644 index 000000000..425c6f407 --- /dev/null +++ b/third_party/joni/src/org/joni/Token.java @@ -0,0 +1,180 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.constants.internal.TokenType; + +final class Token { + TokenType type; + boolean escaped; + int base; /* is number: 8, 16 (used in [....]) */ + int backP; + + // union fields + private int INT1, INT2, INT3, INT4, INT5; + private int []INTA1; + + // union accessors + int getC() { + return INT1; + } + void setC(int c) { + INT1 = c; + } + + int getCode() { + return INT1; + } + void setCode(int code) { + INT1 = code; + } + + int getAnchorSubtype() { + return INT1; + } + void setAnchorSubtype(int anchor) { + INT1 = anchor; + } + + boolean getAnchorASCIIRange() { + return INT2 == 1; + } + + void setAnchorASCIIRange(boolean ascii) { + INT2 = ascii ? 1 : 0; + } + + // repeat union member + int getRepeatLower() { + return INT1; + } + void setRepeatLower(int lower) { + INT1 = lower; + } + + int getRepeatUpper() { + return INT2; + } + void setRepeatUpper(int upper) { + INT2 = upper; + } + + boolean getRepeatGreedy() { + return INT3 != 0; + } + void setRepeatGreedy(boolean greedy) { + INT3 = greedy ? 1 : 0; + } + + boolean getRepeatPossessive() { + return INT4 != 0; + } + void setRepeatPossessive(boolean possessive) { + INT4 = possessive ? 1 : 0; + } + + // backref union member + int getBackrefNum() { + return INT1; + } + void setBackrefNum(int num) { + INT1 = num; + } + + int getBackrefRef1() { + return INT2; + } + void setBackrefRef1(int ref1) { + INT2 = ref1; + } + + int[]getBackrefRefs() { + return INTA1; + } + void setBackrefRefs(int[]refs) { + INTA1 = refs; + } + + boolean getBackrefByName() { + return INT3 != 0; + } + void setBackrefByName(boolean byName) { + INT3 = byName ? 1 : 0; + } + + // USE_BACKREF_AT_LEVEL + boolean getBackrefExistLevel() { + return INT4 != 0; + } + void setBackrefExistLevel(boolean existLevel) { + INT4 = existLevel ? 1 : 0; + } + + int getBackrefLevel() { + return INT5; + } + void setBackrefLevel(int level) { + INT5 = level; + } + + // call union member + int getCallNameP() { + return INT1; + } + void setCallNameP(int nameP) { + INT1 = nameP; + } + + int getCallNameEnd() { + return INT2; + } + void setCallNameEnd(int nameEnd) { + INT2 = nameEnd; + } + + int getCallGNum() { + return INT3; + } + void setCallGNum(int gnum) { + INT3 = gnum; + } + + boolean getCallRel() { + return INT4 != 0; + } + void setCallRel(boolean rel) { + INT4 = rel ? 1 : 0; + } + + // prop union member + int getPropCType() { + return INT1; + } + void setPropCType(int ctype) { + INT1 = ctype; + } + + boolean getPropNot() { + return INT2 != 0; + } + void setPropNot(boolean not) { + INT2 = not ? 1 : 0; + } +} diff --git a/third_party/joni/src/org/joni/UnsetAddrList.java b/third_party/joni/src/org/joni/UnsetAddrList.java new file mode 100644 index 000000000..c0b64f73e --- /dev/null +++ b/third_party/joni/src/org/joni/UnsetAddrList.java @@ -0,0 +1,68 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.ast.EncloseNode; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; + +public final class UnsetAddrList { + EncloseNode[]targets; + int[]offsets; + int num; + + public UnsetAddrList(int size) { + targets = new EncloseNode[size]; + offsets = new int[size]; + } + + public void add(int offset, EncloseNode node) { + if (num >= offsets.length) { + EncloseNode []ttmp = new EncloseNode[targets.length << 1]; + System.arraycopy(targets, 0, ttmp, 0, num); + targets = ttmp; + int[]otmp = new int[offsets.length << 1]; + System.arraycopy(offsets, 0, otmp, 0, num); + offsets = otmp; + } + targets[num] = node; + offsets[num] = offset; + num++; + } + + public void fix(Regex regex) { + for (int i=0; i 0) { + for (int i = 0; i < num; i++) { + value.append("offset + ").append(offsets[i]).append(" target: ").append(targets[i].getAddressName()); + } + } + return value.toString(); + } +} diff --git a/third_party/joni/src/org/joni/WarnCallback.java b/third_party/joni/src/org/joni/WarnCallback.java new file mode 100644 index 000000000..d79d6a1bb --- /dev/null +++ b/third_party/joni/src/org/joni/WarnCallback.java @@ -0,0 +1,40 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** + * @author Ola Bini + */ +public interface WarnCallback { + WarnCallback DEFAULT = new WarnCallback() { + @Override + public void warn(String message) { + System.err.println(message); + } + }; + + WarnCallback NONE = new WarnCallback() { + @Override + public void warn(String message) { + } + }; + + void warn(String message); +} diff --git a/third_party/joni/src/org/joni/ast/AnchorNode.java b/third_party/joni/src/org/joni/ast/AnchorNode.java new file mode 100644 index 000000000..f69a965b2 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/AnchorNode.java @@ -0,0 +1,95 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.AnchorType; + +public final class AnchorNode extends Node { + public final int type; + public final boolean asciiRange; + public Node target; + public int charLength; + + public AnchorNode(int type, boolean asciiRange) { + super(ANCHOR); + this.type = type; + charLength = -1; + this.asciiRange = asciiRange; + } + + public AnchorNode(int type) { + this(type, false); + } + + @Override + protected void setChild(Node child) { + target = child; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(Node tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Anchor"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(); + value.append("\n type: " + typeToString()); + value.append(", ascii: " + asciiRange); + value.append("\n target: " + pad(target, level + 1)); + return value.toString(); + } + + public String typeToString() { + StringBuilder type = new StringBuilder(); + if (isType(AnchorType.BEGIN_BUF)) type.append("BEGIN_BUF "); + if (isType(AnchorType.BEGIN_LINE)) type.append("BEGIN_LINE "); + if (isType(AnchorType.BEGIN_POSITION)) type.append("BEGIN_POSITION "); + if (isType(AnchorType.END_BUF)) type.append("END_BUF "); + if (isType(AnchorType.SEMI_END_BUF)) type.append("SEMI_END_BUF "); + if (isType(AnchorType.END_LINE)) type.append("END_LINE "); + if (isType(AnchorType.WORD_BOUND)) type.append("WORD_BOUND "); + if (isType(AnchorType.NOT_WORD_BOUND)) type.append("NOT_WORD_BOUND "); + if (isType(AnchorType.WORD_BEGIN)) type.append("WORD_BEGIN "); + if (isType(AnchorType.WORD_END)) type.append("WORD_END "); + if (isType(AnchorType.PREC_READ)) type.append("PREC_READ "); + if (isType(AnchorType.PREC_READ_NOT)) type.append("PREC_READ_NOT "); + if (isType(AnchorType.LOOK_BEHIND)) type.append("LOOK_BEHIND "); + if (isType(AnchorType.LOOK_BEHIND_NOT)) type.append("LOOK_BEHIND_NOT "); + if (isType(AnchorType.ANYCHAR_STAR)) type.append("ANYCHAR_STAR "); + if (isType(AnchorType.ANYCHAR_STAR_ML)) type.append("ANYCHAR_STAR_ML "); + return type.toString(); + } + + private boolean isType(int type) { + return (this.type & type) != 0; + } + +} diff --git a/third_party/joni/src/org/joni/ast/AnyCharNode.java b/third_party/joni/src/org/joni/ast/AnyCharNode.java new file mode 100644 index 000000000..33f43c857 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/AnyCharNode.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +public final class AnyCharNode extends Node { + public AnyCharNode(){ + super(CANY); + } + + @Override + public String getName() { + return "Any Char"; + } + + @Override + public String toString(int level) { + String value = ""; + return value; + } +} diff --git a/third_party/joni/src/org/joni/ast/BackRefNode.java b/third_party/joni/src/org/joni/ast/BackRefNode.java new file mode 100644 index 000000000..fd50d021d --- /dev/null +++ b/third_party/joni/src/org/joni/ast/BackRefNode.java @@ -0,0 +1,86 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.Config; +import org.joni.ScanEnvironment; +import org.joni.exception.ErrorMessages; +import org.joni.exception.ValueException; + +public final class BackRefNode extends StateNode { + public final int[] back; + public int backNum; + public int nestLevel; + + private BackRefNode(int backNum, int[]backRefs, boolean byName, ScanEnvironment env) { + super(BREF); + this.backNum = backNum; + if (byName) setNameRef(); + + for (int i=0; i 0) { + back[pos] = n; + pos++; + } + } + backNum = pos; + } + + @Override + public String getName() { + return "Back Ref"; + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(super.toString(level)); + sb.append("\n backNum: " + backNum); + String backs = ""; + for (int i=0; i= sbOut) { + if (j > CR_FROM(mbr, i)) { + addCodeRangeToBuf(env, j, CR_TO(mbr, i)); + i++; + } + // !goto sb_end!, remove duplication! + for (; i= sbOut) { + // !goto sb_end2!, remove duplication + prev = sbOut; + for (i=0; i 1) { + ccAscii.addCodeRangeToBuf(env, 0x00, 0x7F); + } else { + ccAscii.bs.setRange(env, 0x00, 0x7F); + } + ccWork.and(ccAscii, env); + } + or(ccWork, env); + } else { + addCTypeByRange(ctype, not, env, sbOut.value, ranges); + } + return; + } + + int maxCode = asciiRange ? 0x80 : BitSet.SINGLE_BYTE_SIZE; + switch(ctype) { + case CharacterType.ALPHA: + case CharacterType.BLANK: + case CharacterType.CNTRL: + case CharacterType.DIGIT: + case CharacterType.LOWER: + case CharacterType.PUNCT: + case CharacterType.SPACE: + case CharacterType.UPPER: + case CharacterType.XDIGIT: + case CharacterType.ASCII: + case CharacterType.ALNUM: + if (not) { + for (int c=0; c= maxCode) bs.set(env, c); + } + if (asciiRange) addAllMultiByteRange(env); + } else { + for (int c=0; c 0 && /* check invalid code point */ + !(enc.isWord(c) || c >= maxCode)) bs.set(env, c); + } + if (asciiRange) addAllMultiByteRange(env); + } + break; + + default: + throw new InternalException(ErrorMessages.PARSER_BUG); + } // switch + } + + public enum CCVALTYPE { + SB, + CODE_POINT, + CLASS + } + + public enum CCSTATE { + VALUE, + RANGE, + COMPLETE, + START + } + + public static final class CCStateArg { + public int from; + public int to; + public boolean fromIsRaw; + public boolean toIsRaw; + public CCVALTYPE inType; + public CCVALTYPE type; + public CCSTATE state; + } + + public void nextStateClass(CCStateArg arg, CClassNode ascCC, ScanEnvironment env) { + if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + + if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { + if (arg.type == CCVALTYPE.SB) { + bs.set(env, arg.from); + if (ascCC != null) ascCC.bs.set(arg.from); + } else if (arg.type == CCVALTYPE.CODE_POINT) { + addCodeRange(env, arg.from, arg.from); + if (ascCC != null) ascCC.addCodeRange(env, arg.from, arg.from, false); + } + } + arg.state = CCSTATE.VALUE; + arg.type = CCVALTYPE.CLASS; + } + + public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env) { + switch(arg.state) { + case VALUE: + if (arg.type == CCVALTYPE.SB) { + bs.set(env, arg.from); + if (ascCc != null) ascCc.bs.set(arg.from); + } else if (arg.type == CCVALTYPE.CODE_POINT) { + addCodeRange(env, arg.from, arg.from); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + } + break; + + case RANGE: + if (arg.inType == arg.type) { + if (arg.inType == CCVALTYPE.SB) { + if (arg.from > 0xff || arg.to > 0xff) throw new ValueException(ErrorMessages.ERR_INVALID_CODE_POINT_VALUE); + + if (arg.from > arg.to) { + if (env.syntax.allowEmptyRangeInCC()) { + // goto ccs_range_end + arg.state = CCSTATE.COMPLETE; + break; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + bs.setRange(env, arg.from, arg.to); + if (ascCc != null) ascCc.bs.setRange(null, arg.from, arg.to); + } else { + addCodeRange(env, arg.from, arg.to); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.to, false); + } + } else { + if (arg.from > arg.to) { + if (env.syntax.allowEmptyRangeInCC()) { + // goto ccs_range_end + arg.state = CCSTATE.COMPLETE; + break; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + bs.setRange(env, arg.from, arg.to < 0xff ? arg.to : 0xff); + addCodeRange(env, arg.from, arg.to); + if (ascCc != null) { + ascCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); + ascCc.addCodeRange(env, arg.from, arg.to, false); + } + } + // ccs_range_end: + arg.state = CCSTATE.COMPLETE; + break; + + case COMPLETE: + case START: + arg.state = CCSTATE.VALUE; + break; + + default: + break; + + } // switch + + arg.fromIsRaw = arg.toIsRaw; + arg.from = arg.to; + arg.type = arg.inType; + } + + // onig_is_code_in_cc_len + boolean isCodeInCCLength(int encLength, int code) { + boolean found; + + if (encLength > 1 || code >= BitSet.SINGLE_BYTE_SIZE) { + if (mbuf == null) { + found = false; + } else { + found = CodeRange.isInCodeRange(mbuf.getCodeRange(), code); + } + } else { + found = bs.at(code); + } + + if (isNot()) { + return !found; + } else { + return found; + } + } + + // onig_is_code_in_cc + public boolean isCodeInCC(Encoding enc, int code) { + int len; + if (enc.minLength() > 1) { + len = 2; + } else { + len = enc.codeToMbcLength(code); + } + return isCodeInCCLength(len, code); + } + + public void setNot() { + flags |= FLAG_NCCLASS_NOT; + } + + public void clearNot() { + flags &= ~FLAG_NCCLASS_NOT; + } + + public boolean isNot() { + return (flags & FLAG_NCCLASS_NOT) != 0; + } +} diff --git a/third_party/joni/src/org/joni/ast/CTypeNode.java b/third_party/joni/src/org/joni/ast/CTypeNode.java new file mode 100644 index 000000000..fd14de4e6 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CTypeNode.java @@ -0,0 +1,48 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +public final class CTypeNode extends Node { + public final int ctype; + public final boolean not; + public final boolean asciiRange; + + public CTypeNode(int type, boolean not, boolean asciiRange) { + super(CTYPE); + this.ctype= type; + this.not = not; + this.asciiRange = asciiRange; + } + + @Override + public String getName() { + return "Character Type"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(); + value.append("\n ctype: " + ctype); + value.append(", not: " + not); + value.append(", ascii: " + asciiRange); + return value.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/ast/CallNode.java b/third_party/joni/src/org/joni/ast/CallNode.java new file mode 100644 index 000000000..1bc7f9e92 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CallNode.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.UnsetAddrList; + +public final class CallNode extends StateNode { + public final byte[]name; + public final int nameP; + public final int nameEnd; + + public int groupNum; + public EncloseNode target; + public UnsetAddrList unsetAddrList; + + public CallNode(byte[]name, int nameP, int nameEnd, int gnum) { + super(CALL); + this.name = name; + this.nameP = nameP; + this.nameEnd = nameEnd; + this.groupNum = gnum; /* call by number if gnum != 0 */ + } + + @Override + protected void setChild(Node newChild) { + target = (EncloseNode)newChild; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(EncloseNode tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Call"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(super.toString(level)); + value.append("\n name: " + new String(name, nameP, nameEnd - nameP)); + value.append(", groupNum: " + groupNum); + value.append("\n unsetAddrList: " + pad(unsetAddrList, level + 1)); + value.append("\n target: " + pad(target.getAddressName(), level + 1)); + return value.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/ast/CalloutNode.java b/third_party/joni/src/org/joni/ast/CalloutNode.java new file mode 100644 index 000000000..fa6f2c365 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CalloutNode.java @@ -0,0 +1,42 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +/** Zero-width internal node carrying an engine callout-table index. */ +public final class CalloutNode extends StringNode { + public final int calloutId; + + public CalloutNode(int calloutId) { + super(0); + this.calloutId = calloutId; + setRaw(); + setDontGetOptInfo(); + } + + @Override + public String getName() { + return "Callout"; + } + + @Override + public String toString(int level) { + return "\n id: " + calloutId; + } +} diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java new file mode 100644 index 000000000..680ded2b8 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -0,0 +1,130 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.Config; +import org.joni.Option; +import org.joni.constants.internal.EncloseType; + +public final class EncloseNode extends StateNode implements EncloseType { + public final int type; // enclose type + public int regNum; + public int option; + public Node target; /* EncloseNode : ENCLOSE_MEMORY */ + public int callAddr; // AbsAddrType + public int minLength; // OnigDistance + public int maxLength; // OnigDistance + public int charLength; + public int optCount; // referenced count in optimize_node_left() + public Node containingAnchor; // + + // node_new_enclose / onig_node_new_enclose + public EncloseNode(int type) { + super(ENCLOSE); + this.type = type; + callAddr = -1; + } + + public static EncloseNode newMemory(int option, boolean isNamed) { + EncloseNode en = new EncloseNode(MEMORY); + if (Config.USE_SUBEXP_CALL) en.option = option; + if (isNamed) en.setNamedGroup(); + return en; + } + + public static EncloseNode newOption(int option) { + EncloseNode en = new EncloseNode(OPTION); + en.option = option; + return en; + } + + @Override + protected void setChild(Node child) { + target = child; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(Node tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Enclose"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(super.toString(level)); + value.append("\n type: " + typeToString()); + value.append("\n regNum: " + regNum); + value.append(", option: " + Option.toString(option)); + value.append(", callAddr: " + callAddr); + value.append(", minLength: " + minLength); + value.append(", maxLength: " + maxLength); + value.append(", charLength: " + charLength); + value.append(", optCount: " + optCount); + value.append("\n target: " + pad(target, level + 1)); + return value.toString(); + } + + public String typeToString() { + StringBuilder types = new StringBuilder(); + if (isStopBacktrack()) types.append("STOP_BACKTRACK "); + if (isMemory()) types.append("MEMORY "); + if (isOption()) types.append("OPTION "); + if (isCondition()) types.append("CONDITION "); + if (isAbsent()) types.append("ABSENT "); + return types.toString(); + } + + public void setEncloseStatus(int flag) { + state |= flag; + } + + public void clearEncloseStatus(int flag) { + state &= ~flag; + } + + public boolean isMemory() { + return (type & MEMORY) != 0; + } + + public boolean isOption() { + return (type & OPTION) != 0; + } + + public boolean isCondition() { + return (type & CONDITION) != 0; + } + + public boolean isStopBacktrack() { + return (type & STOP_BACKTRACK) != 0; + } + + public boolean isAbsent() { + return (type & ABSENT) != 0; + } +} diff --git a/third_party/joni/src/org/joni/ast/ListNode.java b/third_party/joni/src/org/joni/ast/ListNode.java new file mode 100644 index 000000000..7684f4c51 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/ListNode.java @@ -0,0 +1,96 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; + +public final class ListNode extends Node { + public Node value; + public ListNode tail; + + private ListNode(Node value, ListNode tail, int type) { + super(type); + this.value = value; + if (value != null) value.parent = this; + this.tail = tail; + if (tail != null) tail.parent = this; + } + + public static ListNode newAlt(Node value, ListNode tail) { + return new ListNode(value, tail, ALT); + } + + public static ListNode newList(Node value, ListNode tail) { + return new ListNode(value, tail, LIST); + } + + public static ListNode listAdd(ListNode list, Node value) { + ListNode n = newList(value, null); + if (list != null) { + while (list.tail != null) list = list.tail; + list.setTail(n); + } + return n; + } + + public void toListNode() { + type = LIST; + } + + @Override + protected void setChild(Node child) { + value = child; + } + + @Override + protected Node getChild() { + return value; + } + + public void setValue(Node value) { + this.value = value; + value.parent = this; + } + + public void setTail(ListNode tail) { + this.tail = tail; + } + + @Override + public String getName() { + switch (type) { + case ALT: + return "Alt"; + case LIST: + return "List"; + default: + throw new InternalException(ErrorMessages.PARSER_BUG); + } + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(); + sb.append("\n value: " + pad(value, level + 1)); + sb.append("\n tail: " + (tail == null ? "NULL" : tail.toString())); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/ast/Node.java b/third_party/joni/src/org/joni/ast/Node.java new file mode 100644 index 000000000..759f0917b --- /dev/null +++ b/third_party/joni/src/org/joni/ast/Node.java @@ -0,0 +1,120 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.NodeType; + +public abstract class Node implements NodeType { + public Node parent; + protected int type; + + Node(int type) { + this.type = type; + } + + public final int getType() { + return type; + } + + public final int getType2Bit() { + return 1 << getType(); + } + + protected void setChild(Node tgt){ + // default definition + } + + protected Node getChild(){ + // default definition + return null; + } + + public void replaceWith(Node with) { + with.parent = parent; + parent.setChild(with); + parent = null; + } + + public abstract String getName(); + protected abstract String toString(int level); + + public String getAddressName() { + return getName() + ":0x" + Integer.toHexString(System.identityHashCode(this)); + } + + @Override + public final String toString() { + StringBuilder s = new StringBuilder(); + s.append("<" + getAddressName() + " (" + (parent == null ? "NULL" : parent.getAddressName()) + ")>"); + return s + toString(0); + } + + protected static String pad(Object value, int level) { + if (value == null) return "NULL"; + + StringBuilder pad = new StringBuilder(" "); + for (int i=0; i= 0 && targetQNum >= 0 && env.syntax.warnReduntantNestedRepeat()) { + switch(REDUCE_TABLE[targetQNum][nestQNum]) { + case ASIS: + break; + case DEL: + env.warnings.warn("regular expression has redundant nested repeat operator " + PopularQStr[targetQNum] + " /" + new String(bytes, p, end - p) + "/"); + break; + default: + env.warnings.warn("nested repeat operator '" + PopularQStr[targetQNum] + "' and '" + PopularQStr[nestQNum] + + "' was replaced with '" + ReduceQStr[REDUCE_TABLE[targetQNum][nestQNum].ordinal()] + "' in regular expression " + "/" + new String(bytes, p, end - p) + "/"); + } + } + } // USE_WARNING_REDUNDANT_NESTED_REPEAT_OPERATOR + + if (targetQNum >= 0) { + if (nestQNum >= 0) { + reduceNestedQuantifier(qnt); + return 0; + } else if (targetQNum == 1 || targetQNum == 2) { /* * or + */ + /* (?:a*){n,m}, (?:a+){n,m} => (?:a*){n,n}, (?:a+){n,n} */ + if (!isRepeatInfinite(upper) && upper > 1 && greedy) { + upper = lower == 0 ? 1 : lower; + } + } + } + + default: + break; + } + + setTarget(tgt); + return 0; + } + + public static boolean isRepeatInfinite(int n) { + return n == REPEAT_INFINITE; + } +} diff --git a/third_party/joni/src/org/joni/ast/StateNode.java b/third_party/joni/src/org/joni/ast/StateNode.java new file mode 100644 index 000000000..4feb1c864 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/StateNode.java @@ -0,0 +1,235 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.NodeStatus; + +abstract class StateNode extends Node implements NodeStatus { + protected int state; + + StateNode(int type) { + super(type); + } + + public boolean isMinFixed() { + return (state & NST_MIN_FIXED) != 0; + } + + public void setMinFixed() { + state |= NST_MIN_FIXED; + } + + public void clearMinFixed() { + state &= ~NST_MIN_FIXED; + } + + public boolean isMaxFixed() { + return (state & NST_MAX_FIXED) != 0; + } + + public void setMaxFixed() { + state |= NST_MAX_FIXED; + } + + public void clearMaxFixed() { + state &= ~NST_MAX_FIXED; + } + + public boolean isCLenFixed() { + return (state & NST_CLEN_FIXED) != 0; + } + + public void setCLenFixed() { + state |= NST_CLEN_FIXED; + } + + public void clearCLenFixed() { + state &= ~NST_CLEN_FIXED; + } + + public boolean isMark1() { + return (state & NST_MARK1) != 0; + } + + public void setMark1() { + state |= NST_MARK1; + } + + public void clearMark1() { + state &= ~NST_MARK1; + } + + public boolean isMark2() { + return (state & NST_MARK2) != 0; + } + + public void setMark2() { + state |= NST_MARK2; + } + + public void clearMark2() { + state &= ~NST_MARK2; + } + + public boolean isMemBackrefed() { + return (state & NST_MEM_BACKREFED) != 0; + } + + public void setMemBackrefed() { + state |= NST_MEM_BACKREFED; + } + + public void clearMemBackrefed() { + state &= ~NST_MEM_BACKREFED; + } + + public boolean isStopBtSimpleRepeat() { + return (state & NST_STOP_BT_SIMPLE_REPEAT) != 0; + } + + public void setStopBtSimpleRepeat() { + state |= NST_STOP_BT_SIMPLE_REPEAT; + } + + public void clearStopBtSimpleRepeat() { + state &= ~NST_STOP_BT_SIMPLE_REPEAT; + } + + public boolean isRecursion() { + return (state & NST_RECURSION) != 0; + } + + public void setRecursion() { + state |= NST_RECURSION; + } + + public void clearRecursion() { + state &= ~NST_RECURSION; + } + + public boolean isCalled() { + return (state & NST_CALLED) != 0; + } + + public void setCalled() { + state |= NST_CALLED; + } + + public void clearCAlled() { + state &= ~NST_CALLED; + } + + public boolean isAddrFixed() { + return (state & NST_ADDR_FIXED) != 0; + } + + public void setAddrFixed() { + state |= NST_ADDR_FIXED; + } + + public void clearAddrFixed() { + state &= ~NST_ADDR_FIXED; + } + + public boolean isNamedGroup() { + return (state & NST_NAMED_GROUP) != 0; + } + + public void setNamedGroup() { + state |= NST_NAMED_GROUP; + } + + public void clearNamedGroup() { + state &= ~NST_NAMED_GROUP; + } + + public boolean isNameRef() { + return (state & NST_NAME_REF) != 0; + } + + public void setNameRef() { + state |= NST_NAME_REF; + } + + public void clearNameRef() { + state &= ~NST_NAME_REF; + } + + public boolean isInRepeat() { + return (state & NST_IN_REPEAT) != 0; + } + + public void setInRepeat() { + state |= NST_IN_REPEAT; + } + + public void clearInRepeat() { + state &= ~NST_IN_REPEAT; + } + + public boolean isNestLevel() { + return (state & NST_NEST_LEVEL) != 0; + } + + public void setNestLevel() { + state |= NST_NEST_LEVEL; + } + + public void clearNestLevel() { + state &= ~NST_NEST_LEVEL; + } + + public boolean isByNumber() { + return (state & NST_BY_NUMBER) != 0; + } + + public void setByNumber() { + state |= NST_BY_NUMBER; + } + + public void clearByNumber() { + state &= ~NST_BY_NUMBER; + } + + @Override + public String toString(int level) { + return "\n state: " + stateToString(); + } + + public String stateToString() { + StringBuilder states = new StringBuilder(); + if (isMinFixed()) states.append("MIN_FIXED "); + if (isMaxFixed()) states.append("MAX_FIXED "); + if (isMark1()) states.append("MARK1 "); + if (isMark2()) states.append("MARK2 "); + if (isMemBackrefed()) states.append("MEM_BACKREFED "); + if (isStopBtSimpleRepeat()) states.append("STOP_BT_SIMPLE_REPEAT "); + if (isRecursion()) states.append("RECURSION "); + if (isCalled()) states.append("CALLED "); + if (isAddrFixed()) states.append("ADDR_FIXED "); + if (isNamedGroup()) states.append("NAMED_GROUP "); + if (isNameRef()) states.append("NAME_REF "); + if (isInRepeat()) states.append("IN_REPEAT "); + if (isNestLevel()) states.append("NEST_LEVEL "); + if (isByNumber()) states.append("BY_NUMBER "); + + return states.toString(); + } +} diff --git a/third_party/joni/src/org/joni/ast/StringNode.java b/third_party/joni/src/org/joni/ast/StringNode.java new file mode 100644 index 000000000..2e3e32d62 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/StringNode.java @@ -0,0 +1,216 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.jcodings.Encoding; +import org.joni.Config; +import org.joni.constants.internal.StringType; + +public class StringNode extends Node implements StringType { + private static final int NODE_STR_MARGIN = 16; + private static final int NODE_STR_BUF_SIZE = 24; + public static final StringNode EMPTY = new StringNode(null, Integer.MAX_VALUE, Integer.MAX_VALUE); + + public byte[]bytes; + public int p; + public int end; + public int flag; + + public StringNode(int size) { + super(STR); + this.bytes = new byte[size]; + } + + public StringNode() { + this(NODE_STR_BUF_SIZE); + } + + public static StringNode fromCodePoint(int code, Encoding enc) { + StringNode str = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN); + str.end = enc.codeToMbc(code, str.bytes, str.p); + return str; + } + + public StringNode(byte[]bytes, int p, int end) { + super(STR); + this.bytes = bytes; + this.p = p; + this.end = end; + setShared(); + } + + /* Ensure there is ahead bytes available in node's buffer + * (assumes that the node is not shared) + */ + private void ensure(int ahead) { + int len = (end - p) + ahead; + if (len >= bytes.length) { + byte[]tmp = new byte[len + NODE_STR_MARGIN]; + System.arraycopy(bytes, p, tmp, 0, end - p); + bytes = tmp; + } + } + + /* COW and/or ensure there is ahead bytes available in node's buffer + */ + private void modifyEnsure(int ahead) { + if (isShared()) { + int len = (end - p) + ahead; + byte[]tmp = new byte[len + NODE_STR_MARGIN]; + System.arraycopy(bytes, p, tmp, 0, end - p); + bytes = tmp; + end = end - p; + p = 0; + clearShared(); + } else { + ensure(ahead); + } + } + + @Override + public String getName() { + return "String"; + } + + public int length() { + return end - p; + } + + public int length(Encoding enc) { + return enc.strLength(bytes, p, end); + } + + public StringNode splitLastChar(Encoding enc) { + StringNode n = null; + if (end > p) { + int prev = enc.prevCharHead(bytes, p, end, end); + if (prev != -1 && prev > p) { /* can be split */ + n = new StringNode(bytes, prev, end); + if (isRaw()) n.setRaw(); + end = prev; + } + } + return n; + } + + public boolean canBeSplit(Encoding enc) { + if (end > p) { + return enc.length(bytes, p, end) < (end - p); + } + return false; + } + + public void set(byte[]bytes, int p, int end) { + this.bytes = bytes; + this.p = p; + this.end = end; + setShared(); + } + + public void catBytes(byte[]cat, int catP, int catEnd) { + int len = catEnd - catP; + modifyEnsure(len); + System.arraycopy(cat, catP, bytes, end, len); + end += len; + } + + public void catByte(byte c) { + modifyEnsure(1); + bytes[end++] = c; + } + + public void catCode(int code, Encoding enc) { + modifyEnsure(Config.ENC_CODE_TO_MBC_MAXLEN); + end += enc.codeToMbc(code, bytes, end); + } + + public void setRaw() { + flag |= NSTR_RAW; + } + + public void clearRaw() { + flag &= ~NSTR_RAW; + } + + public boolean isRaw() { + return (flag & NSTR_RAW) != 0; + } + + public void setAmbig() { + flag |= NSTR_AMBIG; + } + + public void clearAmbig() { + flag &= ~NSTR_AMBIG; + } + + public boolean isAmbig() { + return (flag & NSTR_AMBIG) != 0; + } + + public void setDontGetOptInfo() { + flag |= NSTR_DONT_GET_OPT_INFO; + } + + public void clearDontGetOptInfo() { + flag &= ~NSTR_DONT_GET_OPT_INFO; + } + + public boolean isDontGetOptInfo() { + return (flag & NSTR_DONT_GET_OPT_INFO) != 0; + } + + public void setShared() { + flag |= NSTR_SHARED; + } + + public void clearShared() { + flag &= ~NSTR_SHARED; + } + + public boolean isShared() { + return (flag & NSTR_SHARED) != 0; + } + + public String flagsToString() { + StringBuilder flags = new StringBuilder(); + if (isRaw()) flags.append("RAW "); + if (isAmbig()) flags.append("AMBIG "); + if (isDontGetOptInfo()) flags.append("DONT_GET_OPT_INFO "); + if (isShared()) flags.append("SHARED "); + return flags.toString(); + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(); + sb.append("\n flags: " + flagsToString()); + sb.append("\n bytes: '"); + for (int i=p; i= 0x20 && (bytes[i] & 0xff) < 0x7f) { + sb.append((char)bytes[i]); + } else { + sb.append(String.format("[0x%02x]", bytes[i])); + } + } + sb.append("'"); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/bench/AbstractBench.java b/third_party/joni/src/org/joni/bench/AbstractBench.java new file mode 100644 index 000000000..7abcb51b8 --- /dev/null +++ b/third_party/joni/src/org/joni/bench/AbstractBench.java @@ -0,0 +1,50 @@ +package org.joni.bench; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; + +public abstract class AbstractBench { + protected void bench(String _reg, String _str, int warmup, int times) throws Exception { + byte[] reg = _reg.getBytes(); + byte[] str = _str.getBytes(); + + Regex p = new Regex(reg,0,reg.length,Option.DEFAULT,ASCIIEncoding.INSTANCE,Syntax.DEFAULT); + + System.err.println("::: /" + _reg + "/ =~ \"" + _str + "\", " + warmup + " * " + times + " times"); + + for(int j=0;j */ + int OP_ESC_B_WORD_BOUND = (1<<20); /* \b, \B */ + int OP_ESC_S_WHITE_SPACE = (1<<21); /* \s, \S */ + int OP_ESC_D_DIGIT = (1<<22); /* \d, \D */ + int OP_LINE_ANCHOR = (1<<23); /* ^, $ */ + int OP_POSIX_BRACKET = (1<<24); /* [:xxxx:] */ + int OP_QMARK_NON_GREEDY = (1<<25); /* ??,*?,+?,{n,m}? */ + int OP_ESC_CONTROL_CHARS = (1<<26); /* \n,\r,\t,\a ... */ + int OP_ESC_C_CONTROL = (1<<27); /* \cx */ + int OP_ESC_OCTAL3 = (1<<28); /* \OOO */ + int OP_ESC_X_HEX2 = (1<<29); /* \xHH */ + int OP_ESC_X_BRACE_HEX8 = (1<<30); /* \x{7HHHHHHH} */ + int OP_ESC_O_BRACE_OCTAL = (1<<31); /* \o{OOO} */ + + int OP2_ESC_CAPITAL_Q_QUOTE = (1<<0); /* \Q...\E */ + int OP2_QMARK_GROUP_EFFECT = (1<<1); /* (?...); */ + int OP2_OPTION_PERL = (1<<2); /* (?imsxadlu), (?-imsx), (?^imsxalu) */ + int OP2_OPTION_RUBY = (1<<3); /* (?imxadu);, (?-imx); */ + int OP2_PLUS_POSSESSIVE_REPEAT = (1<<4); /* ?+,*+,++ */ + int OP2_PLUS_POSSESSIVE_INTERVAL = (1<<5); /* {n,m}+ */ + int OP2_CCLASS_SET_OP = (1<<6); /* [...&&..[..]..] */ + int OP2_QMARK_LT_NAMED_GROUP = (1<<7); /* (?...); */ + int OP2_ESC_K_NAMED_BACKREF = (1<<8); /* \k */ + int OP2_ESC_G_SUBEXP_CALL = (1<<9); /* \g, \g */ + int OP2_ATMARK_CAPTURE_HISTORY = (1<<10); /* (?@..);,(?@..); */ + int OP2_ESC_CAPITAL_C_BAR_CONTROL = (1<<11); /* \C-x */ + int OP2_ESC_CAPITAL_M_BAR_META = (1<<12); /* \M-x */ + int OP2_ESC_V_VTAB = (1<<13); /* \v as VTAB */ + int OP2_ESC_U_HEX4 = (1<<14); /* \\uHHHH */ + int OP2_ESC_GNU_BUF_ANCHOR = (1<<15); /* \`, \' */ + int OP2_ESC_P_BRACE_CHAR_PROPERTY = (1<<16); /* \p{...}, \P{...} */ + int OP2_ESC_P_BRACE_CIRCUMFLEX_NOT = (1<<17); /* \p{^..}, \P{^..} */ + /* final int OP2_CHAR_PROPERTY_PREFIX_IS = (1<<18); */ + int OP2_ESC_H_XDIGIT = (1<<19); /* \h, \H */ + int OP2_INEFFECTIVE_ESCAPE = (1<<20); /* \ */ + int OP2_ESC_CAPITAL_R_LINEBREAK = (1<<21); /* \R as (?>\x0D\x0A|[\x0A-\x0D\x{85}\x{2028}\x{2029}]) */ + int OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER = (1<<22); /* \X as (?:\P{M}\p{M}*) */ + int OP2_ESC_V_VERTICAL_WHITESPACE = (1<<23); /* \v, \V -- Perl */ + int OP2_ESC_H_HORIZONTAL_WHITESPACE= (1<<24); /* \h, \H -- Perl */ + int OP2_ESC_CAPITAL_K_KEEP = (1<<25); /* \K */ + int OP2_ESC_G_BRACE_BACKREF = (1<<26); /* \g{name}, \g{n} */ + int OP2_QMARK_SUBEXP_CALL = (1<<27); /* (?&name), (?n), (?R), (?0) */ + int OP2_QMARK_BAR_BRANCH_RESET = (1<<28); /* (?|...) */ + int OP2_QMARK_LPAREN_CONDITION = (1<<29); /* (?(cond)yes...|no...) */ + int OP2_QMARK_CAPITAL_P_NAMED_GROUP= (1<<30); /* (?P...), (?P=name), (?P>name) -- Python/PCRE */ + int OP2_QMARK_TILDE_ABSENT = (1<<31); /* (?~...) */ + + int OP3_OPTION_JAVA = (1<<0); /* (?idmsux), (?-idmsux) */ + int OP3_OPTION_ECMASCRIPT = (1<<1); /* EcmaScript quirks */ + + /* syntax (behavior); */ + int CONTEXT_INDEP_ANCHORS = (1<<31); /* not implemented */ + int CONTEXT_INDEP_REPEAT_OPS = (1<<0); /* ?, *, +, {n,m} */ + int CONTEXT_INVALID_REPEAT_OPS = (1<<1); /* error or ignore */ + int ALLOW_UNMATCHED_CLOSE_SUBEXP = (1<<2); /* ...);... */ + int ALLOW_INVALID_INTERVAL = (1<<3); /* {??? */ + int ALLOW_INTERVAL_LOW_ABBREV = (1<<4); /* {,n} => {0,n} */ + int STRICT_CHECK_BACKREF = (1<<5); /* /(\1);/,/\1();/ ..*/ + int DIFFERENT_LEN_ALT_LOOK_BEHIND = (1<<6); /* (?<=a|bc); */ + int CAPTURE_ONLY_NAMED_GROUP = (1<<7); /* see doc/RE */ + int ALLOW_MULTIPLEX_DEFINITION_NAME = (1<<8); /* (?);(?); */ + int FIXED_INTERVAL_IS_GREEDY_ONLY = (1<<9); /* a{n}?=(?:a{n});? */ + int ALLOW_MULTIPLEX_DEFINITION_NAME_CALL = (1<<10); /* (?)(?)(?&x) */ + + /* syntax (behavior); in char class [...] */ + int NOT_NEWLINE_IN_NEGATIVE_CC = (1<<20); /* [^...] */ + int BACKSLASH_ESCAPE_IN_CC = (1<<21); /* [..\w..] etc.. */ + int ALLOW_EMPTY_RANGE_IN_CC = (1<<22); + int ALLOW_DOUBLE_RANGE_OP_IN_CC = (1<<23); /* [0-9-a]=[0-9\-a] */ + /* syntax (behavior); warning */ + int WARN_CC_OP_NOT_ESCAPED = (1<<24); /* [,-,] */ + int WARN_REDUNDANT_NESTED_REPEAT = (1<<25); /* (?:a*);+ */ + int WARN_CC_DUP = (1<<26); /* [aa] */ + + int POSIX_COMMON_OP = + OP_DOT_ANYCHAR | OP_POSIX_BRACKET | + OP_DECIMAL_BACKREF | + OP_BRACKET_CC | OP_ASTERISK_ZERO_INF | + OP_LINE_ANCHOR | + OP_ESC_CONTROL_CHARS; + + int GNU_REGEX_OP = + OP_DOT_ANYCHAR | OP_BRACKET_CC | + OP_POSIX_BRACKET | OP_DECIMAL_BACKREF | + OP_BRACE_INTERVAL | OP_LPAREN_SUBEXP | + OP_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_PLUS_ONE_INF | + OP_QMARK_ZERO_ONE | + OP_ESC_AZ_BUF_ANCHOR | OP_ESC_CAPITAL_G_BEGIN_ANCHOR | + OP_ESC_W_WORD | + OP_ESC_B_WORD_BOUND | OP_ESC_LTGT_WORD_BEGIN_END | + OP_ESC_S_WHITE_SPACE | OP_ESC_D_DIGIT | + OP_LINE_ANCHOR; + + int GNU_REGEX_BV = + CONTEXT_INDEP_ANCHORS | CONTEXT_INDEP_REPEAT_OPS | + CONTEXT_INVALID_REPEAT_OPS | ALLOW_INVALID_INTERVAL | + BACKSLASH_ESCAPE_IN_CC | ALLOW_DOUBLE_RANGE_OP_IN_CC; +} diff --git a/third_party/joni/src/org/joni/constants/internal/AnchorType.java b/third_party/joni/src/org/joni/constants/internal/AnchorType.java new file mode 100644 index 000000000..6851c995f --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/AnchorType.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface AnchorType { + int BEGIN_BUF = (1<<0); + int BEGIN_LINE = (1<<1); + int BEGIN_POSITION = (1<<2); + int END_BUF = (1<<3); + int SEMI_END_BUF = (1<<4); + int END_LINE = (1<<5); + + int WORD_BOUND = (1<<6); + int NOT_WORD_BOUND = (1<<7); + int WORD_BEGIN = (1<<8); + int WORD_END = (1<<9); + int PREC_READ = (1<<10); + int PREC_READ_NOT = (1<<11); + int LOOK_BEHIND = (1<<12); + int LOOK_BEHIND_NOT = (1<<13); + + int ANYCHAR_STAR = (1<<14); /* ".*" optimize info */ + int ANYCHAR_STAR_ML = (1<<15); /* ".*" optimize info (multi-line) */ + + int ANYCHAR_STAR_MASK = (ANYCHAR_STAR | ANYCHAR_STAR_ML); + int END_BUF_MASK = (END_BUF | SEMI_END_BUF); + + int KEEP = (1<<16); + + int ALLOWED_IN_LB = ( LOOK_BEHIND | + LOOK_BEHIND_NOT | + BEGIN_LINE | + END_LINE | + BEGIN_BUF | + BEGIN_POSITION | + KEEP | + WORD_BOUND | + NOT_WORD_BOUND | + WORD_BEGIN | + WORD_END ); + + + int ALLOWED_IN_LB_NOT = ( LOOK_BEHIND | + LOOK_BEHIND_NOT | + BEGIN_LINE | + END_LINE | + BEGIN_BUF | + BEGIN_POSITION | + KEEP | + WORD_BOUND | + NOT_WORD_BOUND | + WORD_BEGIN | + WORD_END ); +} diff --git a/third_party/joni/src/org/joni/constants/internal/Arguments.java b/third_party/joni/src/org/joni/constants/internal/Arguments.java new file mode 100644 index 000000000..02a1896d0 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/Arguments.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface Arguments { + int SPECIAL = -1; + int NON = 0; + int RELADDR = 1; + int ABSADDR = 2; + int LENGTH = 3; + int MEMNUM = 4; + int OPTION = 5; + int STATE_CHECK = 6; +} diff --git a/third_party/joni/src/org/joni/constants/internal/EncloseType.java b/third_party/joni/src/org/joni/constants/internal/EncloseType.java new file mode 100644 index 000000000..67e0b82df --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/EncloseType.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface EncloseType { + int MEMORY = 1<<0; + int OPTION = 1<<1; + int STOP_BACKTRACK = 1<<2; + int CONDITION = 1<<3; + int ABSENT = 1<<4; + + int ALLOWED_IN_LB = MEMORY | OPTION; + int ALLOWED_IN_LB_NOT = OPTION; +} diff --git a/third_party/joni/src/org/joni/constants/internal/NodeStatus.java b/third_party/joni/src/org/joni/constants/internal/NodeStatus.java new file mode 100644 index 000000000..335acd72e --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/NodeStatus.java @@ -0,0 +1,39 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface NodeStatus { + /* status bits */ + int NST_MIN_FIXED = (1<<0); + int NST_MAX_FIXED = (1<<1); + int NST_CLEN_FIXED = (1<<2); + int NST_MARK1 = (1<<3); + int NST_MARK2 = (1<<4); + int NST_MEM_BACKREFED = (1<<5); + int NST_STOP_BT_SIMPLE_REPEAT= (1<<6); + int NST_RECURSION = (1<<7); + int NST_CALLED = (1<<8); + int NST_ADDR_FIXED = (1<<9); + int NST_NAMED_GROUP = (1<<10); + int NST_NAME_REF = (1<<11); + int NST_IN_REPEAT = (1<<12); /* STK_REPEAT is nested in stack. */ + int NST_NEST_LEVEL = (1<<13); + int NST_BY_NUMBER = (1<<14); /* {n,m} */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/NodeType.java b/third_party/joni/src/org/joni/constants/internal/NodeType.java new file mode 100644 index 000000000..dd1acda70 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/NodeType.java @@ -0,0 +1,66 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface NodeType { + /* node type */ + int STR = 0; + int CCLASS = 1; + int CTYPE = 2; + int CANY = 3; + int BREF = 4; + int QTFR = 5; + int ENCLOSE = 6; + int ANCHOR = 7; + int LIST = 8; + int ALT = 9; + int CALL = 10; + + int BIT_STR = 1 << STR; + int BIT_CCLASS = 1 << CCLASS; + int BIT_CTYPE = 1 << CTYPE; + int BIT_CANY = 1 << CANY; + int BIT_BREF = 1 << BREF; + int BIT_QTFR = 1 << QTFR; + int BIT_ENCLOSE = 1 << ENCLOSE; + int BIT_ANCHOR = 1 << ANCHOR; + int BIT_LIST = 1 << LIST; + int BIT_ALT = 1 << ALT; + int BIT_CALL = 1 << CALL; + + /* allowed node types in look-behind */ + int ALLOWED_IN_LB = ( BIT_LIST | + BIT_ALT | + BIT_STR | + BIT_CCLASS | + BIT_CTYPE | + BIT_CANY | + BIT_ANCHOR | + BIT_ENCLOSE | + BIT_QTFR | + BIT_CALL ); + + int SIMPLE = ( BIT_STR | + BIT_CCLASS | + BIT_CTYPE | + BIT_CANY | + BIT_BREF); + +} diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java new file mode 100644 index 000000000..4029d2d20 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -0,0 +1,355 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +import org.joni.Config; + +public interface OPCode { + int FINISH = 0; /* matching process terminator (no more alternative) */ + int END = 1; /* pattern code terminator (success end) */ + + int EXACT1 = 2; /* single byte, N = 1 */ + int EXACT2 = 3; /* single byte, N = 2 */ + int EXACT3 = 4; /* single byte, N = 3 */ + int EXACT4 = 5; /* single byte, N = 4 */ + int EXACT5 = 6; /* single byte, N = 5 */ + int EXACTN = 7; /* single byte */ + int EXACTMB2N1 = 8; /* mb-length = 2 N = 1 */ + int EXACTMB2N2 = 9; /* mb-length = 2 N = 2 */ + int EXACTMB2N3 = 10; /* mb-length = 2 N = 3 */ + int EXACTMB2N = 11; /* mb-length = 2 */ + int EXACTMB3N = 12; /* mb-length = 3 */ + int EXACTMBN = 13; /* other length */ + + int EXACT1_IC = 14; /* single byte, N = 1, ignore case */ + int EXACTN_IC = 15; /* single byte, ignore case */ + + int CCLASS = 16; + int CCLASS_MB = 17; + int CCLASS_MIX = 18; + int CCLASS_NOT = 19; + int CCLASS_MB_NOT = 20; + int CCLASS_MIX_NOT = 21; + + int ANYCHAR = 22; /* "." */ + int ANYCHAR_ML = 23; /* "." multi-line */ + int ANYCHAR_STAR = 24; /* ".*" */ + int ANYCHAR_ML_STAR = 25; /* ".*" multi-line */ + int ANYCHAR_STAR_PEEK_NEXT = 26; + int ANYCHAR_ML_STAR_PEEK_NEXT = 27; + + int WORD = 28; + int NOT_WORD = 29; + int WORD_BOUND = 30; + int NOT_WORD_BOUND = 31; + int WORD_BEGIN = 32; + int WORD_END = 33; + + int ASCII_WORD = 34; + int ASCII_NOT_WORD = 35; + int ASCII_WORD_BOUND = 36; + int ASCII_NOT_WORD_BOUND = 37; + int ASCII_WORD_BEGIN = 38; + int ASCII_WORD_END = 39; + + int BEGIN_BUF = 40; + int END_BUF = 41; + int BEGIN_LINE = 42; + int END_LINE = 43; + int SEMI_END_BUF = 44; + int BEGIN_POSITION = 45; + + int BACKREF1 = 46; + int BACKREF2 = 47; + int BACKREFN = 48; + int BACKREFN_IC = 49; + int BACKREF_MULTI = 50; + int BACKREF_MULTI_IC = 51; + int BACKREF_WITH_LEVEL = 52; /* \k, \k */ + + int MEMORY_START = 53; + int MEMORY_START_PUSH = 54; /* push back-tracker to stack */ + int MEMORY_END_PUSH = 55; /* push back-tracker to stack */ + int MEMORY_END_PUSH_REC = 56; /* push back-tracker to stack */ + int MEMORY_END = 57; + int MEMORY_END_REC = 58; /* push marker to stack */ + + int KEEP = 59; + int FAIL = 60; /* pop stack and move */ + int JUMP = 61; + int PUSH = 62; + int POP = 63; + int PUSH_OR_JUMP_EXACT1 = 64; /* if match exact then push, else jump. */ + int PUSH_IF_PEEK_NEXT = 65; /* if match exact then push, else none. */ + + int REPEAT = 66; /* {n,m} */ + int REPEAT_NG = 67; /* {n,m}? (non greedy) */ + int REPEAT_INC = 68; + int REPEAT_INC_NG = 69; /* non greedy */ + int REPEAT_INC_SG = 70; /* search and get in stack */ + int REPEAT_INC_NG_SG = 71; /* search and get in stack (non greedy) */ + + int NULL_CHECK_START = 72; /* null loop checker start */ + int NULL_CHECK_END = 73; /* null loop checker end */ + int NULL_CHECK_END_MEMST = 74; /* null loop checker end (with capture status) */ + int NULL_CHECK_END_MEMST_PUSH = 75; /* with capture status and push check-end */ + + int PUSH_POS = 76; /* (?=...) start */ + int POP_POS = 77; /* (?=...) end */ + int PUSH_POS_NOT = 78; /* (?!...) start */ + int FAIL_POS = 79; /* (?!...) end */ + int PUSH_STOP_BT = 80; /* (?>...) start */ + int POP_STOP_BT = 81; /* (?>...) end */ + int LOOK_BEHIND = 82; /* (?<=...) start (no needs end opcode) */ + int PUSH_LOOK_BEHIND_NOT = 83; /* (? */ + int RETURN = 89; + int CONDITION = 90; + + int STATE_CHECK_PUSH = 91; /* combination explosion check and push */ + int STATE_CHECK_PUSH_OR_JUMP = 92; /* check ok -> push, else jump */ + int STATE_CHECK = 93; /* check only */ + int STATE_CHECK_ANYCHAR_STAR = 94; + int STATE_CHECK_ANYCHAR_ML_STAR = 95; + + /* no need: IS_DYNAMIC_OPTION() == 0 */ + int SET_OPTION_PUSH = 96; /* set option and push recover option */ + int SET_OPTION = 97; /* set option */ + + int EXACT1_IC_SB = 98; /* single byte, N = 1, ignore case */ + int EXACTN_IC_SB = 99; /* single byte, ignore case */ + int CALLOUT = 100; /* generic match-time callout */ + + String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { + "finish", /*OP_FINISH*/ + "end", /*OP_END*/ + "exact1", /*OP_EXACT1*/ + "exact2", /*OP_EXACT2*/ + "exact3", /*OP_EXACT3*/ + "exact4", /*OP_EXACT4*/ + "exact5", /*OP_EXACT5*/ + "exactn", /*OP_EXACTN*/ + "exactmb2-n1", /*OP_EXACTMB2N1*/ + "exactmb2-n2", /*OP_EXACTMB2N2*/ + "exactmb2-n3", /*OP_EXACTMB2N3*/ + "exactmb2-n", /*OP_EXACTMB2N*/ + "exactmb3n", /*OP_EXACTMB3N*/ + "exactmbn", /*OP_EXACTMBN*/ + "exact1-ic", /*OP_EXACT1_IC*/ + "exactn-ic", /*OP_EXACTN_IC*/ + "cclass", /*OP_CCLASS*/ + "cclass-mb", /*OP_CCLASS_MB*/ + "cclass-mix", /*OP_CCLASS_MIX*/ + "cclass-not", /*OP_CCLASS_NOT*/ + "cclass-mb-not", /*OP_CCLASS_MB_NOT*/ + "cclass-mix-not", /*OP_CCLASS_MIX_NOT*/ + "anychar", /*OP_ANYCHAR*/ + "anychar-ml", /*OP_ANYCHAR_ML*/ + "anychar*", /*OP_ANYCHAR_STAR*/ + "anychar-ml*", /*OP_ANYCHAR_ML_STAR*/ + "anychar*-peek-next", /*OP_ANYCHAR_STAR_PEEK_NEXT*/ + "anychar-ml*-peek-next", /*OP_ANYCHAR_ML_STAR_PEEK_NEXT*/ + "word", /*OP_WORD*/ + "not-word", /*OP_NOT_WORD*/ + "word-bound", /*OP_WORD_BOUND*/ + "not-word-bound", /*OP_NOT_WORD_BOUND*/ + "word-begin", /*OP_WORD_BEGIN*/ + "word-end", /*OP_WORD_END*/ + "ascii-word", /*OP_ASCII_WORD*/ + "not-ascii-word", /*OP_NOT_ASCII_WORD*/ + "ascii-word-bound", /*OP_ASCII_WORD_BOUND*/ + "not-ascii-word-bound", /*OP_NOT_ASCII_WORD_BOUND*/ + "ascii-word-begin", /*OP_ASCII_WORD_BEGIN*/ + "ascii-word-end", /*OP_ASCII_WORD_END*/ + "begin-buf", /*OP_BEGIN_BUF*/ + "end-buf", /*OP_END_BUF*/ + "begin-line", /*OP_BEGIN_LINE*/ + "end-line", /*OP_END_LINE*/ + "semi-end-buf", /*OP_SEMI_END_BUF*/ + "begin-position", /*OP_BEGIN_POSITION*/ + "backref1", /*OP_BACKREF1*/ + "backref2", /*OP_BACKREF2*/ + "backrefn", /*OP_BACKREFN*/ + "backrefn-ic", /*OP_BACKREFN_IC*/ + "backref_multi", /*OP_BACKREF_MULTI*/ + "backref_multi-ic", /*OP_BACKREF_MULTI_IC*/ + "backref_at_level", /*OP_BACKREF_AT_LEVEL*/ + "mem-start", /*OP_MEMORY_START*/ + "mem-start-push", /*OP_MEMORY_START_PUSH*/ + "mem-end-push", /*OP_MEMORY_END_PUSH*/ + "mem-end-push-rec", /*OP_MEMORY_END_PUSH_REC*/ + "mem-end", /*OP_MEMORY_END*/ + "mem-end-rec", /*OP_MEMORY_END_REC*/ + "keep", /*OP_KEEP*/ + "fail", /*OP_FAIL*/ + "jump", /*OP_JUMP*/ + "push", /*OP_PUSH*/ + "pop", /*OP_POP*/ + "push-or-jump-e1", /*OP_PUSH_OR_JUMP_EXACT1*/ + "push-if-peek-next", /*OP_PUSH_IF_PEEK_NEXT*/ + "repeat", /*OP_REPEAT*/ + "repeat-ng", /*OP_REPEAT_NG*/ + "repeat-inc", /*OP_REPEAT_INC*/ + "repeat-inc-ng", /*OP_REPEAT_INC_NG*/ + "repeat-inc-sg", /*OP_REPEAT_INC_SG*/ + "repeat-inc-ng-sg", /*OP_REPEAT_INC_NG_SG*/ + "null-check-start", /*OP_NULL_CHECK_START*/ + "null-check-end", /*OP_NULL_CHECK_END*/ + "null-check-end-memst", /*OP_NULL_CHECK_END_MEMST*/ + "null-check-end-memst-push", /*OP_NULL_CHECK_END_MEMST_PUSH*/ + "push-pos", /*OP_PUSH_POS*/ + "pop-pos", /*OP_POP_POS*/ + "push-pos-not", /*OP_PUSH_POS_NOT*/ + "fail-pos", /*OP_FAIL_POS*/ + "push-stop-bt", /*OP_PUSH_STOP_BT*/ + "pop-stop-bt", /*OP_POP_STOP_BT*/ + "look-behind", /*OP_LOOK_BEHIND*/ + "push-look-behind-not", /*OP_PUSH_LOOK_BEHIND_NOT*/ + "fail-look-behind-not", /*OP_FAIL_LOOK_BEHIND_NOT*/ + "push-absent-pos", /*OP_PUSH_ABSENT_POS*/ + "absent", /*OP_ABSENT*/ + "absent-end", /*OP_ABSENT_END*/ + "call", /*OP_CALL*/ + "return", /*OP_RETURN*/ + "condition", /*OP_CONDITION*/ + "state-check-push", /*OP_STATE_CHECK_PUSH*/ + "state-check-push-or-jump", /*OP_STATE_CHECK_PUSH_OR_JUMP*/ + "state-check", /*OP_STATE_CHECK*/ + "state-check-anychar*", /*OP_STATE_CHECK_ANYCHAR_STAR*/ + "state-check-anychar-ml*", /*OP_STATE_CHECK_ANYCHAR_ML_STAR*/ + "set-option-push", /*OP_SET_OPTION_PUSH*/ + "set-option", /*OP_SET_OPTION*/ + + "exact1-ic-sb", /*OP_EXACT1_IC*/ + "exactn-ic-sb", /*OP_EXACTN_IC*/ + "callout", /*OP_CALLOUT*/ + } : null; + + int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { + Arguments.NON, /*OP_FINISH*/ + Arguments.NON, /*OP_END*/ + Arguments.SPECIAL, /*OP_EXACT1*/ + Arguments.SPECIAL, /*OP_EXACT2*/ + Arguments.SPECIAL, /*OP_EXACT3*/ + Arguments.SPECIAL, /*OP_EXACT4*/ + Arguments.SPECIAL, /*OP_EXACT5*/ + Arguments.SPECIAL, /*OP_EXACTN*/ + Arguments.SPECIAL, /*OP_EXACTMB2N1*/ + Arguments.SPECIAL, /*OP_EXACTMB2N2*/ + Arguments.SPECIAL, /*OP_EXACTMB2N3*/ + Arguments.SPECIAL, /*OP_EXACTMB2N*/ + Arguments.SPECIAL, /*OP_EXACTMB3N*/ + Arguments.SPECIAL, /*OP_EXACTMBN*/ + Arguments.SPECIAL, /*OP_EXACT1_IC*/ + Arguments.SPECIAL, /*OP_EXACTN_IC*/ + Arguments.SPECIAL, /*OP_CCLASS*/ + Arguments.SPECIAL, /*OP_CCLASS_MB*/ + Arguments.SPECIAL, /*OP_CCLASS_MIX*/ + Arguments.SPECIAL, /*OP_CCLASS_NOT*/ + Arguments.SPECIAL, /*OP_CCLASS_MB_NOT*/ + Arguments.SPECIAL, /*OP_CCLASS_MIX_NOT*/ + Arguments.NON, /*OP_ANYCHAR*/ + Arguments.NON, /*OP_ANYCHAR_ML*/ + Arguments.NON, /*OP_ANYCHAR_STAR*/ + Arguments.NON, /*OP_ANYCHAR_ML_STAR*/ + Arguments.SPECIAL, /*OP_ANYCHAR_STAR_PEEK_NEXT*/ + Arguments.SPECIAL, /*OP_ANYCHAR_ML_STAR_PEEK_NEXT*/ + Arguments.NON, /*OP_WORD*/ + Arguments.NON, /*OP_NOT_WORD*/ + Arguments.NON, /*OP_WORD_BOUND*/ + Arguments.NON, /*OP_NOT_WORD_BOUND*/ + Arguments.NON, /*OP_WORD_BEGIN*/ + Arguments.NON, /*OP_WORD_END*/ + Arguments.NON, /*OP_ASCII_WORD*/ + Arguments.NON, /*OP_NOT_ASCII_WORD*/ + Arguments.NON, /*OP_ASCII_WORD_BOUND*/ + Arguments.NON, /*OP_NOT_ASCII_WORD_BOUND*/ + Arguments.NON, /*OP_ASCII_WORD_BEGIN*/ + Arguments.NON, /*OP_ASCII_WORD_END*/ + Arguments.NON, /*OP_BEGIN_BUF*/ + Arguments.NON, /*OP_END_BUF*/ + Arguments.NON, /*OP_BEGIN_LINE*/ + Arguments.NON, /*OP_END_LINE*/ + Arguments.NON, /*OP_SEMI_END_BUF*/ + Arguments.NON, /*OP_BEGIN_POSITION*/ + Arguments.NON, /*OP_BACKREF1*/ + Arguments.NON, /*OP_BACKREF2*/ + Arguments.MEMNUM, /*OP_BACKREFN*/ + Arguments.SPECIAL, /*OP_BACKREFN_IC*/ + Arguments.SPECIAL, /*OP_BACKREF_MULTI*/ + Arguments.SPECIAL, /*OP_BACKREF_MULTI_IC*/ + Arguments.SPECIAL, /*OP_BACKREF_AT_LEVEL*/ + Arguments.MEMNUM, /*OP_MEMORY_START*/ + Arguments.MEMNUM, /*OP_MEMORY_START_PUSH*/ + Arguments.MEMNUM, /*OP_MEMORY_END_PUSH*/ + Arguments.MEMNUM, /*OP_MEMORY_END_PUSH_REC*/ + Arguments.MEMNUM, /*OP_MEMORY_END*/ + Arguments.MEMNUM, /*OP_MEMORY_END_REC*/ + Arguments.NON, /*OP_KEEP*/ + Arguments.NON, /*OP_FAIL*/ + Arguments.RELADDR, /*OP_JUMP*/ + Arguments.RELADDR, /*OP_PUSH*/ + Arguments.NON, /*OP_POP*/ + Arguments.SPECIAL, /*OP_PUSH_OR_JUMP_EXACT1*/ + Arguments.SPECIAL, /*OP_PUSH_IF_PEEK_NEXT*/ + Arguments.SPECIAL, /*OP_REPEAT*/ + Arguments.SPECIAL, /*OP_REPEAT_NG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_NG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_SG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_NG_SG*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_START*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END_MEMST*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END_MEMST_PUSH*/ + Arguments.NON, /*OP_PUSH_POS*/ + Arguments.NON, /*OP_POP_POS*/ + Arguments.RELADDR, /*OP_PUSH_POS_NOT*/ + Arguments.NON, /*OP_FAIL_POS*/ + Arguments.NON, /*OP_PUSH_STOP_BT*/ + Arguments.NON, /*OP_POP_STOP_BT*/ + Arguments.SPECIAL, /*OP_LOOK_BEHIND*/ + Arguments.SPECIAL, /*OP_PUSH_LOOK_BEHIND_NOT*/ + Arguments.NON, /*OP_FAIL_LOOK_BEHIND_NOT*/ + Arguments.NON, /*OP_PUSH_ABSENT_POS*/ + Arguments.RELADDR, /*OP_ABSENT*/ + Arguments.NON, /*OP_ABSENT_END*/ + Arguments.ABSADDR, /*OP_CALL*/ + Arguments.NON, /*OP_RETURN*/ + Arguments.SPECIAL, /*OP_CONDITION*/ + Arguments.SPECIAL, /*OP_STATE_CHECK_PUSH*/ + Arguments.SPECIAL, /*OP_STATE_CHECK_PUSH_OR_JUMP*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK_ANYCHAR_STAR*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK_ANYCHAR_ML_STAR*/ + Arguments.OPTION, /*OP_SET_OPTION_PUSH*/ + Arguments.OPTION, /*OP_SET_OPTION*/ + + Arguments.SPECIAL, /*OP_EXACT1_IC*/ + Arguments.SPECIAL, /*OP_EXACTN_IC*/ + Arguments.MEMNUM, /*OP_CALLOUT*/ + } : null; +} diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java new file mode 100644 index 000000000..de452b5fc --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -0,0 +1,81 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface OPSize { + + // this might be helpful for potential byte[] migration + int OPCODE = 1; + int RELADDR = 1; + int ABSADDR = 1; + int LENGTH = 1; + int MEMNUM = 1; + int STATE_CHECK_NUM = 1; + int REPEATNUM = 1; + int OPTION = 1; + int CODE_POINT = 1; + int POINTER = 1; + int INDEX = 1; + + /* op-code + arg size */ + + int ANYCHAR_STAR = OPCODE; + int ANYCHAR_STAR_PEEK_NEXT = (OPCODE + 1); + int JUMP = (OPCODE + RELADDR); + int PUSH = (OPCODE + RELADDR); + int POP = OPCODE; + int PUSH_OR_JUMP_EXACT1 = (OPCODE + RELADDR + 1); + int PUSH_IF_PEEK_NEXT = (OPCODE + RELADDR + 1); + int REPEAT_INC = (OPCODE + MEMNUM); + int REPEAT_INC_NG = (OPCODE + MEMNUM); + int PUSH_POS = OPCODE; + int PUSH_POS_NOT = (OPCODE + RELADDR); + int POP_POS = OPCODE; + int FAIL_POS = OPCODE; + int SET_OPTION = (OPCODE + OPTION); + int SET_OPTION_PUSH = (OPCODE + OPTION); + int FAIL = OPCODE; + int MEMORY_START = (OPCODE + MEMNUM); + int MEMORY_START_PUSH = (OPCODE + MEMNUM); + int MEMORY_END_PUSH = (OPCODE + MEMNUM); + int MEMORY_END_PUSH_REC = (OPCODE + MEMNUM); + int MEMORY_END = (OPCODE + MEMNUM); + int MEMORY_END_REC = (OPCODE + MEMNUM); + int PUSH_STOP_BT = OPCODE; + int POP_STOP_BT = OPCODE; + int NULL_CHECK_START = (OPCODE + MEMNUM); + int NULL_CHECK_END = (OPCODE + MEMNUM); + int LOOK_BEHIND = (OPCODE + LENGTH); + int PUSH_LOOK_BEHIND_NOT = (OPCODE + RELADDR + LENGTH); + int FAIL_LOOK_BEHIND_NOT = OPCODE; + int CALL = (OPCODE + ABSADDR); + int RETURN = OPCODE; + int CONDITION = (OPCODE + MEMNUM + RELADDR); + int PUSH_ABSENT_POS = OPCODE; + int ABSENT = (OPCODE + RELADDR); + int ABSENT_END = OPCODE; + int CALLOUT = (OPCODE + MEMNUM); + + // #ifdef USE_COMBINATION_EXPLOSION_CHECK + int STATE_CHECK = (OPCODE + STATE_CHECK_NUM); + int STATE_CHECK_PUSH = (OPCODE + STATE_CHECK_NUM + RELADDR); + int STATE_CHECK_PUSH_OR_JUMP = (OPCODE + STATE_CHECK_NUM + RELADDR); + int STATE_CHECK_ANYCHAR_STAR = (OPCODE + STATE_CHECK_NUM); +} diff --git a/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java b/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java new file mode 100644 index 000000000..c27ad5593 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StackPopLevel { + int FREE = 0; + int MEM_START = 1; + int ALL = 2; + +} diff --git a/third_party/joni/src/org/joni/constants/internal/StackType.java b/third_party/joni/src/org/joni/constants/internal/StackType.java new file mode 100644 index 000000000..8f7224076 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StackType.java @@ -0,0 +1,54 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StackType { + /** stack **/ + int INVALID_STACK_INDEX = -1; + + /* stack type */ + /* used by normal-POP */ + int ALT = 0x0001; + int LOOK_BEHIND_NOT = 0x0002; + int POS_NOT = 0x0003; + /* handled by normal-POP */ + int MEM_START = 0x0100; + int MEM_END = 0x8200; + int REPEAT_INC = 0x0300; + int STATE_CHECK_MARK = 0x1000; + /* avoided by normal-POP */ + int NULL_CHECK_START = 0x3000; + int NULL_CHECK_END = 0x5000; /* for recursive call */ + int MEM_END_MARK = 0x8400; + int POS = 0x0500; /* used when POP-POS */ + int STOP_BT = 0x0600; /* mark for "(?>...)" */ + int REPEAT = 0x0700; + int CALL_FRAME = 0x0800; + int RETURN = 0x0900; + int VOID = 0x0a00; /* for fill a blank */ + int ABSENT_POS = 0x0b00; /* for absent */ + int ABSENT = 0x0c00; /* absent inner loop marker */ + int CALLOUT = 0x0d00; /* match-time callback unwind token */ + + /* stack type check mask */ + int MASK_POP_USED = 0x00ff; + int MASK_TO_VOID_TARGET = 0x10ff; + int MASK_MEM_END_OR_MARK = 0x8000; /* MEM_END or MEM_END_MARK */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/StringType.java b/third_party/joni/src/org/joni/constants/internal/StringType.java new file mode 100644 index 000000000..032e452a2 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StringType.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StringType { + int NSTR_RAW = 1<<0; + int NSTR_AMBIG = 1<<1; + int NSTR_DONT_GET_OPT_INFO = 1<<2; + int NSTR_SHARED = 1<<3; +} diff --git a/third_party/joni/src/org/joni/constants/internal/TargetInfo.java b/third_party/joni/src/org/joni/constants/internal/TargetInfo.java new file mode 100644 index 000000000..cad12f2d8 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/TargetInfo.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface TargetInfo { + int ISNOT_EMPTY = 0; + int IS_EMPTY = 1; + int IS_EMPTY_MEM = 2; + int IS_EMPTY_REC = 3; +} diff --git a/third_party/joni/src/org/joni/constants/internal/TokenType.java b/third_party/joni/src/org/joni/constants/internal/TokenType.java new file mode 100644 index 000000000..2a4891765 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/TokenType.java @@ -0,0 +1,51 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public enum TokenType { + EOT, /* end of token */ + RAW_BYTE, + CHAR, + STRING, + CODE_POINT, + ANYCHAR, + CHAR_TYPE, + BACKREF, + CALL, + ANCHOR, + OP_REPEAT, + INTERVAL, + ANYCHAR_ANYTIME, /* SQL '%' == .* */ + ALT, + SUBEXP_OPEN, + SUBEXP_CLOSE, + CC_OPEN, + QUOTE_OPEN, + CHAR_PROPERTY, /* \p{...}, \P{...} */ + LINEBREAK, + EXTENDED_GRAPHEME_CLUSTER, + KEEP, + /* in cc */ + CC_CLOSE, + CC_RANGE, + POSIX_BRACKET_OPEN, + CC_AND, /* && */ + CC_CC_OPEN /* [ */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/Traverse.java b/third_party/joni/src/org/joni/constants/internal/Traverse.java new file mode 100644 index 000000000..5fab1ec16 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/Traverse.java @@ -0,0 +1,26 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface Traverse { + int TRAVERSE_CALLBACK_AT_FIRST = 1; + int TRAVERSE_CALLBACK_AT_LAST = 2; + int TRAVERSE_CALLBACK_AT_BOTH = TRAVERSE_CALLBACK_AT_FIRST | TRAVERSE_CALLBACK_AT_LAST; +} diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java new file mode 100644 index 000000000..b80340b72 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -0,0 +1,89 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +import org.joni.Config; + +public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { + /* internal error */ + String PARSER_BUG = "internal parser error (bug)"; + String UNDEFINED_BYTECODE = "undefined bytecode (bug)"; + String UNEXPECTED_BYTECODE = "unexpected bytecode (bug)"; + String TOO_MANY_CAPTURE_GROUPS = "too many capture groups are specified"; + + /* general error */ + String INVALID_ARGUMENT = "invalid argument"; + + /* syntax error */ + String REGEX_TOO_LONG = "regex length too long"; + String END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace"; + String END_PATTERN_AT_LEFT_BRACKET = "end pattern at left bracket"; + String EMPTY_CHAR_CLASS = "empty char-class"; + String PREMATURE_END_OF_CHAR_CLASS = "premature end of char-class"; + String END_PATTERN_AT_ESCAPE = "end pattern at escape"; + String END_PATTERN_AT_META = "end pattern at meta"; + String END_PATTERN_AT_CONTROL = "end pattern at control"; + String META_CODE_SYNTAX = "invalid meta-code syntax"; + String CONTROL_CODE_SYNTAX = "invalid control-code syntax"; + String CHAR_CLASS_VALUE_AT_END_OF_RANGE = "char-class value at end of range"; + String CHAR_CLASS_VALUE_AT_START_OF_RANGE = "char-class value at start of range"; + String UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS = "unmatched range specifier in char-class"; + String TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED = "target of repeat operator is not specified"; + String TARGET_OF_REPEAT_OPERATOR_INVALID = "target of repeat operator is invalid"; + String NESTED_REPEAT_NOT_ALLOWED = "nested repeat is not allowed"; + String NESTED_REPEAT_OPERATOR = "nested repeat operator"; + String UNMATCHED_CLOSE_PARENTHESIS = "unmatched close parenthesis"; + String END_PATTERN_WITH_UNMATCHED_PARENTHESIS = "end pattern with unmatched parenthesis"; + String END_PATTERN_IN_GROUP = "end pattern in group"; + String UNDEFINED_GROUP_OPTION = "undefined group option"; + String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; + String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; + String INVALID_REPEAT_RANGE_PATTERN = "invalid repeat range {lower,upper}"; + String INVALID_CONDITION_PATTERN = "invalid conditional pattern"; + + /* values error (syntax error) */ + String TOO_BIG_NUMBER = "too big number"; + String TOO_BIG_NUMBER_FOR_REPEAT_RANGE = "too big number for repeat range"; + String UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE = "upper is smaller than lower in repeat range"; + String EMPTY_RANGE_IN_CHAR_CLASS = "empty range in char class"; + String MISMATCH_CODE_LENGTH_IN_CLASS_RANGE = "mismatch multibyte code length in char-class range"; + String TOO_MANY_MULTI_BYTE_RANGES = "too many multibyte code ranges are specified"; + String TOO_SHORT_MULTI_BYTE_STRING = "too short multibyte code string"; + String TOO_BIG_BACKREF_NUMBER = "too big backref number"; + String INVALID_BACKREF = Config.USE_NAMED_GROUP ? "invalid backref number/name" : "invalid backref number"; + String NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED = "numbered backref/call is not allowed. (use name)"; + String TOO_SHORT_DIGITS = "too short digits"; + String INVALID_WIDE_CHAR_VALUE = "invalid wide-char value"; + String EMPTY_GROUP_NAME = "group name is empty"; + String INVALID_GROUP_NAME = "invalid group name <%n>"; + String INVALID_CHAR_IN_GROUP_NAME = Config.USE_NAMED_GROUP ? "invalid char in group name <%n>" : "invalid char in group number <%n>"; + String UNDEFINED_NAME_REFERENCE = "undefined name <%n> reference"; + String UNDEFINED_GROUP_REFERENCE = "undefined group <%n> reference"; + String MULTIPLEX_DEFINED_NAME = "multiplex defined name <%n>"; + String MULTIPLEX_DEFINITION_NAME_CALL = "multiplex definition name <%n> call"; + String PROPERTY_NAME_NEVER_TERMINATED = "property name never terminated \\p{%n"; + String NEVER_ENDING_RECURSION = "never ending recursion"; + String GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY = "group number is too big for capture history"; + String NOT_SUPPORTED_ENCODING_COMBINATION = "not supported encoding combination"; + String INVALID_COMBINATION_OF_OPTIONS = "invalid combination of options"; + String OVER_THREAD_PASS_LIMIT_COUNT = "over thread pass limit count"; + String TOO_BIG_SB_CHAR_VALUE = "too big singlebyte char value"; + +} diff --git a/third_party/joni/src/org/joni/exception/InternalException.java b/third_party/joni/src/org/joni/exception/InternalException.java new file mode 100644 index 000000000..b41e0aaf3 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/InternalException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class InternalException extends JOniException{ + private static final long serialVersionUID = -3871816465397927992L; + + public InternalException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/JOniException.java b/third_party/joni/src/org/joni/exception/JOniException.java new file mode 100644 index 000000000..c2c5d4794 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/JOniException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class JOniException extends RuntimeException{ + private static final long serialVersionUID = -6027192180014164667L; + + public JOniException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/SyntaxException.java b/third_party/joni/src/org/joni/exception/SyntaxException.java new file mode 100644 index 000000000..7e50cf5ba --- /dev/null +++ b/third_party/joni/src/org/joni/exception/SyntaxException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class SyntaxException extends JOniException{ + private static final long serialVersionUID = 7862720128961874288L; + + public SyntaxException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/TimeoutException.java b/third_party/joni/src/org/joni/exception/TimeoutException.java new file mode 100644 index 000000000..914892540 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/TimeoutException.java @@ -0,0 +1,9 @@ +package org.joni.exception; + +public class TimeoutException extends InterruptedException { + private static final long serialVersionUID = 1L; + + public TimeoutException() { + super(); + } +} diff --git a/third_party/joni/src/org/joni/exception/ValueException.java b/third_party/joni/src/org/joni/exception/ValueException.java new file mode 100644 index 000000000..790f3546c --- /dev/null +++ b/third_party/joni/src/org/joni/exception/ValueException.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class ValueException extends SyntaxException{ + private static final long serialVersionUID = -196013852479929134L; + + public ValueException(String message) { + super(message); + } + + public ValueException(String message, String str) { + super(message.replaceAll("%n", str)); + } + + public ValueException(String message, byte[]bytes, int p, int end) { + this(message, new String(bytes, p, end - p)); + } + +} diff --git a/third_party/joni/test/org/joni/test/Test.java b/third_party/joni/test/org/joni/test/Test.java new file mode 100644 index 000000000..91c00be66 --- /dev/null +++ b/third_party/joni/test/org/joni/test/Test.java @@ -0,0 +1,303 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.io.UnsupportedEncodingException; + +import org.jcodings.Encoding; +import org.jcodings.exception.CharacterPropertyException; +import org.joni.Config; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Region; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.JOniException; + +public abstract class Test { + static final boolean VERBOSE = false; + + int nsucc; + int nerror; + int nfail; + + public abstract int option(); + public abstract Encoding encoding(); + public abstract String testEncoding(); + public abstract Syntax syntax(); + + protected String repr(byte[] bytes) { + return new String(bytes); + } + + protected int length(byte[] bytes) { + return bytes.length; + } + + protected String reprTest(byte[] pattern, byte[]str, int option) { + StringBuilder sb = new StringBuilder(); + sb.append("Pattern: [/").append(repr(pattern)).append("/]"); + sb.append(" Str: [\"").append(repr(str)).append("\"]"); + sb.append(" Encoding: [" + encoding() + "]"); + sb.append(" Option: [" + Option.toString(option) + "]"); + sb.append(" Syntax: [" + syntax().name + "]"); + return sb.toString(); + } + + protected void assertTrue(boolean expression, String failMessage) { + if (expression) { + nsucc++; + } else { + Config.err.println(failMessage); + nfail++; + } + } + + public void xerrs(String pattern, String msg) throws Exception { + xerr(pattern.getBytes(testEncoding()), msg, option()); + } + + public void xerrs(String patternIsSubstring, int p, int end, String msg, WarnCallback warnCallback) throws Exception { + xerrs(patternIsSubstring, p, end, msg, option(), warnCallback); + } + + public void xerrs(String pattern, String msg, int option) throws Exception { + xerr(pattern.getBytes(testEncoding()), msg, option); + } + + public void xerrs(String patternIsSubstring, int p, int end, String msg, int option, WarnCallback warnCallback) throws Exception { + p = length(patternIsSubstring.substring(0, p).getBytes(testEncoding())); + end = length(patternIsSubstring.substring(0, end).getBytes(testEncoding())); + xerr(patternIsSubstring.getBytes(testEncoding()), p, end, msg, option, warnCallback); + } + + public void xerr(byte[] pattern, String msg, int option) throws Exception { + xerr(pattern, 0, length(pattern), msg, option, WarnCallback.NONE); + } + + public void xerr(byte[] pattern, int p, int end, String msg, int option, WarnCallback warnCallback) throws Exception { + try { + new Regex(pattern, p, end, option, encoding(), syntax(), warnCallback); + nfail++; + } catch (JOniException | CharacterPropertyException cpe) { + nsucc++; + assertEquals(cpe.getMessage(), msg); + } + } + + public void xx(byte[] pattern, byte[] str, int from, int to, int mem, boolean not) throws InterruptedException { + xx(pattern, str, from, to, mem, not, option()); + } + + public void xx(byte[] pattern, byte[] str, int gpos, int from, int to, int mem, boolean not) throws InterruptedException { + xx(pattern, str, gpos, 0, from, to, mem, not, option()); + } + + static boolean is7bit(byte[]bytes, int p, int end) { + for (int i = p; i < end; i++) { + if ((bytes[i] & 0xff) >= 0x80) return false; + } + return true; + } + + public int xx(byte[] pattern, byte[] str, int from, int to, int mem, boolean not, int option) throws InterruptedException { + return xx(pattern, str, 0, 0, from, to, mem, not, option); + } + + public int xx(byte[] pattern, byte[] str, int gpos, int searchStart, int from, int to, int mem, boolean not, int option) throws InterruptedException { + return xx(pattern, str, 0, length(pattern), gpos, searchStart, from, to, mem, not, option, WarnCallback.NONE); + } + + public int xx(byte[] pattern, byte[] str, int p, int end, int gpos, int searchStart, int from, int to, int mem, boolean not, int option, WarnCallback warnCallback) throws InterruptedException { + Regex reg; + + try { + reg = new Regex(pattern, p, end, option, encoding(), syntax(), warnCallback); + } catch (JOniException je) { + Config.err.println(reprTest(pattern, str, option)); + je.printStackTrace(Config.err); + Config.err.println("ERROR: " + je.getMessage()); + nerror++; + return Matcher.FAILED; + } catch (Exception e) { + Config.err.println(reprTest(pattern, str, option)); + e.printStackTrace(Config.err); + Config.err.println("SEVERE ERROR: " + e.getMessage()); + nerror++; + return Matcher.FAILED; + } + + if ((!encoding().isSingleByte()) && encoding().isAsciiCompatible() && is7bit(str, 0, str.length)) { + check(reg, pattern, str, option | Option.CR_7_BIT, gpos, searchStart, from, to, mem, not); + } + + return check(reg, pattern, str, option, gpos, searchStart, from, to, mem, not); + } + + private int check(Regex reg, byte[]pattern, byte[] str, int option, int gpos, int searchStart, int from, int to, int mem, boolean not) throws InterruptedException { + Matcher m = reg.matcher(str, 0, length(str)); + final Region region; + final int result; + try { + result = m.searchInterruptible(gpos, searchStart, length(str), option); + region = m.getEagerRegion(); + } catch (JOniException je) { + Config.err.println("Pattern: " + reprTest(pattern, str, option)); + je.printStackTrace(Config.err); + Config.err.println("ERROR: " + je.getMessage()); + nerror++; + return Matcher.FAILED; + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + Config.err.println("Pattern: " + reprTest(pattern, str, option)); + e.printStackTrace(Config.err); + Config.err.println("SEVERE ERROR: " + e.getMessage()); + nerror++; + return Matcher.FAILED; + } + + if (result == -1) { + if (not) { + if (VERBOSE) Config.log.println("OK(NOT): " + reprTest(pattern, str, option)); + nsucc++; + } else { + Config.log.println("FAIL: " + reprTest(pattern, str, option) + " GPOS: " + + gpos + " Start: " + searchStart); + nfail++; + } + } else { + if (not) { + Config.log.println("FAIL(NOT): " + reprTest(pattern, str, option)); + nfail++; + } else { + if (region.getBeg(mem) == from && region.getEnd(mem) == to) { + if (VERBOSE) Config.log.println("OK: " + reprTest(pattern, str, option)); + nsucc++; + } else { + Config.log.println("FAIL: " + reprTest(pattern, str, option) + " GPOS: " + gpos + " Start: " + + searchStart + " Groups: [Exp " + from + "-" + to + ", Act " + region.getBeg(mem) + "-" + + region.getEnd(mem) + "]"); + nfail++; + } + } + } + return result; + } + + protected void x2(byte[] pattern, byte[] str, int from, int to) throws InterruptedException { + xx(pattern, str, from, to, 0, false); + } + + protected void x2(byte[] pattern, byte[] str, int from, int to, int option) throws InterruptedException { + xx(pattern, str, from, to, 0, false, option); + } + + protected void x3(byte[] pattern, byte[] str, int from, int to, int mem) throws InterruptedException { + xx(pattern, str, from, to, mem, false); + } + + protected void n(byte[] pattern, byte[] str) throws InterruptedException { + xx(pattern, str, 0, 0, 0, true); + } + + protected void n(byte[] pattern, byte[] str, int option) throws InterruptedException { + xx(pattern, str, 0, 0, 0, true, option); + } + + protected void n(byte[] pattern, byte[] str, int gpos, int option) throws InterruptedException { + xx(pattern, str, gpos, 0, 0, 0, 0, true, option); + } + + public void xxs(String pattern, String str, int from, int to, int mem, boolean not) throws InterruptedException { + xxs(pattern, str, from, to, mem, not, option()); + } + + public void xxs(String pattern, String str, int from, int to, int mem, boolean not, int option) + throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, 0, 0, from, to, mem, not, option, WarnCallback.NONE); + } + + public int x2s(String pattern, String str, int from, int to) throws InterruptedException { + return x2s(pattern, str, from, to, option()); + } + + public int x2s(String pattern, String str, int from, int to, int option) throws InterruptedException { + return x2s(pattern, str, 0, 0, from, to, option); + } + + public int x2s(String pattern, String str, int gpos, int searchStart, int from, int to) throws InterruptedException { + return x2s(pattern, str, gpos, searchStart, from, to, option()); + } + + public int x2s(String pattern, String str, int gpos, int searchStart, int from, int to, int option) throws InterruptedException { + return x2s(pattern, 0, pattern.length(), str, gpos, searchStart, from, to, 0, false, option, WarnCallback.NONE); + } + + public int x2s(String patternIsSubstring, int p, int end, String str, int from, int to, boolean not, WarnCallback warnCallback) throws InterruptedException { + return x2s(patternIsSubstring, p, end, str, 0, 0, from, to, 0, not, option(), warnCallback); + } + + public int x2s(String patternIsSubstring, int p, int end, String str, int gpos, int searchStart, int from, int to, int mem, boolean not, int option, WarnCallback warnCallback) throws InterruptedException { + try { + byte[] patternBytes = patternIsSubstring.substring(p, end).getBytes(testEncoding()); + // convert char p and end to byte + byte[] prefixBytes = patternIsSubstring.substring(0, p).getBytes(testEncoding()); + p = prefixBytes.length == 0 ? 0 : length(prefixBytes); + end = p + length(patternBytes); + return xx(patternIsSubstring.getBytes(testEncoding()), str.getBytes(testEncoding()), p, end, gpos, searchStart, from, to, mem, not, option, warnCallback); + } catch (UnsupportedEncodingException uee) { + uee.printStackTrace(); + return Matcher.FAILED; + } + } + + public void x3s(String pattern, String str, int from, int to, int mem) throws InterruptedException { + x3s(pattern, str, from, to, mem, option()); + } + + public void x3s(String pattern, String str, int from, int to, int mem, int option) throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, 0, 0, from, to, mem, false, option, WarnCallback.NONE); + } + + public void ns(String pattern, String str) throws InterruptedException { + ns(pattern, str, option()); + } + + public void ns(String pattern, String str, int option) throws InterruptedException { + ns(pattern, str, 0, option); + } + + public void ns(String pattern, String str, int gpos, int option) throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, gpos, 0, 0, 0, 0, true, option, WarnCallback.NONE); + } + + @org.junit.Test + public void testRegexp() throws Exception { + test(); + Config.log.println("RESULT SUCC: " + nsucc + ", FAIL: " + nfail + ", ERROR: " + nerror + " Test: " + getClass().getSimpleName() + ", Encoding: " + encoding()); + assertEquals(0, nfail + nerror); + } + + public abstract void test() throws Exception; +} diff --git a/third_party/joni/test/org/joni/test/TestA.java b/third_party/joni/test/org/joni/test/TestA.java new file mode 100644 index 000000000..d814e3687 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestA.java @@ -0,0 +1,557 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.ErrorMessages; + +public class TestA extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "iso-8859-2"; + } + + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + x2s("", "", 0, 0); + x2s("^", "", 0, 0); + x2s("$", "", 0, 0); + x2s("\\G", "", 0, 0); + x2s("\\A", "", 0, 0); + x2s("\\Z", "", 0, 0); + x2s("\\z", "", 0, 0); + x2s("^$", "", 0, 0); + x2s("\\ca", "\001", 0, 1); + x2s("\\C-b", "\002", 0, 1); + x2s("\\c\\\\", "\034", 0, 1); + x2s("q[\\c\\\\]", "q\034", 0, 2); + x2s("", "a", 0, 0); + x2s("a", "a", 0, 1); + x2s("\\x61", "a", 0, 1); + x2s("aa", "aa", 0, 2); + x2s("aaa", "aaa", 0, 3); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35); + x2s("ab", "ab", 0, 2); + x2s("b", "ab", 1, 2); + x2s("bc", "abc", 1, 3); + x2s("(?i:#RET#)", "#INS##RET#", 5, 10); + x2s("\\17", "\017", 0, 1); + x2s("\\x1f", "\u001f", 0, 1); + x2s("\\xED\\xF2", "\u00ed\u0148", 0, 2); + x2s("a(?#....\\\\JJJJ)b", "ab", 0, 2); + x2s("(?x) G (o O(?-x)oO) g L", "GoOoOgLe", 0, 7); + x2s(".", "a", 0, 1); + ns(".", ""); + x2s("..", "ab", 0, 2); + x2s("\\w", "e", 0, 1); + ns("\\W", "e"); + x2s("\\s", " ", 0, 1); + x2s("\\S", "b", 0, 1); + x2s("\\d", "4", 0, 1); + ns("\\D", "4"); + x2s("\\b", "z ", 0, 0); + x2s("\\b", " z", 1, 1); + x2s("\\B", "zz ", 1, 1); + x2s("\\B", "z ", 2, 2); + x2s("\\B", " z", 0, 0); + x2s("[ab]", "b", 0, 1); + ns("[ab]", "c"); + x2s("[a-z]", "t", 0, 1); + ns("[^a]", "a"); + x2s("[^a]", "\n", 0, 1); + x2s("[]]", "]", 0, 1); + ns("[^]]", "]"); + x2s("[\\^]+", "0^^1", 1, 3); + x2s("[b-]", "b", 0, 1); + x2s("[b-]", "-", 0, 1); + x2s("[\\w]", "z", 0, 1); + ns("[\\w]", " "); + x2s("[\\W]", "b$", 1, 2); + x2s("[\\d]", "5", 0, 1); + ns("[\\d]", "e"); + x2s("[\\D]", "t", 0, 1); + ns("[\\D]", "3"); + x2s("[\\s]", " ", 0, 1); + ns("[\\s]", "a"); + x2s("[\\S]", "b", 0, 1); + ns("[\\S]", " "); + x2s("[\\w\\d]", "2", 0, 1); + ns("[\\w\\d]", " "); + x2s("[[:upper:]]", "B", 0, 1); + x2s("[*[:xdigit:]+]", "+", 0, 1); + x2s("[*[:xdigit:]+]", "GHIKK-9+*", 6, 7); + x2s("[*[:xdigit:]+]", "-@^+", 3, 4); + ns("[[:upper]]", "A"); + x2s("[[:upper]]", ":", 0, 1); + x2s("[\\044-\\047]", "\046", 0, 1); + x2s("[\\x5a-\\x5c]", "\u005b", 0, 1); + x2s("[\\x6A-\\x6D]", "\u006c", 0, 1); + ns("[\\x6A-\\x6D]", "\u006e"); + ns("^[0-9A-F]+ 0+ UNDEF ", "75F 00000000 SECT14A notype () External | _rb_apply"); + x2s("[\\[]", "[", 0, 1); + x2s("[\\]]", "]", 0, 1); + x2s("[&]", "&", 0, 1); + x2s("[[ab]]", "b", 0, 1); + x2s("[[ab]c]", "c", 0, 1); + ns("[[^a]]", "a"); + ns("[^[a]]", "a"); + x2s("[[ab]&&bc]", "b", 0, 1); + ns("[[ab]&&bc]", "a"); + ns("[[ab]&&bc]", "c"); + x2s("[a-z&&b-y&&c-x]", "w", 0, 1); + ns("[^a-z&&b-y&&c-x]", "w"); + x2s("[[^a&&a]&&a-z]", "b", 0, 1); + ns("[[^a&&a]&&a-z]", "a"); + x2s("[[^a-z&&bcdef]&&[^c-g]]", "h", 0, 1); + ns("[[^a-z&&bcdef]&&[^c-g]]", "c"); + x2s("[^[^abc]&&[^cde]]", "c", 0, 1); + x2s("[^[^abc]&&[^cde]]", "e", 0, 1); + ns("[^[^abc]&&[^cde]]", "f"); + x2s("[a-&&-a]", "-", 0, 1); + ns("[a\\-&&\\-a]", "&"); + ns("\\wabc", " abc"); + x2s("a\\Wbc", "a bc", 0, 4); + x2s("a.b.c", "aabbc", 0, 5); + x2s(".\\wb\\W..c", "abb bcc", 0, 7); + x2s("\\s\\wzzz", " zzzz", 0, 5); + x2s("aa.b", "aabb", 0, 4); + ns(".a", "ab"); + x2s(".a", "aa", 0, 2); + x2s("^a", "a", 0, 1); + x2s("^a$", "a", 0, 1); + x2s("^\\w$", "a", 0, 1); + ns("^\\w$", " "); + x2s("^\\wab$", "zab", 0, 3); + x2s("^\\wabcdef$", "zabcdef", 0, 7); + x2s("^\\w...def$", "zabcdef", 0, 7); + x2s("\\w\\w\\s\\Waaa\\d", "aa aaa4", 0, 8); + x2s("\\A\\Z", "", 0, 0); + x2s("\\Axyz", "xyz", 0, 3); + x2s("xyz\\Z", "xyz", 0, 3); + x2s("xyz\\z", "xyz", 0, 3); + x2s("a\\Z", "a", 0, 1); + x2s("0\\gA", 1, 4, "doesntMatter", 0, 0, true, WarnCallback.DEFAULT); + x2s("\\Gaz", "az", 0, 2); + ns("\\Gz", "bza"); + ns("az\\G", "az"); + ns("az\\A", "az"); + ns("a\\Az", "az"); + x2s("\\^\\$", "^$", 0, 2); + x2s("^x?y", "xy", 0, 2); + x2s("^(x?y)", "xy", 0, 2); + x2s("\\w", "_", 0, 1); + ns("\\W", "_"); + x2s("(?=z)z", "z", 0, 1); + ns("(?=z).", "a"); + x2s("(?!z)a", "a", 0, 1); + ns("(?!z)a", "z"); + x2s("(?i:a)", "a", 0, 1); + x2s("(?i:a)", "A", 0, 1); + x2s("(?i:A)", "a", 0, 1); + ns("(?i:A)", "b"); + x2s("(?i:[A-Z])", "a", 0, 1); + x2s("(?i:[f-m])", "H", 0, 1); + x2s("(?i:[f-m])", "h", 0, 1); + ns("(?i:[f-m])", "e"); + x2s("(?i:[A-c])", "D", 0, 1); + x2s("(?i:[!-k])", "Z", 0, 1); + x2s("(?i:[!-k])", "7", 0, 1); + x2s("(?i:[T-}])", "b", 0, 1); + x2s("(?i:[T-}])", "{", 0, 1); + x2s("(?i:\\?a)", "?A", 0, 2); + x2s("(?i:\\*A)", "*a", 0, 2); + ns(".", "\n"); + x2s("(?m:.)", "\n", 0, 1); + x2s("(?m:a.)", "a\n", 0, 2); + x2s("(?m:.b)", "a\nb", 1, 3); + x2s(".*abc", "dddabdd\nddabc", 8, 13); + x2s("(?m:.*abc)", "dddabddabc", 0, 10); + ns("(?i)(?-i)a", "A"); + ns("(?i)(?-i:a)", "A"); + x2s("a?", "", 0, 0); + x2s("a?", "b", 0, 0); + x2s("a?", "a", 0, 1); + x2s("a*", "", 0, 0); + x2s("a*", "a", 0, 1); + x2s("a*", "aaa", 0, 3); + x2s("a*", "baaaa", 0, 0); + ns("a+", ""); + x2s("a+", "a", 0, 1); + x2s("a+", "aaaa", 0, 4); + x2s("a+", "aabbb", 0, 2); + x2s("a+", "baaaa", 1, 5); + x2s(".?", "", 0, 0); + x2s(".?", "f", 0, 1); + x2s(".?", "\n", 0, 0); + x2s(".*", "", 0, 0); + x2s(".*", "abcde", 0, 5); + x2s(".+", "z", 0, 1); + x2s(".+", "zdswer\n", 0, 6); + x2s("(.*)a\\1f", "babfbac", 0, 4); + x2s("(.*)a\\1f", "bacbabf", 3, 7); + x2s("((.*)a\\2f)", "bacbabf", 3, 7); + x2s("(.*)a\\1f", "baczzzzzz\nbazz\nzzzzbabf", 19, 23); + x2s("a|b", "a", 0, 1); + x2s("a|b", "b", 0, 1); + x2s("|a", "a", 0, 0); + x2s("(|a)", "a", 0, 0); + x2s("ab|bc", "ab", 0, 2); + x2s("ab|bc", "bc", 0, 2); + x2s("z(?:ab|bc)", "zbc", 0, 3); + x2s("a(?:ab|bc)c", "aabc", 0, 4); + x2s("ab|(?:ac|az)", "az", 0, 2); + x2s("a|b|c", "dc", 1, 2); + x2s("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "pqr", 0, 2); + ns("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "mn"); + x2s("a|^z", "ba", 1, 2); + x2s("a|^z", "za", 0, 1); + x2s("a|\\Gz", "bza", 2, 3); + x2s("a|\\Gz", "za", 0, 1); + x2s("a|\\Az", "bza", 2, 3); + x2s("a|\\Az", "za", 0, 1); + x2s("a|b\\Z", "ba", 1, 2); + x2s("a|b\\Z", "b", 0, 1); + x2s("a|b\\z", "ba", 1, 2); + x2s("a|b\\z", "b", 0, 1); + x2s("\\w|\\s", " ", 0, 1); + ns("\\w|\\w", " "); + x2s("\\w|%", "%", 0, 1); + x2s("\\w|[&$]", "&", 0, 1); + x2s("[b-d]|[^e-z]", "a", 0, 1); + x2s("(?:a|[c-f])|bz", "dz", 0, 1); + x2s("(?:a|[c-f])|bz", "bz", 0, 2); + x2s("abc|(?=zz)..f", "zzf", 0, 3); + x2s("abc|(?!zz)..f", "abf", 0, 3); + x2s("(?=za)..a|(?=zz)..a", "zza", 0, 3); + ns("(?>a|abd)c", "abdc"); + x2s("(?>abd|a)c", "abdc", 0, 4); + x2s("a?|b", "a", 0, 1); + x2s("a?|b", "b", 0, 0); + x2s("a?|b", "", 0, 0); + x2s("a*|b", "aa", 0, 2); + x2s("a*|b*", "ba", 0, 0); + x2s("a*|b*", "ab", 0, 1); + x2s("a+|b*", "", 0, 0); + x2s("a+|b*", "bbb", 0, 3); + x2s("a+|b*", "abbb", 0, 1); + ns("a+|b+", ""); + x2s("(a|b)?", "b", 0, 1); + x2s("(a|b)*", "ba", 0, 2); + x2s("(a|b)+", "bab", 0, 3); + x2s("(ab|ca)+", "caabbc", 0, 4); + x2s("(ab|ca)+", "aabca", 1, 5); + x2s("(ab|ca)+", "abzca", 0, 2); + x2s("(a|bab)+", "ababa", 0, 5); + x2s("(a|bab)+", "ba", 1, 2); + x2s("(a|bab)+", "baaaba", 1, 4); + x2s("(?:a|b)(?:a|b)", "ab", 0, 2); + x2s("(?:a*|b*)(?:a*|b*)", "aaabbb", 0, 3); + x2s("(?:a*|b*)(?:a+|b+)", "aaabbb", 0, 6); + x2s("(?:a+|b+){2}", "aaabbb", 0, 6); + x2s("h{0,}", "hhhh", 0, 4); + x2s("(?:a+|b+){1,2}", "aaabbb", 0, 6); + ns("ax{2}*a", "0axxxa1"); + ns("a.{0,2}a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXXa0"); + x2s("^a{2,}?a$", "aaa", 0, 3); + x2s("^[a-z]{2,}?$", "aaa", 0, 3); + x2s("(?:a+|\\Ab*)cc", "cc", 0, 2); + ns("(?:a+|\\Ab*)cc", "abcc"); + x2s("(?:^a+|b+)*c", "aabbbabc", 6, 8); + x2s("(?:^a+|b+)*c", "aabbbbc", 0, 7); + x2s("a|(?i)c", "C", 0, 1); + x2s("(?i)c|a", "C", 0, 1); + x2s("(?i)c|a", "A", 0, 1); + x2s("(?i:c)|a", "C", 0, 1); + ns("(?i:c)|a", "A"); + x2s("[abc]?", "abc", 0, 1); + x2s("[abc]*", "abc", 0, 3); + x2s("[^abc]*", "abc", 0, 0); + ns("[^abc]+", "abc"); + x2s("a??", "aaa", 0, 0); + x2s("ba??b", "bab", 0, 3); + x2s("a*?", "aaa", 0, 0); + x2s("ba*?", "baa", 0, 1); + x2s("ba*?b", "baab", 0, 4); + x2s("a+?", "aaa", 0, 1); + x2s("ba+?", "baa", 0, 2); + x2s("ba+?b", "baab", 0, 4); + x2s("(?:a?)??", "a", 0, 0); + x2s("(?:a??)?", "a", 0, 0); + x2s("(?:a?)+?", "aaa", 0, 1); + x2s("(?:a+)??", "aaa", 0, 0); + x2s("(?:a+)??b", "aaab", 0, 4); + x2s("(?:ab)?{2}", "", 0, 0); + x2s("(?:ab)?{2}", "ababa", 0, 4); + x2s("(?:ab)*{0}", "ababa", 0, 0); + x2s("(?:ab){3,}", "abababab", 0, 8); + ns("(?:ab){3,}", "abab"); + x2s("(?:ab){2,4}", "ababab", 0, 6); + x2s("(?:ab){2,4}", "ababababab", 0, 8); + x2s("(?:ab){2,4}?", "ababababab", 0, 4); + x2s("(?:ab){,}", "ab{,}", 0, 5); + x2s("(?:abc)+?{2}", "abcabcabc", 0, 6); + x2s("(?:X*)(?i:xa)", "XXXa", 0, 4); + x2s("(d+)([^abc]z)", "dddz", 0, 4); + x2s("([^abc]*)([^abc]z)", "dddz", 0, 4); + x2s("(\\w+)(\\wz)", "dddz", 0, 4); + x3s("(a)", "a", 0, 1, 1); + x3s("(ab)", "ab", 0, 2, 1); + x2s("((ab))", "ab", 0, 2); + x3s("((ab))", "ab", 0, 2, 1); + x3s("((ab))", "ab", 0, 2, 2); + x3s("((((((((((((((((((((ab))))))))))))))))))))", "ab", 0, 2, 20); + x3s("(ab)(cd)", "abcd", 0, 2, 1); + x3s("(ab)(cd)", "abcd", 2, 4, 2); + x3s("()(a)bc(def)ghijk", "abcdefghijk", 3, 6, 3); + x3s("(()(a)bc(def)ghijk)", "abcdefghijk", 3, 6, 4); + x2s("(^a)", "a", 0, 1); + x3s("(a)|(a)", "ba", 1, 2, 1); + x3s("(^a)|(a)", "ba", 1, 2, 2); + x3s("(a?)", "aaa", 0, 1, 1); + x3s("(a*)", "aaa", 0, 3, 1); + x3s("(a*)", "", 0, 0, 1); + x3s("(a+)", "aaaaaaa", 0, 7, 1); + x3s("(a+|b*)", "bbbaa", 0, 3, 1); + x3s("(a+|b?)", "bbbaa", 0, 1, 1); + x3s("(abc)?", "abc", 0, 3, 1); + x3s("(abc)*", "abc", 0, 3, 1); + x3s("(abc)+", "abc", 0, 3, 1); + x3s("(xyz|abc)+", "abc", 0, 3, 1); + x3s("([xyz][abc]|abc)+", "abc", 0, 3, 1); + x3s("((?i:abc))", "AbC", 0, 3, 1); + x2s("(abc)(?i:\\1)", "abcABC", 0, 6); + x3s("((?m:a.c))", "a\nc", 0, 3, 1); + x3s("((?=az)a)", "azb", 0, 1, 1); + x3s("abc|(.abd)", "zabd", 0, 4, 1); + x2s("(?:abc)|(ABC)", "abc", 0, 3); + x3s("(?i:(abc))|(zzz)", "ABC", 0, 3, 1); + x3s("a*(.)", "aaaaz", 4, 5, 1); + x3s("a*?(.)", "aaaaz", 0, 1, 1); + x3s("a*?(c)", "aaaac", 4, 5, 1); + x3s("[bcd]a*(.)", "caaaaz", 5, 6, 1); + x3s("(\\Abb)cc", "bbcc", 0, 2, 1); + ns("(\\Abb)cc", "zbbcc"); + x3s("(^bb)cc", "bbcc", 0, 2, 1); + ns("(^bb)cc", "zbbcc"); + x3s("cc(bb$)", "ccbb", 2, 4, 1); + ns("cc(bb$)", "ccbbb"); + ns("(\\1)", ""); + ns("\\1(a)", "aa"); + ns("(a(b)\\1)\\2+", "ababb"); + ns("(?:(?:\\1|z)(a))+$", "zaa"); + x2s("(?:(?:\\1|z)(a))+$", "zaaa", 0, 4); + x2s("(a)(?=\\1)", "aa", 0, 1); + ns("(a)$|\\1", "az"); + x2s("(a)\\1", "aa", 0, 2); + ns("(a)\\1", "ab"); + x2s("(a?)\\1", "aa", 0, 2); + x2s("(a??)\\1", "aa", 0, 0); + x2s("(a*)\\1", "aaaaa", 0, 4); + x3s("(a*)\\1", "aaaaa", 0, 2, 1); + x2s("a(b*)\\1", "abbbb", 0, 5); + x2s("a(b*)\\1", "ab", 0, 1); + x2s("(a*)(b*)\\1\\2", "aaabbaaabb", 0, 10); + x2s("(a*)(b*)\\2", "aaabbbb", 0, 7); + x2s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 8); + x3s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 3, 7); + x2s("(a)(b)(c)\\2\\1\\3", "abcbac", 0, 6); + x2s("([a-d])\\1", "cc", 0, 2); + x2s("(\\w\\d\\s)\\1", "f5 f5 ", 0, 6); + ns("(\\w\\d\\s)\\1", "f5 f5"); + x2s("(who|[a-c]{3})\\1", "whowho", 0, 6); + x2s("...(who|[a-c]{3})\\1", "abcwhowho", 0, 9); + x2s("(who|[a-c]{3})\\1", "cbccbc", 0, 6); + x2s("(^a)\\1", "aa", 0, 2); + ns("(^a)\\1", "baa"); + ns("(a$)\\1", "aa"); + ns("(ab\\Z)\\1", "ab"); + x2s("(a*\\Z)\\1", "a", 1, 1); + x2s(".(a*\\Z)\\1", "ba", 1, 2); + x3s("(.(abc)\\2)", "zabcabc", 0, 7, 1); + x3s("(.(..\\d.)\\2)", "z12341234", 0, 9, 1); + x2s("((?i:az))\\1", "AzAz", 0, 4); + ns("((?i:az))\\1", "Azaz"); + x2s("(?<=a)b", "ab", 1, 2); + ns("(?<=a)b", "bb"); + x2s("(?<=a|b)b", "bb", 1, 2); + x2s("(?<=a|bc)b", "bcb", 2, 3); + x2s("(?<=a|bc)b", "ab", 1, 2); + x2s("(?<=a|bc||defghij|klmnopq|r)z", "rz", 1, 2); + x2s("(a)\\g<1>", "aa", 0, 2); + x2s("(?a)", "a", 0, 1); + x2s("(?ab)\\g", "abab", 0, 4); + x2s("(?.zv.)\\k", "azvbazvb", 0, 8); + x2s("(?<=\\g)|-\\zEND (?XyZ)", "XyZ", 3, 3); + x2s("(?|a\\g)+", "", 0, 0); + x2s("(?|\\(\\g\\))+$", "()(())", 0, 6); + x3s("\\g(?.){0}", "X", 0, 1, 1); + x2s("\\g(abc|df(?.YZ){2,8}){0}", "XYZ", 0, 3); + x2s("\\A(?(a\\g)|)\\z", "aaaa", 0, 4); + x2s("(?|\\g\\g)\\z|\\zEND (?a|(b)\\g)", "bbbbabba", 0, 8); + x2s("(?\\w+\\sx)a+\\k", " fg xaaaaaaaafg x", 2, 18); + x3s("(z)()()(?<_9>a)\\g<_9>", "zaa", 2, 3, 1); + x2s("(.)(((?<_>a)))\\k<_>", "zaa", 0, 3); + x2s("((?\\d)|(?\\w))(\\k|\\k)", "ff", 0, 2); + x2s("(?:(?)|(?efg))\\k", "", 0, 0); + x2s("(?:(?abc)|(?efg))\\k", "abcefgefg", 3, 9); + ns("(?:(?abc)|(?efg))\\k", "abcefg"); + x2s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "a-pyumpyum", 2, 10); + x3s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "xxxxabcdefghijklmnabcdefghijklmn", 4, 18, 14); + x3s("(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?aaa)(?)$", "aaa", 0, 3, 16); + x2s("(?a|\\(\\g\\))", "a", 0, 1); + x2s("(?a|\\(\\g\\))", "((((((a))))))", 0, 13); + x3s("(?a|\\(\\g\\))", "((((((((a))))))))", 0, 17, 1); + x2s("\\g|\\zEND(?.*abc$)", "abcxxxabc", 0, 9); + x2s("\\g<1>|\\zEND(.a.)", "bac", 0, 3); + x3s("\\g<_A>\\g<_A>|\\zEND(.a.)(?<_A>.b.)", "xbxyby", 3, 6, 1); + x2s("\\A(?:\\g|\\g|\\zEND (?a|c\\gc)(?b|d\\gd))$", "cdcbcdc", 0, 7); + x2s("\\A(?|a\\g)\\z|\\zEND (?\\g)", "aaaa", 0, 4); + x2s("(?(a|b\\gc){3,5})", "baaaaca", 1, 5); + x2s("(?(a|b\\gc){3,5})", "baaaacaaaaa", 0, 10); + x2s("(?\\(([^\\(\\)]++|\\g)*+\\))", "((a))", 0, 5); + x2s("()*\\1", "", 0, 0); + x2s("(?:()|())*\\1\\2", "", 0, 0); + x3s("(?:\\1a|())*", "a", 0, 0, 1); + x2s("x((.)*)*x", "0x1x2x3", 1, 6); + x2s("x((.)*)*x(?i:\\1)\\Z", "0x1x2x1X2", 1, 9); + x2s("(?:()|()|()|()|()|())*\\2\\5", "", 0, 0); + x2s("(?:()|()|()|(x)|()|())*\\2b\\5", "b", 0, 1); + + x3s("\\A(?|.|(?:(?.)\\g\\k))\\z", "reer", 0, 4, 1); + x3s("(?-i:\\g)(?i:(?a)){0}", "A", 0, 1, 1); + + String pat = + "(? \\g \\g* \\g ){0}" + + "(? < \\g \\s* > ){0}" + + "(? [a-zA-Z_:]+ ){0}" + + "(? [^<&]+ (\\g | [^<&]+)* ){0}" + + "(? >){0}" + + "\\g"; + + String str = "fbbbf"; + + x3s(pat, str, 0, 27, 0, Option.EXTEND); + x3s(pat, str, 0, 27, 1, Option.EXTEND); + x3s(pat, str, 6, 11, 2, Option.EXTEND); + x3s(pat, str, 7, 10, 3, Option.EXTEND); + x3s(pat, str, 5, 21, 4, Option.EXTEND); + x3s(pat, str, 21, 27, 5, Option.EXTEND); + + x2s("(a)b\\k<1>", "aba", 0, 3); + x2s("^(?>(?=a)(a|))++$", "a", 0, 1); + x2s("\\k", "k", 0, 1); + x2s("\\kx", "kx", 0, 2); + x2s("\\g", "g", 0, 1); + x2s("\\gx", "gx", 0, 2); + x2s("\\k\\g", "kg", 0, 2); + ns("\\00", "00"); + ns("\\70", "70"); + x2s("\\80", "80", 0, 2); + x2s("\\90", "90", 0, 2); + + ns("(?a)|(?b))(?:(?()cd|x)e|fg)", "bxe", 0, 3); + ns("(?:(?a)|(?b))(?:(?()cd|x)e|fg)", "bxe"); + x2s("((?<=a))?(?(1)b|c)", "abc", 1, 2); + x2s("((?<=a))?(?(1)b|c)", "bc", 1, 2); + x2s("((?x)|(?y))(?()y|x)", "xy", 0, 2); + x2s("((?x)|(?y))(?()y|x)", "yx", 0, 2); + ns("((?x)|(?y))(?()y|x)", "xx"); + ns("((?x)|(?y))(?()y|x)", "yy"); + xerrs("(a)?(?b)?(?(1)a)(?()b)", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + xerrs("()(?(2))", ErrorMessages.INVALID_BACKREF); + xerrs("(?(700000))", ErrorMessages.INVALID_BACKREF); + + x2s("\\R", "\n", 0, 1); + x2s("\\R", "\r", 0, 1); + x2s("\\R{3}", "\r\r\n\n", 0, 4); + + x2s("\\X{5}", "あいab\n", 0, 5); + x2s("\\X{5}", "あいab\n", 0, 5); + x2s("(?<=a).*b", "aab", 1, 3); + + x2s("([.])", ".", 0, 1); + x2s("([a])", "a", 0, 1); + x2s("([\\w])", "a", 0, 1); + + // gpos + ns("\\Gabc", "123abcdef", 2, Option.DEFAULT); + x2s("\\Gabc", "123abcdef", 3, 0, 3, 6); + x2s("\\Gabc", "123abcdef", 3, 3, 3, 6); + ns("\\Gabc", "123abcdef", 0, 3); + + x2s("(?!\\G)", "abcd", 2, 3, 3, 3); + x2s("(?!\\G)", "abcd", 3, 3, 4, 4); + } +} diff --git a/third_party/joni/test/org/joni/test/TestC.java b/third_party/joni/test/org/joni/test/TestC.java new file mode 100755 index 000000000..d9d5d16ad --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestC.java @@ -0,0 +1,732 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.EUCJPEncoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestC extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return EUCJPEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "cp1250"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + x2s("", "", 0, 0); + x2s("^", "", 0, 0); + x2s("$", "", 0, 0); + x2s("\\G", "", 0, 0); + x2s("\\A", "", 0, 0); + x2s("\\Z", "", 0, 0); + x2s("\\z", "", 0, 0); + x2s("^$", "", 0, 0); + x2s("\\ca", "\001", 0, 1); + x2s("\\C-b", "\002", 0, 1); + x2s("\\c\\\\", "\034", 0, 1); + x2s("q[\\c\\\\]", "q\034", 0, 2); + x2s("", "a", 0, 0); + x2s("a", "a", 0, 1); + x2s("\\x61", "a", 0, 1); + x2s("aa", "aa", 0, 2); + x2s("aaa", "aaa", 0, 3); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35); + x2s("ab", "ab", 0, 2); + x2s("b", "ab", 1, 2); + x2s("bc", "abc", 1, 3); + x2s("(?i:#RET#)", "#INS##RET#", 5, 10); + x2s("\\17", "\017", 0, 1); + x2s("\\x1f", "\u001f", 0, 1); + x2s("a(?#....\\\\JJJJ)b", "ab", 0, 2); + x2s("(?x) G (o O(?-x)oO) g L", "GoOoOgLe", 0, 7); + x2s(".", "a", 0, 1); + ns(".", ""); + x2s("..", "ab", 0, 2); + x2s("\\w", "e", 0, 1); + ns("\\W", "e"); + x2s("\\s", " ", 0, 1); + x2s("\\S", "b", 0, 1); + x2s("\\d", "4", 0, 1); + ns("\\D", "4"); + x2s("\\b", "z ", 0, 0); + x2s("\\b", " z", 1, 1); + x2s("\\B", "zz ", 1, 1); + x2s("\\B", "z ", 2, 2); + x2s("\\B", " z", 0, 0); + x2s("[ab]", "b", 0, 1); + ns("[ab]", "c"); + x2s("[a-z]", "t", 0, 1); + ns("[^a]", "a"); + x2s("[^a]", "\n", 0, 1); + x2s("[]]", "]", 0, 1); + ns("[^]]", "]"); + x2s("[\\^]+", "0^^1", 1, 3); + x2s("[b-]", "b", 0, 1); + x2s("[b-]", "-", 0, 1); + x2s("[\\w]", "z", 0, 1); + ns("[\\w]", " "); + x2s("[\\W]", "b$", 1, 2); + x2s("[\\d]", "5", 0, 1); + ns("[\\d]", "e"); + x2s("[\\D]", "t", 0, 1); + ns("[\\D]", "3"); + x2s("[\\s]", " ", 0, 1); + ns("[\\s]", "a"); + x2s("[\\S]", "b", 0, 1); + ns("[\\S]", " "); + x2s("[\\w\\d]", "2", 0, 1); + ns("[\\w\\d]", " "); + x2s("[[:upper:]]", "B", 0, 1); + x2s("[*[:xdigit:]+]", "+", 0, 1); + x2s("[*[:xdigit:]+]", "GHIKK-9+*", 6, 7); + x2s("[*[:xdigit:]+]", "-@^+", 3, 4); + ns("[[:upper]]", "A"); + x2s("[[:upper]]", ":", 0, 1); + x2s("[\\044-\\047]", "\046", 0, 1); + x2s("[\\x5a-\\x5c]", "\u005b", 0, 1); + x2s("[\\x6A-\\x6D]", "\u006c", 0, 1); + ns("[\\x6A-\\x6D]", "\u006e"); + ns("^[0-9A-F]+ 0+ UNDEF ", "75F 00000000 SECT14A notype () External | _rb_apply"); + x2s("[\\[]", "[", 0, 1); + x2s("[\\]]", "]", 0, 1); + x2s("[&]", "&", 0, 1); + x2s("[[ab]]", "b", 0, 1); + x2s("[[ab]c]", "c", 0, 1); + ns("[[^a]]", "a"); + ns("[^[a]]", "a"); + x2s("[[ab]&&bc]", "b", 0, 1); + ns("[[ab]&&bc]", "a"); + ns("[[ab]&&bc]", "c"); + x2s("[a-z&&b-y&&c-x]", "w", 0, 1); + ns("[^a-z&&b-y&&c-x]", "w"); + x2s("[[^a&&a]&&a-z]", "b", 0, 1); + ns("[[^a&&a]&&a-z]", "a"); + x2s("[[^a-z&&bcdef]&&[^c-g]]", "h", 0, 1); + ns("[[^a-z&&bcdef]&&[^c-g]]", "c"); + x2s("[^[^abc]&&[^cde]]", "c", 0, 1); + x2s("[^[^abc]&&[^cde]]", "e", 0, 1); + ns("[^[^abc]&&[^cde]]", "f"); + x2s("[a-&&-a]", "-", 0, 1); + ns("[a\\-&&\\-a]", "&"); + ns("\\wabc", " abc"); + x2s("a\\Wbc", "a bc", 0, 4); + x2s("a.b.c", "aabbc", 0, 5); + x2s(".\\wb\\W..c", "abb bcc", 0, 7); + x2s("\\s\\wzzz", " zzzz", 0, 5); + x2s("aa.b", "aabb", 0, 4); + ns(".a", "ab"); + x2s(".a", "aa", 0, 2); + x2s("^a", "a", 0, 1); + x2s("^a$", "a", 0, 1); + x2s("^\\w$", "a", 0, 1); + ns("^\\w$", " "); + x2s("^\\wab$", "zab", 0, 3); + x2s("^\\wabcdef$", "zabcdef", 0, 7); + x2s("^\\w...def$", "zabcdef", 0, 7); + x2s("\\w\\w\\s\\Waaa\\d", "aa aaa4", 0, 8); + x2s("\\A\\Z", "", 0, 0); + x2s("\\Axyz", "xyz", 0, 3); + x2s("xyz\\Z", "xyz", 0, 3); + x2s("xyz\\z", "xyz", 0, 3); + x2s("a\\Z", "a", 0, 1); + x2s("\\Gaz", "az", 0, 2); + ns("\\Gz", "bza"); + ns("az\\G", "az"); + ns("az\\A", "az"); + ns("a\\Az", "az"); + x2s("\\^\\$", "^$", 0, 2); + x2s("^x?y", "xy", 0, 2); + x2s("^(x?y)", "xy", 0, 2); + x2s("\\w", "_", 0, 1); + ns("\\W", "_"); + x2s("(?=z)z", "z", 0, 1); + ns("(?=z).", "a"); + x2s("(?!z)a", "a", 0, 1); + ns("(?!z)a", "z"); + x2s("(?i:a)", "a", 0, 1); + x2s("(?i:a)", "A", 0, 1); + x2s("(?i:A)", "a", 0, 1); + ns("(?i:A)", "b"); + x2s("(?i:[A-Z])", "a", 0, 1); + x2s("(?i:[f-m])", "H", 0, 1); + x2s("(?i:[f-m])", "h", 0, 1); + ns("(?i:[f-m])", "e"); + x2s("(?i:[A-c])", "D", 0, 1); + ns("(?i:[^a-z])", "A"); + ns("(?i:[^a-z])", "a"); + x2s("(?i:[!-k])", "Z", 0, 1); + x2s("(?i:[!-k])", "7", 0, 1); + x2s("(?i:[T-}])", "b", 0, 1); + x2s("(?i:[T-}])", "{", 0, 1); + x2s("(?i:\\?a)", "?A", 0, 2); + x2s("(?i:\\*A)", "*a", 0, 2); + ns(".", "\n"); + x2s("(?m:.)", "\n", 0, 1); + x2s("(?m:a.)", "a\n", 0, 2); + x2s("(?m:.b)", "a\nb", 1, 3); + x2s(".*abc", "dddabdd\nddabc", 8, 13); + x2s("(?m:.*abc)", "dddabddabc", 0, 10); + ns("(?i)(?-i)a", "A"); + ns("(?i)(?-i:a)", "A"); + x2s("a?", "", 0, 0); + x2s("a?", "b", 0, 0); + x2s("a?", "a", 0, 1); + x2s("a*", "", 0, 0); + x2s("a*", "a", 0, 1); + x2s("a*", "aaa", 0, 3); + x2s("a*", "baaaa", 0, 0); + ns("a+", ""); + x2s("a+", "a", 0, 1); + x2s("a+", "aaaa", 0, 4); + x2s("a+", "aabbb", 0, 2); + x2s("a+", "baaaa", 1, 5); + x2s(".?", "", 0, 0); + x2s(".?", "f", 0, 1); + x2s(".?", "\n", 0, 0); + x2s(".*", "", 0, 0); + x2s(".*", "abcde", 0, 5); + x2s(".+", "z", 0, 1); + x2s(".+", "zdswer\n", 0, 6); + x2s("(.*)a\\1f", "babfbac", 0, 4); + x2s("(.*)a\\1f", "bacbabf", 3, 7); + x2s("((.*)a\\2f)", "bacbabf", 3, 7); + x2s("(.*)a\\1f", "baczzzzzz\nbazz\nzzzzbabf", 19, 23); + x2s("a|b", "a", 0, 1); + x2s("a|b", "b", 0, 1); + x2s("|a", "a", 0, 0); + x2s("(|a)", "a", 0, 0); + x2s("ab|bc", "ab", 0, 2); + x2s("ab|bc", "bc", 0, 2); + x2s("z(?:ab|bc)", "zbc", 0, 3); + x2s("a(?:ab|bc)c", "aabc", 0, 4); + x2s("ab|(?:ac|az)", "az", 0, 2); + x2s("a|b|c", "dc", 1, 2); + x2s("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "pqr", 0, 2); + ns("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "mn"); + x2s("a|^z", "ba", 1, 2); + x2s("a|^z", "za", 0, 1); + x2s("a|\\Gz", "bza", 2, 3); + x2s("a|\\Gz", "za", 0, 1); + x2s("a|\\Az", "bza", 2, 3); + x2s("a|\\Az", "za", 0, 1); + x2s("a|b\\Z", "ba", 1, 2); + x2s("a|b\\Z", "b", 0, 1); + x2s("a|b\\z", "ba", 1, 2); + x2s("a|b\\z", "b", 0, 1); + x2s("\\w|\\s", " ", 0, 1); + ns("\\w|\\w", " "); + x2s("\\w|%", "%", 0, 1); + x2s("\\w|[&$]", "&", 0, 1); + x2s("[b-d]|[^e-z]", "a", 0, 1); + x2s("(?:a|[c-f])|bz", "dz", 0, 1); + x2s("(?:a|[c-f])|bz", "bz", 0, 2); + x2s("abc|(?=zz)..f", "zzf", 0, 3); + x2s("abc|(?!zz)..f", "abf", 0, 3); + x2s("(?=za)..a|(?=zz)..a", "zza", 0, 3); + ns("(?>a|abd)c", "abdc"); + x2s("(?>abd|a)c", "abdc", 0, 4); + x2s("a?|b", "a", 0, 1); + x2s("a?|b", "b", 0, 0); + x2s("a?|b", "", 0, 0); + x2s("a*|b", "aa", 0, 2); + x2s("a*|b*", "ba", 0, 0); + x2s("a*|b*", "ab", 0, 1); + x2s("a+|b*", "", 0, 0); + x2s("a+|b*", "bbb", 0, 3); + x2s("a+|b*", "abbb", 0, 1); + ns("a+|b+", ""); + x2s("(a|b)?", "b", 0, 1); + x2s("(a|b)*", "ba", 0, 2); + x2s("(a|b)+", "bab", 0, 3); + x2s("(ab|ca)+", "caabbc", 0, 4); + x2s("(ab|ca)+", "aabca", 1, 5); + x2s("(ab|ca)+", "abzca", 0, 2); + x2s("(a|bab)+", "ababa", 0, 5); + x2s("(a|bab)+", "ba", 1, 2); + x2s("(a|bab)+", "baaaba", 1, 4); + x2s("(?:a|b)(?:a|b)", "ab", 0, 2); + x2s("(?:a*|b*)(?:a*|b*)", "aaabbb", 0, 3); + x2s("(?:a*|b*)(?:a+|b+)", "aaabbb", 0, 6); + x2s("(?:a+|b+){2}", "aaabbb", 0, 6); + x2s("h{0,}", "hhhh", 0, 4); + x2s("(?:a+|b+){1,2}", "aaabbb", 0, 6); + ns("ax{2}*a", "0axxxa1"); + ns("a.{0,2}a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXXa0"); + x2s("^a{2,}?a$", "aaa", 0, 3); + x2s("^[a-z]{2,}?$", "aaa", 0, 3); + x2s("(?:a+|\\Ab*)cc", "cc", 0, 2); + ns("(?:a+|\\Ab*)cc", "abcc"); + x2s("(?:^a+|b+)*c", "aabbbabc", 6, 8); + x2s("(?:^a+|b+)*c", "aabbbbc", 0, 7); + x2s("a|(?i)c", "C", 0, 1); + x2s("(?i)c|a", "C", 0, 1); + x2s("(?i)c|a", "A", 0, 1); + x2s("(?i:c)|a", "C", 0, 1); + ns("(?i:c)|a", "A"); + x2s("[abc]?", "abc", 0, 1); + x2s("[abc]*", "abc", 0, 3); + x2s("[^abc]*", "abc", 0, 0); + ns("[^abc]+", "abc"); + x2s("a??", "aaa", 0, 0); + x2s("ba??b", "bab", 0, 3); + x2s("a*?", "aaa", 0, 0); + x2s("ba*?", "baa", 0, 1); + x2s("ba*?b", "baab", 0, 4); + x2s("a+?", "aaa", 0, 1); + x2s("ba+?", "baa", 0, 2); + x2s("ba+?b", "baab", 0, 4); + x2s("(?:a?)??", "a", 0, 0); + x2s("(?:a??)?", "a", 0, 0); + x2s("(?:a?)+?", "aaa", 0, 1); + x2s("(?:a+)??", "aaa", 0, 0); + x2s("(?:a+)??b", "aaab", 0, 4); + x2s("(?:ab)?{2}", "", 0, 0); + x2s("(?:ab)?{2}", "ababa", 0, 4); + x2s("(?:ab)*{0}", "ababa", 0, 0); + x2s("(?:ab){3,}", "abababab", 0, 8); + ns("(?:ab){3,}", "abab"); + x2s("(?:ab){2,4}", "ababab", 0, 6); + x2s("(?:ab){2,4}", "ababababab", 0, 8); + x2s("(?:ab){2,4}?", "ababababab", 0, 4); + x2s("(?:ab){,}", "ab{,}", 0, 5); + x2s("(?:abc)+?{2}", "abcabcabc", 0, 6); + x2s("(?:X*)(?i:xa)", "XXXa", 0, 4); + x2s("(d+)([^abc]z)", "dddz", 0, 4); + x2s("([^abc]*)([^abc]z)", "dddz", 0, 4); + x2s("(\\w+)(\\wz)", "dddz", 0, 4); + x3s("(a)", "a", 0, 1, 1); + x3s("(ab)", "ab", 0, 2, 1); + x2s("((ab))", "ab", 0, 2); + x3s("((ab))", "ab", 0, 2, 1); + x3s("((ab))", "ab", 0, 2, 2); + x3s("((((((((((((((((((((ab))))))))))))))))))))", "ab", 0, 2, 20); + x3s("(ab)(cd)", "abcd", 0, 2, 1); + x3s("(ab)(cd)", "abcd", 2, 4, 2); + x3s("()(a)bc(def)ghijk", "abcdefghijk", 3, 6, 3); + x3s("(()(a)bc(def)ghijk)", "abcdefghijk", 3, 6, 4); + x2s("(^a)", "a", 0, 1); + x3s("(a)|(a)", "ba", 1, 2, 1); + x3s("(^a)|(a)", "ba", 1, 2, 2); + x3s("(a?)", "aaa", 0, 1, 1); + x3s("(a*)", "aaa", 0, 3, 1); + x3s("(a*)", "", 0, 0, 1); + x3s("(a+)", "aaaaaaa", 0, 7, 1); + x3s("(a+|b*)", "bbbaa", 0, 3, 1); + x3s("(a+|b?)", "bbbaa", 0, 1, 1); + x3s("(abc)?", "abc", 0, 3, 1); + x3s("(abc)*", "abc", 0, 3, 1); + x3s("(abc)+", "abc", 0, 3, 1); + x3s("(xyz|abc)+", "abc", 0, 3, 1); + x3s("([xyz][abc]|abc)+", "abc", 0, 3, 1); + x3s("((?i:abc))", "AbC", 0, 3, 1); + x2s("(abc)(?i:\\1)", "abcABC", 0, 6); + x3s("((?m:a.c))", "a\nc", 0, 3, 1); + x3s("((?=az)a)", "azb", 0, 1, 1); + x3s("abc|(.abd)", "zabd", 0, 4, 1); + x2s("(?:abc)|(ABC)", "abc", 0, 3); + x3s("(?i:(abc))|(zzz)", "ABC", 0, 3, 1); + x3s("a*(.)", "aaaaz", 4, 5, 1); + x3s("a*?(.)", "aaaaz", 0, 1, 1); + x3s("a*?(c)", "aaaac", 4, 5, 1); + x3s("[bcd]a*(.)", "caaaaz", 5, 6, 1); + x3s("(\\Abb)cc", "bbcc", 0, 2, 1); + ns("(\\Abb)cc", "zbbcc"); + x3s("(^bb)cc", "bbcc", 0, 2, 1); + ns("(^bb)cc", "zbbcc"); + x3s("cc(bb$)", "ccbb", 2, 4, 1); + ns("cc(bb$)", "ccbbb"); + ns("(\\1)", ""); + ns("\\1(a)", "aa"); + ns("(a(b)\\1)\\2+", "ababb"); + ns("(?:(?:\\1|z)(a))+$", "zaa"); + x2s("(?:(?:\\1|z)(a))+$", "zaaa", 0, 4); + x2s("(a)(?=\\1)", "aa", 0, 1); + ns("(a)$|\\1", "az"); + x2s("(a)\\1", "aa", 0, 2); + ns("(a)\\1", "ab"); + x2s("(a?)\\1", "aa", 0, 2); + x2s("(a??)\\1", "aa", 0, 0); + x2s("(a*)\\1", "aaaaa", 0, 4); + x3s("(a*)\\1", "aaaaa", 0, 2, 1); + x2s("a(b*)\\1", "abbbb", 0, 5); + x2s("a(b*)\\1", "ab", 0, 1); + x2s("(a*)(b*)\\1\\2", "aaabbaaabb", 0, 10); + x2s("(a*)(b*)\\2", "aaabbbb", 0, 7); + x2s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 8); + x3s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 3, 7); + x2s("(a)(b)(c)\\2\\1\\3", "abcbac", 0, 6); + x2s("([a-d])\\1", "cc", 0, 2); + x2s("(\\w\\d\\s)\\1", "f5 f5 ", 0, 6); + ns("(\\w\\d\\s)\\1", "f5 f5"); + x2s("(who|[a-c]{3})\\1", "whowho", 0, 6); + x2s("...(who|[a-c]{3})\\1", "abcwhowho", 0, 9); + x2s("(who|[a-c]{3})\\1", "cbccbc", 0, 6); + x2s("(^a)\\1", "aa", 0, 2); + ns("(^a)\\1", "baa"); + ns("(a$)\\1", "aa"); + ns("(ab\\Z)\\1", "ab"); + x2s("(a*\\Z)\\1", "a", 1, 1); + x2s(".(a*\\Z)\\1", "ba", 1, 2); + x3s("(.(abc)\\2)", "zabcabc", 0, 7, 1); + x3s("(.(..\\d.)\\2)", "z12341234", 0, 9, 1); + x2s("((?i:az))\\1", "AzAz", 0, 4); + ns("((?i:az))\\1", "Azaz"); + x2s("(?<=a)b", "ab", 1, 2); + ns("(?<=a)b", "bb"); + x2s("(?<=a|b)b", "bb", 1, 2); + x2s("(?<=a|bc)b", "bcb", 2, 3); + x2s("(?<=a|bc)b", "ab", 1, 2); + x2s("(?<=a|bc||defghij|klmnopq|r)z", "rz", 1, 2); + x2s("(a)\\g<1>", "aa", 0, 2); + x2s("(?a)", "a", 0, 1); + x2s("(?ab)\\g", "abab", 0, 4); + x2s("(?.zv.)\\k", "azvbazvb", 0, 8); + x2s("(?<=\\g)|-\\zEND (?XyZ)", "XyZ", 3, 3); + x2s("(?|a\\g)+", "", 0, 0); + x2s("(?|\\(\\g\\))+$", "()(())", 0, 6); + x3s("\\g(?.){0}", "X", 0, 1, 1); + x2s("\\g(abc|df(?.YZ){2,8}){0}", "XYZ", 0, 3); + x2s("\\A(?(a\\g)|)\\z", "aaaa", 0, 4); + x2s("(?|\\g\\g)\\z|\\zEND (?a|(b)\\g)", "bbbbabba", 0, 8); + x2s("(?\\w+\\sx)a+\\k", " fg xaaaaaaaafg x", 2, 18); + x3s("(z)()()(?<_9>a)\\g<_9>", "zaa", 2, 3, 1); + x2s("(.)(((?<_>a)))\\k<_>", "zaa", 0, 3); + x2s("((?\\d)|(?\\w))(\\k|\\k)", "ff", 0, 2); + x2s("(?:(?)|(?efg))\\k", "", 0, 0); + x2s("(?:(?abc)|(?efg))\\k", "abcefgefg", 3, 9); + ns("(?:(?abc)|(?efg))\\k", "abcefg"); + x2s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "a-pyumpyum", 2, 10); + x3s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "xxxxabcdefghijklmnabcdefghijklmn", 4, 18, 14); + x3s("(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?aaa)(?)$", "aaa", 0, 3, 16); + x2s("(?a|\\(\\g\\))", "a", 0, 1); + x2s("(?a|\\(\\g\\))", "((((((a))))))", 0, 13); + x3s("(?a|\\(\\g\\))", "((((((((a))))))))", 0, 17, 1); + x2s("\\g|\\zEND(?.*abc$)", "abcxxxabc", 0, 9); + x2s("\\g<1>|\\zEND(.a.)", "bac", 0, 3); + x3s("\\g<_A>\\g<_A>|\\zEND(.a.)(?<_A>.b.)", "xbxyby", 3, 6, 1); + x2s("\\A(?:\\g|\\g|\\zEND (?a|c\\gc)(?b|d\\gd))$", "cdcbcdc", 0, 7); + x2s("\\A(?|a\\g)\\z|\\zEND (?\\g)", "aaaa", 0, 4); + x2s("(?(a|b\\gc){3,5})", "baaaaca", 1, 5); + x2s("(?(a|b\\gc){3,5})", "baaaacaaaaa", 0, 10); + x2s("(?\\(([^\\(\\)]++|\\g)*+\\))", "((a))", 0, 5); + x2s("()*\\1", "", 0, 0); + x2s("(?:()|())*\\1\\2", "", 0, 0); + x3s("(?:\\1a|())*", "a", 0, 0, 1); + x2s("x((.)*)*x", "0x1x2x3", 1, 6); + x2s("x((.)*)*x(?i:\\1)\\Z", "0x1x2x1X2", 1, 9); + x2s("(?:()|()|()|()|()|())*\\2\\5", "", 0, 0); + x2s("(?:()|()|()|(x)|()|())*\\2b\\5", "b", 0, 1); + x2s("\\xED\\xF2", "\u00ed\u0148", 0, 2); + x2s("", "\u00a4\u02d8", 0, 0); + x2s("\u00a4\u02d8", "\u00a4\u02d8", 0, 2); + ns("\u00a4\u00a4", "\u00a4\u02d8"); + x2s("\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a6\u00a4\u00a6", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6); + x2s("\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", "\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", 0, 70); + x2s("\u00a4\u02d8", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x2s("\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 2, 6); + x2s("\\xca\\xb8", "\u0118\u00b8", 0, 2); + x2s(".", "\u00a4\u02d8", 0, 2); + x2s("..", "\u00a4\u00ab\u00a4\u00ad", 0, 4); + x2s("\\w", "\u00a4\u015e", 0, 2); + ns("\\W", "\u00a4\u02d8"); + x2s("[\\W]", "\u00a4\u00a6$", 2, 3); + x2s("\\S", "\u00a4\u02dd", 0, 2); + x2s("\\S", "\u00b4\u00c1", 0, 2); + x2s("\\b", "\u00b5\u00a4 ", 0, 0); + x2s("\\b", " \u00a4\u0170", 1, 1); + x2s("\\B", "\u00a4\u00bb\u00a4\u02dd ", 2, 2); + x2s("\\B", "\u00a4\u00a6 ", 3, 3); + x2s("\\B", " \u00a4\u00a4", 0, 0); + x2s("[\u00a4\u017c\u00a4\u00c1]", "\u00a4\u00c1", 0, 2); + ns("[\u00a4\u0118\u00a4\u00cb]", "\u00a4\u011a"); + x2s("[\u00a4\u00a6-\u00a4\u015e]", "\u00a4\u00a8", 0, 2); + ns("[^\u00a4\u00b1]", "\u00a4\u00b1"); + x2s("[\\w]", "\u00a4\u00cd", 0, 2); + ns("[\\d]", "\u00a4\u0150"); + x2s("[\\D]", "\u00a4\u010e", 0, 2); + ns("[\\s]", "\u00a4\u017b"); + x2s("[\\S]", "\u00a4\u0158", 0, 2); + x2s("[\\w\\d]", "\u00a4\u010d", 0, 2); + x2s("[\\w\\d]", " \u00a4\u010d", 3, 5); + ns("\\w\u00b5\u00b4\u013d\u00d6", " \u00b5\u00b4\u013d\u00d6"); + x2s("\u00b5\u00b4\\W\u013d\u00d6", "\u00b5\u00b4 \u013d\u00d6", 0, 5); + x2s("\u00a4\u02d8.\u00a4\u00a4.\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6", 0, 10); + x2s(".\\w\u00a4\u00a6\\W..\u00a4\u013e", "\u00a4\u00a8\u00a4\u00a6\u00a4\u00a6 \u00a4\u00a6\u00a4\u013e\u00a4\u013e", 0, 13); + x2s("\\s\\w\u00a4\u0142\u00a4\u0142\u00a4\u0142", " \u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", 0, 9); + x2s("\u00a4\u02d8\u00a4\u02d8.\u00a4\u00b1", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00b1\u00a4\u00b1", 0, 8); + ns(".\u00a4\u00a4", "\u00a4\u00a4\u00a4\u00a8"); + x2s(".\u00a4\u015e", "\u00a4\u015e\u00a4\u015e", 0, 4); + x2s("^\u00a4\u02d8", "\u00a4\u02d8", 0, 2); + x2s("^\u00a4\u0155$", "\u00a4\u0155", 0, 2); + x2s("^\\w$", "\u00a4\u00cb", 0, 2); + x2s("^\\w\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142$", "z\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 0, 11); + x2s("^\\w...\u00a4\u00a6\u00a4\u00a8\u00a4\u015e$", "z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6\u00a4\u00a8\u00a4\u015e", 0, 13); + x2s("\\w\\w\\s\\W\u00a4\u015e\u00a4\u015e\u00a4\u015e\\d", "a\u00a4\u015e \u00a4\u015e\u00a4\u015e\u00a4\u015e4", 0, 12); + x2s("\\A\u00a4\u017c\u00a4\u00c1\u00a4\u00c4", "\u00a4\u017c\u00a4\u00c1\u00a4\u00c4", 0, 6); + x2s("\u00a4\u0155\u00a4\u00e1\u00a4\u00e2\\Z", "\u00a4\u0155\u00a4\u00e1\u00a4\u00e2", 0, 6); + x2s("\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\\z", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 6); + x2s("\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\\Z", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\n", 0, 6); + x2s("\\G\u00a4\u00dd\u00a4\u00d4", "\u00a4\u00dd\u00a4\u00d4", 0, 4); + ns("\\G\u00a4\u00a8", "\u00a4\u00a6\u00a4\u00a8\u00a4\u015e"); + ns("\u00a4\u010c\u00a4\u0106\\G", "\u00a4\u010c\u00a4\u0106"); + ns("\u00a4\u0162\u00a4\u00df\\A", "\u00a4\u0162\u00a4\u00df"); + ns("\u00a4\u0162\\A\u00a4\u00df", "\u00a4\u0162\u00a4\u00df"); + x2s("(?=\u00a4\u00bb)\u00a4\u00bb", "\u00a4\u00bb", 0, 2); + ns("(?=\u00a4\u00a6).", "\u00a4\u00a4"); + x2s("(?!\u00a4\u00a6)\u00a4\u00ab", "\u00a4\u00ab", 0, 2); + ns("(?!\u00a4\u010c)\u00a4\u02d8", "\u00a4\u010c"); + x2s("(?i:\u00a4\u02d8)", "\u00a4\u02d8", 0, 2); + x2s("(?i:\u00a4\u00d6\u00a4\u016e)", "\u00a4\u00d6\u00a4\u016e", 0, 4); + ns("(?i:\u00a4\u00a4)", "\u00a4\u00a6"); + x2s("(?m:\u00a4\u010d.)", "\u00a4\u010d\n", 0, 3); + x2s("(?m:.\u00a4\u00e1)", "\u00a4\u0162\n\u00a4\u00e1", 2, 5); + x2s("\u00a4\u02d8?", "", 0, 0); + x2s("\u0118\u0143?", "\u02db\u02dd", 0, 0); + x2s("\u0118\u0143?", "\u0118\u0143", 0, 2); + x2s("\u00ce\u011a*", "", 0, 0); + x2s("\u00ce\u011a*", "\u00ce\u011a", 0, 2); + x2s("\u00bb\u0147*", "\u00bb\u0147\u00bb\u0147\u00bb\u0147", 0, 6); + x2s("\u00c7\u010e*", "\u013d\u017b\u00c7\u010e\u00c7\u010e\u00c7\u010e\u00c7\u010e", 0, 0); + ns("\u00bb\u0142+", ""); + x2s("\u02db\u010e+", "\u02db\u010e", 0, 2); + x2s("\u00bb\u0163+", "\u00bb\u0163\u00bb\u0163\u00bb\u0163\u00bb\u0163", 0, 8); + x2s("\u00a4\u00a8+", "\u00a4\u00a8\u00a4\u00a8\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 0, 4); + x2s("\u00a4\u00a6+", "\u00a4\u015e\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 2, 10); + x2s(".?", "\u00a4\u017c", 0, 2); + x2s(".*", "\u00a4\u0143\u00a4\u00d4\u00a4\u00d7\u00a4\u00da", 0, 8); + x2s(".+", "\u00a4\u00ed", 0, 2); + x2s(".+", "\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u00ab\n", 0, 8); + x2s("\u00a4\u02d8|\u00a4\u00a4", "\u00a4\u02d8", 0, 2); + x2s("\u00a4\u02d8|\u00a4\u00a4", "\u00a4\u00a4", 0, 2); + x2s("\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a4\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a6", 0, 4); + x2s("\u00a4\u0148(?:\u00a4\u00ab\u00a4\u00ad|\u00a4\u00ad\u00a4\u017b)", "\u00a4\u0148\u00a4\u00ab\u00a4\u00ad", 0, 6); + x2s("\u00a4\u0148(?:\u00a4\u00ab\u00a4\u00ad|\u00a4\u00ad\u00a4\u017b)\u00a4\u00b1", "\u00a4\u0148\u00a4\u00ad\u00a4\u017b\u00a4\u00b1", 0, 8); + x2s("\u00a4\u02d8\u00a4\u00a4|(?:\u00a4\u02d8\u00a4\u00a6|\u00a4\u02d8\u00a4\u0148)", "\u00a4\u02d8\u00a4\u0148", 0, 4); + x2s("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6", "\u00a4\u00a8\u00a4\u00a6", 2, 4); + x2s("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8|\u00a4\u015e\u00a4\u00ab\u00a4\u00ad|\u00a4\u017b|\u00a4\u00b1\u00a4\u0142\u00a4\u00b5|\u00a4\u00b7\u00a4\u0105\u00a4\u00bb|\u00a4\u02dd|\u00a4\u017c\u00a4\u00c1|\u00a4\u00c4\u00a4\u0106\u00a4\u010c\u00a4\u0118\u00a4\u00cb|\u00a4\u011a\u00a4\u00cd", "\u00a4\u00b7\u00a4\u0105\u00a4\u00bb", 0, 6); + ns("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8|\u00a4\u015e\u00a4\u00ab\u00a4\u00ad|\u00a4\u017b|\u00a4\u00b1\u00a4\u0142\u00a4\u00b5|\u00a4\u00b7\u00a4\u0105\u00a4\u00bb|\u00a4\u02dd|\u00a4\u017c\u00a4\u00c1|\u00a4\u00c4\u00a4\u0106\u00a4\u010c\u00a4\u0118\u00a4\u00cb|\u00a4\u011a\u00a4\u00cd", "\u00a4\u0105\u00a4\u00bb"); + x2s("\u00a4\u02d8|^\u00a4\u010f", "\u00a4\u00d6\u00a4\u02d8", 2, 4); + x2s("\u00a4\u02d8|^\u00a4\u0148", "\u00a4\u0148\u00a4\u02d8", 0, 2); + x2s("\u00b5\u00b4|\\G\u013d\u00d6", "\u00a4\u00b1\u013d\u00d6\u00b5\u00b4", 4, 6); + x2s("\u00b5\u00b4|\\G\u013d\u00d6", "\u013d\u00d6\u00b5\u00b4", 0, 2); + x2s("\u00b5\u00b4|\\A\u013d\u00d6", "b\u013d\u00d6\u00b5\u00b4", 3, 5); + x2s("\u00b5\u00b4|\\A\u013d\u00d6", "\u013d\u00d6", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6\u00b5\u00b4", 2, 4); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6\n", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\z", "\u013d\u00d6\u00b5\u00b4", 2, 4); + x2s("\u00b5\u00b4|\u013d\u00d6\\z", "\u013d\u00d6", 0, 2); + x2s("\\w|\\s", "\u00a4\u015e", 0, 2); + x2s("\\w|%", "%\u00a4\u015e", 0, 1); + x2s("\\w|[&$]", "\u00a4\u00a6&", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]", "\u00a4\u00a6", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]|[^\u00a4\u00ab-\u00a4\u0142]", "\u00a4\u02d8", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]|[^\u00a4\u00ab-\u00a4\u0142]", "\u00a4\u00ab", 0, 2); + x2s("[^\u00a4\u02d8]", "\n", 0, 1); + x2s("(?:\u00a4\u02d8|[\u00a4\u00a6-\u00a4\u00ad])|\u00a4\u00a4\u00a4\u0148", "\u00a4\u00a6\u00a4\u0148", 0, 2); + x2s("(?:\u00a4\u02d8|[\u00a4\u00a6-\u00a4\u00ad])|\u00a4\u00a4\u00a4\u0148", "\u00a4\u00a4\u00a4\u0148", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(?=\u00a4\u00b1\u00a4\u00b1)..\u00a4\u0170", "\u00a4\u00b1\u00a4\u00b1\u00a4\u0170", 0, 6); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(?!\u00a4\u00b1\u00a4\u00b1)..\u00a4\u0170", "\u00a4\u02d8\u00a4\u00a4\u00a4\u0170", 0, 6); + // x2s("(?=\u00a4\u0148\u00a4\u02d8)..\u00a4\u02d8|(?=\u00a4\u0148\u00a4\u0148)..\u00a4\u02d8", "\u00a4\u0148\u00a4\u0148\u00a4\u02d8", 0, 6); + x2s("(?<=\u00a4\u02d8|\u00a4\u00a4\u00a4\u00a6)\u00a4\u00a4", "\u00a4\u00a4\u00a4\u00a6\u00a4\u00a4", 4, 6); + ns("(?>\u00a4\u02d8|\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8)\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8\u00a4\u00a6"); + x2s("(?>\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8|\u00a4\u02d8)\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8\u00a4\u00a6", 0, 8); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "\u00a4\u02d8", 0, 2); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "\u00a4\u00a4", 0, 0); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "", 0, 0); + x2s("\u00a4\u02d8*|\u00a4\u00a4", "\u00a4\u02d8\u00a4\u02d8", 0, 4); + x2s("\u00a4\u02d8*|\u00a4\u00a4*", "\u00a4\u00a4\u00a4\u02d8", 0, 0); + x2s("\u00a4\u02d8*|\u00a4\u00a4*", "\u00a4\u02d8\u00a4\u00a4", 0, 2); + x2s("[a\u00a4\u02d8]*|\u00a4\u00a4*", "a\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 3); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "", 0, 0); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 6); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 2); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "a\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 0); + ns("\u00a4\u02d8+|\u00a4\u00a4+", ""); + x2s("(\u00a4\u02d8|\u00a4\u00a4)?", "\u00a4\u00a4", 0, 2); + x2s("(\u00a4\u02d8|\u00a4\u00a4)*", "\u00a4\u00a4\u00a4\u02d8", 0, 4); + x2s("(\u00a4\u02d8|\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4", 0, 6); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u00a6\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 0, 8); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8)+", "\u00a4\u00a6\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 4, 12); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u02d8", 2, 10); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u0148\u00a4\u00a6\u00a4\u02d8", 0, 4); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "$$zzzz\u00a4\u02d8\u00a4\u00a4\u00a4\u0148\u00a4\u00a6\u00a4\u02d8", 6, 10); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8", 0, 10); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8", 2, 8); + x2s("(?:\u00a4\u02d8|\u00a4\u00a4)(?:\u00a4\u02d8|\u00a4\u00a4)", "\u00a4\u02d8\u00a4\u00a4", 0, 4); + x2s("(?:\u00a4\u02d8*|\u00a4\u00a4*)(?:\u00a4\u02d8*|\u00a4\u00a4*)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 6); + x2s("(?:\u00a4\u02d8*|\u00a4\u00a4*)(?:\u00a4\u02d8+|\u00a4\u00a4+)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\u00a4\u00a4+){2}", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\u00a4\u00a4+){1,2}", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\\A\u00a4\u00a4*)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a6\u00a4\u00a6", 0, 4); + ns("(?:\u00a4\u02d8+|\\A\u00a4\u00a4*)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x2s("(?:^\u00a4\u02d8+|\u00a4\u00a4+)*\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 12, 16); + x2s("(?:^\u00a4\u02d8+|\u00a4\u00a4+)*\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6", 0, 14); + x2s("\u00a4\u00a6{0,}", "\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 0, 8); + x2s("\u00a4\u02d8|(?i)c", "C", 0, 1); + x2s("(?i)c|\u00a4\u02d8", "C", 0, 1); + x2s("(?i:\u00a4\u02d8)|a", "a", 0, 1); + ns("(?i:\u00a4\u02d8)|a", "A"); + x2s("[\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]?", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 2); + x2s("[\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6); + x2s("[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 0); + ns("[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6"); + x2s("\u00a4\u02d8??", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 0); + x2s("\u00a4\u00a4\u00a4\u02d8??\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4", 0, 6); + x2s("\u00a4\u02d8*?", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 0); + x2s("\u00a4\u00a4\u00a4\u02d8*?", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 2); + x2s("\u00a4\u00a4\u00a4\u02d8*?\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4", 0, 8); + x2s("\u00a4\u02d8+?", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 2); + x2s("\u00a4\u00a4\u00a4\u02d8+?", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 4); + x2s("\u00a4\u00a4\u00a4\u02d8+?\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4", 0, 8); + x2s("(?:\u0139\u00b7?)??", "\u0139\u00b7", 0, 0); + x2s("(?:\u0139\u00b7??)?", "\u0139\u00b7", 0, 0); + x2s("(?:\u011a\u00b4?)+?", "\u011a\u00b4\u011a\u00b4\u011a\u00b4", 0, 2); + x2s("(?:\u00c9\u00f7+)??", "\u00c9\u00f7\u00c9\u00f7\u00c9\u00f7", 0, 0); + x2s("(?:\u0154\u0103+)??\u00c1\u00fa", "\u0154\u0103\u0154\u0103\u0154\u0103\u00c1\u00fa", 0, 8); + x2s("(?:\u00a4\u02d8\u00a4\u00a4)?{2}", "", 0, 0); + x2s("(?:\u00b5\u00b4\u013d\u00d6)?{2}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4", 0, 8); + x2s("(?:\u00b5\u00b4\u013d\u00d6)*{0}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4", 0, 0); + x2s("(?:\u00b5\u00b4\u013d\u00d6){3,}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 16); + ns("(?:\u00b5\u00b4\u013d\u00d6){3,}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6"); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 12); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 16); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}?", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 8); + x2s("(?:\u00b5\u00b4\u013d\u00d6){,}", "\u00b5\u00b4\u013d\u00d6{,}", 0, 7); + x2s("(?:\u00a4\u00ab\u00a4\u00ad\u00a4\u017b)+?{2}", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 12); + x3s("(\u02db\u0110)", "\u02db\u0110", 0, 2, 1); + x3s("(\u02db\u0110\u017c\u013a)", "\u02db\u0110\u017c\u013a", 0, 4, 1); + x2s("((\u00bb\u0163\u00b4\u00d6))", "\u00bb\u0163\u00b4\u00d6", 0, 4); + x3s("((\u00c9\u00f7\u017c\u013a))", "\u00c9\u00f7\u017c\u013a", 0, 4, 1); + x3s("((\u015f\u0148\u0106\u00fc))", "\u015f\u0148\u0106\u00fc", 0, 4, 2); + x3s("((((((((((((((((((((\u00ce\u011a\u00bb\u0147))))))))))))))))))))", "\u00ce\u011a\u00bb\u0147", 0, 4, 20); + x3s("(\u00a4\u02d8\u00a4\u00a4)(\u00a4\u00a6\u00a4\u00a8)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 0, 4, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4)(\u00a4\u00a6\u00a4\u00a8)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 4, 8, 2); + x3s("()(\u00a4\u02d8)\u00a4\u00a4\u00a4\u00a6(\u00a4\u00a8\u00a4\u015e\u00a4\u00ab)\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 6, 12, 3); + x3s("(()(\u00a4\u02d8)\u00a4\u00a4\u00a4\u00a6(\u00a4\u00a8\u00a4\u015e\u00a4\u00ab)\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 6, 12, 4); + x3s(".*(\u0104\u0150\u0104\u00a9)\u0104\u00f3\u02c7\u00a6\u0104\u0162(\u0104\u00f3()\u0104\u00b7\u0104\u013a\u0104\u017c)\u0104\u00a4\u0104\u00f3", "\u0104\u0150\u0104\u00a9\u0104\u00f3\u02c7\u00a6\u0104\u0162\u0104\u00f3\u0104\u00b7\u0104\u013a\u0104\u017c\u0104\u00a4\u0104\u00f3", 10, 18, 2); + x2s("(^\u00a4\u02d8)", "\u00a4\u02d8", 0, 2); + x3s("(\u00a4\u02d8)|(\u00a4\u02d8)", "\u00a4\u00a4\u00a4\u02d8", 2, 4, 1); + x3s("(^\u00a4\u02d8)|(\u00a4\u02d8)", "\u00a4\u00a4\u00a4\u02d8", 2, 4, 2); + x3s("(\u00a4\u02d8?)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 2, 1); + x3s("(\u00a4\u0162*)", "\u00a4\u0162\u00a4\u0162\u00a4\u0162", 0, 6, 1); + x3s("(\u00a4\u010c*)", "", 0, 0, 1); + x3s("(\u00a4\u00eb+)", "\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb", 0, 14, 1); + x3s("(\u00a4\u0150+|\u00a4\u0158*)", "\u00a4\u0150\u00a4\u0150\u00a4\u0150\u00a4\u0158\u00a4\u0158", 0, 6, 1); + x3s("(\u00a4\u02d8+|\u00a4\u00a4?)", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 2, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)?", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u00b5\u00a4\u00b7\u00a4\u0105|\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("([\u00a4\u0118\u00a4\u00cb\u00a4\u011a][\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]|\u00a4\u00ab\u00a4\u00ad\u00a4\u017b)+", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 6, 1); + x3s("((?i:\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6))", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("((?m:\u00a4\u02d8.\u00a4\u00a6))", "\u00a4\u02d8\n\u00a4\u00a6", 0, 5, 1); + x3s("((?=\u00a4\u02d8\u00a4\u00f3)\u00a4\u02d8)", "\u00a4\u02d8\u00a4\u00f3\u00a4\u00a4", 0, 2, 1); + x3s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(.\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8)", "\u00a4\u00f3\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8", 0, 8, 1); + x3s("\u00a4\u02d8*(.)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 8, 10, 1); + x3s("\u00a4\u02d8*?(.)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 0, 2, 1); + x3s("\u00a4\u02d8*?(\u00a4\u00f3)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 8, 10, 1); + x3s("[\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8]\u00a4\u02d8*(.)", "\u00a4\u00a8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 10, 12, 1); + x3s("(\\A\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6", 0, 4, 1); + ns("(\\A\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00f3\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x3s("(^\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6", 0, 4, 1); + ns("(^\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00f3\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x3s("\u00a4\u00ed\u00a4\u00ed(\u00a4\u00eb\u00a4\u00eb$)", "\u00a4\u00ed\u00a4\u00ed\u00a4\u00eb\u00a4\u00eb", 4, 8, 1); + ns("\u00a4\u00ed\u00a4\u00ed(\u00a4\u00eb\u00a4\u00eb$)", "\u00a4\u00ed\u00a4\u00ed\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb"); + x2s("(\u011a\u00b5)\\1", "\u011a\u00b5\u011a\u00b5", 0, 4); + ns("(\u011a\u00b5)\\1", "\u011a\u00b5\u00c9\u0111"); + x2s("(\u00b6\u0151?)\\1", "\u00b6\u0151\u00b6\u0151", 0, 4); + x2s("(\u00b6\u0151??)\\1", "\u00b6\u0151\u00b6\u0151", 0, 0); + x2s("(\u00b6\u0151*)\\1", "\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151", 0, 8); + x3s("(\u00b6\u0151*)\\1", "\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151", 0, 4, 1); + x2s("\u00a4\u02d8(\u00a4\u00a4*)\\1", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 10); + x2s("\u00a4\u02d8(\u00a4\u00a4*)\\1", "\u00a4\u02d8\u00a4\u00a4", 0, 2); + x2s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\1\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4", 0, 20); + x2s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 14); + x3s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 6, 10, 2); + x2s("(((((((\u00a4\u00dd*)\u00a4\u00da))))))\u00a4\u00d4\\7", "\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd\u00a4\u00da\u00a4\u00d4\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd", 0, 16); + x3s("(((((((\u00a4\u00dd*)\u00a4\u00da))))))\u00a4\u00d4\\7", "\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd\u00a4\u00da\u00a4\u00d4\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd", 0, 6, 7); + x2s("(\u00a4\u010e)(\u00a4\u0147)(\u00a4\u0150)\\2\\1\\3", "\u00a4\u010e\u00a4\u0147\u00a4\u0150\u00a4\u0147\u00a4\u010e\u00a4\u0150", 0, 12); + x2s("([\u00a4\u00ad-\u00a4\u00b1])\\1", "\u00a4\u017b\u00a4\u017b", 0, 4); + x2s("(\\w\\d\\s)\\1", "\u00a4\u02d85 \u00a4\u02d85 ", 0, 8); + ns("(\\w\\d\\s)\\1", "\u00a4\u02d85 \u00a4\u02d85"); + x2s("(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u0102\u017b\u02c7\u00a9\u0102\u017b\u02c7\u00a9", 0, 8); + x2s("...(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u00a4\u02d8a\u00a4\u02d8\u0102\u017b\u02c7\u00a9\u0102\u017b\u02c7\u00a9", 0, 13); + x2s("(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u00a4\u00a6\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6\u00a4\u00a4\u00a4\u00a6", 0, 12); + x2s("(^\u00a4\u0142)\\1", "\u00a4\u0142\u00a4\u0142", 0, 4); + ns("(^\u00a4\u0155)\\1", "\u00a4\u00e1\u00a4\u0155\u00a4\u0155"); + ns("(\u00a4\u02d8$)\\1", "\u00a4\u02d8\u00a4\u02d8"); + ns("(\u00a4\u02d8\u00a4\u00a4\\Z)\\1", "\u00a4\u02d8\u00a4\u00a4"); + x2s("(\u00a4\u02d8*\\Z)\\1", "\u00a4\u02d8", 2, 2); + x2s(".(\u00a4\u02d8*\\Z)\\1", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x3s("(.(\u00a4\u00e4\u00a4\u00a4\u00a4\u0107)\\2)", "z\u00a4\u00e4\u00a4\u00a4\u00a4\u0107\u00a4\u00e4\u00a4\u00a4\u00a4\u0107", 0, 13, 1); + x3s("(.(..\\d.)\\2)", "\u00a4\u02d812341234", 0, 10, 1); + x2s("((?i:\u00a4\u02d8v\u00a4\u015f))\\1", "\u00a4\u02d8v\u00a4\u015f\u00a4\u02d8v\u00a4\u015f", 0, 10); + x2s("(?<\u00b6\u0148\u00a4\u00ab>\u0118\u0143|\\(\\g<\u00b6\u0148\u00a4\u00ab>\\))", "((((((\u0118\u0143))))))", 0, 14); + x2s("\\A(?:\\g<\u00b0\u00a4_1>|\\g<\u00b1\u013e_2>|\\z\u02dd\u015e\u00ce\u00bb (?<\u00b0\u00a4_1>\u00b4\u0143|\u013d\u00ab\\g<\u00b1\u013e_2>\u013d\u00ab)(?<\u00b1\u013e_2>\u015f\u00df|\u0118\u00ee\u00bb\u00a7\\g<\u00b0\u00a4_1>\u0118\u00ee\u00bb\u00a7))$", "\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u015f\u00df\u013d\u00ab\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u0118\u00ee\u00bb\u00a7", 0, 26); + x2s("[[\u00a4\u0147\u00a4\u0150]]", "\u00a4\u0150", 0, 2); + x2s("[[\u00a4\u00a4\u00a4\u015e\u00a4\u00a6]\u00a4\u00ab]", "\u00a4\u00ab", 0, 2); + ns("[[^\u00a4\u02d8]]", "\u00a4\u02d8"); + ns("[^[\u00a4\u02d8]]", "\u00a4\u02d8"); + x2s("[^[^\u00a4\u02d8]]", "\u00a4\u02d8", 0, 2); + x2s("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u017b", 0, 2); + ns("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u00ab"); + ns("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u00b1"); + x2s("[\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4-\u00a4\u0148&&\u00a4\u00a6-\u00a4\u0144]", "\u00a4\u0144", 0, 2); + ns("[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4-\u00a4\u0148&&\u00a4\u00a6-\u00a4\u0144]", "\u00a4\u0144"); + x2s("[[^\u00a4\u02d8&&\u00a4\u02d8]&&\u00a4\u02d8-\u00a4\u00f3]", "\u00a4\u00a4", 0, 2); + ns("[[^\u00a4\u02d8&&\u00a4\u02d8]&&\u00a4\u02d8-\u00a4\u00f3]", "\u00a4\u02d8"); + x2s("[[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]&&[^\u00a4\u00a6-\u00a4\u00ab]]", "\u00a4\u00ad", 0, 2); + ns("[[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]&&[^\u00a4\u00a6-\u00a4\u00ab]]", "\u00a4\u00a4"); + x2s("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00a6", 0, 2); + x2s("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00a8", 0, 2); + ns("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00ab"); + x2s("[\u00a4\u02d8-&&-\u00a4\u02d8]", "-", 0, 1); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]q-w]", "\u00a4\u00a8", 0, 2); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "f", 0, 1); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "g", 0, 1); + ns("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "2"); + x2s("a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9<\\/b>", "a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9", 0, 32); + x2s(".\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9<\\/b>", "a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9", 0, 32); + } + +} diff --git a/third_party/joni/test/org/joni/test/TestCallout.java b/third_party/joni/test/org/joni/test/TestCallout.java new file mode 100644 index 000000000..228c4ef29 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCallout.java @@ -0,0 +1,245 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.CalloutHandler; +import org.joni.CalloutResult; +import org.joni.MatchView; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.exception.TimeoutException; +import org.junit.Test; + +public class TestCallout { + private static Regex regex(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.US_ASCII); + return new Regex(bytes, 0, bytes.length, Option.NONE, ASCIIEncoding.INSTANCE, Syntax.RUBY); + } + + private static int search(Regex regex, String value) { + return search(regex, value, null); + } + + private static int search(Regex regex, String value, CalloutHandler handler) { + byte[] bytes = value.getBytes(StandardCharsets.US_ASCII); + Matcher matcher = regex.matcher(bytes); + matcher.setCalloutHandler(handler); + return matcher.search(0, bytes.length, Option.NONE); + } + + @Test + public void exposesCurrentPositionAndProvisionalCaptures() { + List events = new ArrayList<>(); + Regex regex = regex("(a)(?{=CALL:7})b"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add(id + ":" + match.currentBytePosition() + ":" + + match.captureBegin(0) + "-" + match.captureEnd(0) + ":" + + match.captureBegin(1) + "-" + match.captureEnd(1)); + return CalloutResult.CONTINUE; + } + + @Override + public void unwind(Object token) { + } + }; + + assertEquals(0, search(regex, "ab", handler)); + assertEquals(Arrays.asList("7:1:0-1:0-1"), events); + } + + @Test + public void unwindsBacktrackedAndSuccessfulPathsExactlyOnce() { + List events = new ArrayList<>(); + Regex regex = regex("(a(?{=CALL:1})b|a(?{=CALL:2})c)"); + CalloutHandler handler = recordingHandler(events, false); + + assertEquals(0, search(regex, "ac", handler)); + assertEquals(Arrays.asList("execute:1", "unwind:1", "execute:2", "unwind:2"), events); + } + + @Test + public void failResultBacktracksAfterUnwindingItsToken() { + List events = new ArrayList<>(); + Regex regex = regex("(a(?{=CALL:1})b|ac)"); + CalloutHandler handler = recordingHandler(events, true); + + assertEquals(0, search(regex, "ac", handler)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + @Test + public void unwindsInReverseOrderOnSuccess() { + List events = new ArrayList<>(); + Regex regex = regex("a(?{=CALL:1})(?{=CALL:2})b"); + CalloutHandler handler = recordingHandler(events, false); + + assertEquals(0, search(regex, "ab", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:2", "unwind:1"), events); + } + + @Test + public void preservesCalloutsInsideFixedQuantifiers() { + List events = new ArrayList<>(); + int[] execution = {0}; + Regex regex = regex("(?{=CALL:1}){2}"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + int token = ++execution[0]; + events.add("execute:" + token); + return CalloutResult.continueWith(token); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + + assertEquals(0, search(regex, "", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:2", "unwind:1"), events); + } + + @Test + public void unwindsPriorTokensWhenHandlerThrows() { + List events = new ArrayList<>(); + Regex regex = regex("a(?{=CALL:1})(?{=CALL:2})b"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + if (id == 2) throw new IllegalStateException("boom"); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + + assertThrows(IllegalStateException.class, () -> search(regex, "ab", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:1"), events); + } + + @Test + public void requiresAHandlerForCalloutPatterns() { + Regex regex = regex("(?{=CALL:1})"); + assertThrows(IllegalStateException.class, () -> search(regex, "")); + } + + @Test + public void keepsHandlersLocalToEachMatcher() { + Regex regex = regex("(?{=CALL:1})"); + byte[] input = new byte[0]; + List firstEvents = new ArrayList<>(); + List secondEvents = new ArrayList<>(); + Matcher first = regex.matcher(input); + Matcher second = regex.matcher(input); + first.setCalloutHandler(recordingHandler(firstEvents, false)); + second.setCalloutHandler(recordingHandler(secondEvents, false)); + + assertEquals(0, first.search(0, 0, Option.NONE)); + assertEquals(0, second.search(0, 0, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), firstEvents); + assertEquals(Arrays.asList("execute:1", "unwind:1"), secondEvents); + } + + @Test + public void unwindsActiveTokensWhenInterrupted() { + List events = new ArrayList<>(); + Regex regex = regex("(?{=CALL:1})a"); + byte[] input = "a".getBytes(StandardCharsets.US_ASCII); + Matcher matcher = regex.matcher(input); + matcher.setCalloutHandler(new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + matcher.interrupt(); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }); + + assertThrows(InterruptedException.class, + () -> matcher.searchInterruptible(0, input.length, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + @Test + public void unwindsActiveTokensOnTimeout() { + List events = new ArrayList<>(); + StringBuilder pattern = new StringBuilder("(?{=CALL:1})"); + for (int i = 0; i < 140; i++) pattern.append("()"); + Regex regex = regex(pattern.toString()); + byte[] input = new byte[0]; + Matcher matcher = regex.matcher(input); + matcher.setTimeout(1_000_000_000L); + matcher.setCalloutHandler(new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + matcher.setTimeout(0); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }); + + assertThrows(TimeoutException.class, + () -> matcher.searchInterruptible(0, input.length, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + private static CalloutHandler recordingHandler(List events, boolean fail) { + return new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + return fail ? CalloutResult.failWith(id) : CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + } +} diff --git a/third_party/joni/test/org/joni/test/TestCaseInsensitive.java b/third_party/joni/test/org/joni/test/TestCaseInsensitive.java new file mode 100644 index 000000000..ef37c9719 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCaseInsensitive.java @@ -0,0 +1,32 @@ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestCaseInsensitive extends Test { + + @Override + public int option() { + return Option.IGNORECASE; + } + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + xx("^\\d\\d\\d-".getBytes(), new byte[]{-30, -126, -84, 48, 45}, 0, 0, 0, true); + x2s("ab", "\uD835\uDC4D ab", 5, 7); + } +} diff --git a/third_party/joni/test/org/joni/test/TestCornerCases.java b/third_party/joni/test/org/joni/test/TestCornerCases.java new file mode 100755 index 000000000..310477244 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCornerCases.java @@ -0,0 +1,59 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Config; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; + +public class TestCornerCases extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "cp1250"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + byte[] reg = "l.".getBytes(); + byte[] str = "hello,lo".getBytes(); + + Regex p = new Regex(reg,0,reg.length,Option.DEFAULT,ASCIIEncoding.INSTANCE,Syntax.DEFAULT); + int result = p.matcher(str, 0, str.length).search(3, 0, Option.NONE); + if(result != 3) { + Config.log.println("FAIL: /l./ 'hello,lo' - with reverse, 3,0"); + nfail++; + } + } +} diff --git a/third_party/joni/test/org/joni/test/TestCrnl.java b/third_party/joni/test/org/joni/test/TestCrnl.java new file mode 100755 index 000000000..85044304e --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCrnl.java @@ -0,0 +1,84 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Config; +import org.joni.Option; +import org.joni.Syntax; +import org.junit.Ignore; +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; + +@Ignore +public class TestCrnl extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "ascii"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + x2s("", "\r\n", 0, 0); + x2s(".", "\r\n", 0, 1); + ns("..", "\r\n"); + x2s("^", "\r\n", 0, 0); + x2s("\\n^", "\r\nf", 1, 2); + x2s("\\n^a", "\r\na", 1, 3); + x2s("$", "\r\n", 0, 0); + x2s("T$", "T\r\n", 0, 1); + x2s("T$", "T\raT\r\n", 3, 4); + x2s("\\z", "\r\n", 2, 2); + ns("a\\z", "a\r\n"); + x2s("\\Z", "\r\n", 0, 0); + x2s("\\Z", "\r\na", 3, 3); + x2s("\\Z", "\r\n\r\n\n", 4, 4); + x2s("\\Z", "\r\n\r\nX", 5, 5); + x2s("a\\Z", "a\r\n", 0, 1); + x2s("aaaaaaaaaaaaaaa\\Z", "aaaaaaaaaaaaaaa\r\n", 0, 15); + x2s("a|$", "b\r\n", 1, 1); + x2s("$|b", "\rb", 1, 2); + x2s("a$|ab$", "\r\nab\r\n", 2, 4); + + x2s("a|\\Z", "b\r\n", 1, 1); + x2s("\\Z|b", "\rb", 1, 2); + x2s("a\\Z|ab\\Z", "\r\nab\r\n", 2, 4); + x2s("(?=a$).", "a\r\n", 0, 1); + ns("(?=a$).", "a\r"); + x2s("(?!a$)..", "a\r", 0, 2); + x2s("(?<=a$).\\n", "a\r\n", 1, 3); + ns("(? 0 || nerror > 0) Config.err.println("make sure to enable USE_CRNL_AS_LINE_TERMINATOR"); + } +} diff --git a/third_party/joni/test/org/joni/test/TestError.java b/third_party/joni/test/org/joni/test/TestError.java new file mode 100755 index 000000000..4051611e5 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestError.java @@ -0,0 +1,106 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.ErrorMessages; + +public class TestError extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "iso-8859-2"; + } + + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + xerrs("(", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("[[:WoRd:]]", ErrorMessages.INVALID_POSIX_BRACKET_TYPE); + xerrs("(0?0|(?(1)||)|(?(1)||))?", ErrorMessages.INVALID_CONDITION_PATTERN); + xerrs("[\\40000000000", ErrorMessages.TOO_BIG_NUMBER); + xerrs("[\\40000000000\n", ErrorMessages.TOO_BIG_NUMBER); + xerrs("[]", ErrorMessages.EMPTY_CHAR_CLASS); + xerrs("[c-a]", ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + xerrs("\\x{FFFFFFFF}", ErrorMessages.ERR_TOO_BIG_WIDE_CHAR_VALUE); + xerrs("\\x{100000000}", ErrorMessages.ERR_TOO_LONG_WIDE_CHAR_VALUE); + xerrs("\\u026x", ErrorMessages.TOO_SHORT_DIGITS); + xerrs("()(?\\!(?'a')\\1)", ErrorMessages.UNDEFINED_GROUP_OPTION); + xerrs("\\((", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("(|", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("'/g\\\u00ff\u00ff\u00ff\u00ff&))", ErrorMessages.UNMATCHED_CLOSE_PARENTHESIS); + xerrs("0'/g\\\u00ff\u00ff\u00ff\u00ff&))", 1, 12, ErrorMessages.UNMATCHED_CLOSE_PARENTHESIS, WarnCallback.DEFAULT); + xerrs("0\\1**?", 1, 6, ErrorMessages.INVALID_BACKREF, WarnCallback.DEFAULT); + xerrs("[0-0-\u00ff ", ErrorMessages.PREMATURE_END_OF_CHAR_CLASS); // \xe2 + xerrs("\\p{foobarbaz}", ErrorMessages.ERR_INVALID_CHAR_PROPERTY_NAME.replace("%n", "foobarbaz")); + //xerrs("\\p{あ}", ErrorMessages.ERR_INVALID_CHAR_PROPERTY_NAME.replace("%n", "あ")); + + xerrs("a{100001}", ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + xerrs("a{0,100001}", ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + xerrs("a{5,1}", ErrorMessages.UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE); + xerrs("[\\6000", ErrorMessages.TOO_BIG_NUMBER); // CVE-2017-9226 + xerrs("[\\H- ]", ErrorMessages.UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); // CVE-2017-9228 + xerrs("[a-\\d]", ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + + xerrs("(?:ab|cd)*\\1", ErrorMessages.INVALID_BACKREF); + xerrs("(ab|cd)*\\1", ErrorMessages.INVALID_BACKREF, Option.DONT_CAPTURE_GROUP); + + xerrs("(.(?=\\g<1>))", ErrorMessages.NEVER_ENDING_RECURSION); + xerrs("(a)(?b)\\g<1>\\g", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + + // xerrs("(?<", ErrorMessages.ERR_END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("(?<>)", ErrorMessages.EMPTY_GROUP_NAME); + //xerrs("(?<.>)", ErrorMessages.ERR_INVALID_CHAR_IN_GROUP_NAME); + xerrs("\\g<1->", ErrorMessages.INVALID_CHAR_IN_GROUP_NAME.replace("%n", "1->")); + xerrs("\\k<1/>", ErrorMessages.INVALID_GROUP_NAME.replace("%n", "1/")); + // xerrs("\\k<1-1/>", ErrorMessages.ERR_INVALID_GROUP_NAME.replace("%n", "1-1/>")); + // xerrs("\\k", ErrorMessages.ERR_INVALID_CHAR_IN_GROUP_NAME.replace("%n", "a/")); + // xerrs("\\g<1>", ErrorMessages.UNDEFINED_GROUP_REFERENCE); + + xerrs("*", ErrorMessages.TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + xerrs("{1}", ErrorMessages.TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + + xerrs("(?a)(?b)\\g", ErrorMessages.MULTIPLEX_DEFINITION_NAME_CALL.replace("%n", "a")); + + xerrs("(a)?(?b)?(?(1)a)(?()b)", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + + xerrs("()(?(2))", ErrorMessages.INVALID_BACKREF); + xerrs("(?(700000))", ErrorMessages.INVALID_BACKREF); + + xerrs("(? " + acceptableMaximumTime); + } +} diff --git a/third_party/joni/test/org/joni/test/TestNSU8.java b/third_party/joni/test/org/joni/test/TestNSU8.java new file mode 100755 index 000000000..9f00ed4b7 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestNSU8.java @@ -0,0 +1,61 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.NonStrictUTF8Encoding; + +public class TestNSU8 extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return NonStrictUTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32, (byte)32, (byte)32}, 0, 5, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32, (byte)32}, 0, 4, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240}, 0, 1, 1, false); + + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32, (byte)32, (byte)32}, 0, 4, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224}, 0, 1, 1, false); + + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192}, 0, 1, 1, false); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerl.java b/third_party/joni/test/org/joni/test/TestPerl.java new file mode 100755 index 000000000..98bbf09f8 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerl.java @@ -0,0 +1,47 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; + +public class TestPerl extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "iso-8859-2"; + } + @Override + public Syntax syntax() { + return Syntax.PerlNG; + } + @Override + public void test() throws Exception { + } +} diff --git a/third_party/joni/test/org/joni/test/TestU.java b/third_party/joni/test/org/joni/test/TestU.java new file mode 100755 index 000000000..f8b027f23 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestU.java @@ -0,0 +1,802 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.UTF16BEEncoding; + +public class TestU extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return UTF16BEEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "iso-8859-1"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + private int ulen(byte[]bytes) { + return encoding().strByteLengthNull(bytes, 0, bytes.length); + } + + private String uconv(byte []bytes, int len) { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < len; i += 2) { + int c = bytes[i] & 0xff; + // sb.append(String.format("\\%03o", c)); + if (c == 0) { + c = bytes[i+1] & 0xff; + if (c < 0x20 || c >= 0x7f || c == 0x5c || c == 0x22) { + sb.append(String.format("\\%03o", c)); + } else { + sb.append(new String(new byte[]{(byte)c})); + } + } else { + sb.append(String.format("\\%03o", c)); + c = bytes[i+1] & 0xff; + sb.append(String.format("\\%03o", c)); + } + } + + return sb.toString(); + } + + @Override + protected String repr(byte[]bytes) { + return uconv(bytes, ulen(bytes)); + } + + @Override + protected int length(byte[]bytes) { + return ulen(bytes); + } + + @Override + public void test() throws Exception { + x2s("\000\000", "\000\000", 0, 0); + x2s("\000^\000\000", "\000\000", 0, 0); + x2s("\000$\000\000", "\000\000", 0, 0); + x2s("\000\134\000G\000\000", "\000\000", 0, 0); + x2s("\000\134\000A\000\000", "\000\000", 0, 0); + x2s("\000\134\000Z\000\000", "\000\000", 0, 0); + x2s("\000\134\000z\000\000", "\000\000", 0, 0); + x2s("\000^\000$\000\000", "\000\000", 0, 0); + x2s("\000\134\000c\000a\000\000", "\000\001\000\000", 0, 2); + x2s("\000\134\000C\000-\000b\000\000", "\000\002\000\000", 0, 2); + x2s("\000\134\000c\000\134\000\134\000\000", "\000\034\000\000", 0, 2); + x2s("\000q\000[\000\134\000c\000\134\000\134\000]\000\000", "\000q\000\034\000\000", 0, 4); + x2s("\000\000", "\000a\000\000", 0, 0); + x2s("\000a\000\000", "\000a\000\000", 0, 2); + x2s("\000\134\000x\0000\0000\000\134\000x\0006\0001\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000a\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000a\000a\000a\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000\000", "\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000\000", 0, 70); + x2s("\000a\000b\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000b\000\000", "\000a\000b\000\000", 2, 4); + x2s("\000b\000c\000\000", "\000a\000b\000c\000\000", 2, 6); + x2s("\000(\000?\000i\000:\000#\000R\000E\000T\000#\000)\000\000", "\000#\000I\000N\000S\000#\000#\000R\000E\000T\000#\000\000", 10, 20); + x2s("\000\134\0000\0000\0000\000\134\0001\0007\000\000", "\000\017\000\000", 0, 2); + x2s("\000\134\000x\0000\0000\000\134\000x\0001\000f\000\000", "\000\037\000\000", 0, 2); + x2s("\000a\000(\000?\000#\000.\000.\000.\000.\000\134\000\134\000J\000J\000J\000J\000)\000b\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000(\000?\000x\000)\000 \000 \000G\000 \000(\000o\000 \000O\000(\000?\000-\000x\000)\000o\000O\000)\000 \000g\000 \000L\000\000", "\000G\000o\000O\000o\000O\000g\000L\000e\000\000", 0, 14); + x2s("\000.\000\000", "\000a\000\000", 0, 2); + ns("\000.\000\000", "\000\000"); + x2s("\000.\000.\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000\134\000w\000\000", "\000e\000\000", 0, 2); + ns("\000\134\000W\000\000", "\000e\000\000"); + x2s("\000\134\000s\000\000", "\000 \000\000", 0, 2); + x2s("\000\134\000S\000\000", "\000b\000\000", 0, 2); + x2s("\000\134\000d\000\000", "\0004\000\000", 0, 2); + ns("\000\134\000D\000\000", "\0004\000\000"); + x2s("\000\134\000b\000\000", "\000z\000 \000\000", 0, 0); + x2s("\000\134\000b\000\000", "\000 \000z\000\000", 2, 2); + x2s("\000\134\000B\000\000", "\000z\000z\000 \000\000", 2, 2); + x2s("\000\134\000B\000\000", "\000z\000 \000\000", 4, 4); + x2s("\000\134\000B\000\000", "\000 \000z\000\000", 0, 0); + x2s("\000[\000a\000b\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000a\000b\000]\000\000", "\000c\000\000"); + x2s("\000[\000a\000-\000z\000]\000\000", "\000t\000\000", 0, 2); + ns("\000[\000^\000a\000]\000\000", "\000a\000\000"); + x2s("\000[\000^\000a\000]\000\000", "\000\012\000\000", 0, 2); + x2s("\000[\000]\000]\000\000", "\000]\000\000", 0, 2); + ns("\000[\000^\000]\000]\000\000", "\000]\000\000"); + x2s("\000[\000\134\000^\000]\000+\000\000", "\0000\000^\000^\0001\000\000", 2, 6); + x2s("\000[\000b\000-\000]\000\000", "\000b\000\000", 0, 2); + x2s("\000[\000b\000-\000]\000\000", "\000-\000\000", 0, 2); + x2s("\000[\000\134\000w\000]\000\000", "\000z\000\000", 0, 2); + ns("\000[\000\134\000w\000]\000\000", "\000 \000\000"); + x2s("\000[\000\134\000W\000]\000\000", "\000b\000$\000\000", 2, 4); + x2s("\000[\000\134\000d\000]\000\000", "\0005\000\000", 0, 2); + ns("\000[\000\134\000d\000]\000\000", "\000e\000\000"); + x2s("\000[\000\134\000D\000]\000\000", "\000t\000\000", 0, 2); + ns("\000[\000\134\000D\000]\000\000", "\0003\000\000"); + x2s("\000[\000\134\000s\000]\000\000", "\000 \000\000", 0, 2); + ns("\000[\000\134\000s\000]\000\000", "\000a\000\000"); + x2s("\000[\000\134\000S\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000\134\000S\000]\000\000", "\000 \000\000"); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "\0002\000\000", 0, 2); + ns("\000[\000\134\000w\000\134\000d\000]\000\000", "\000 \000\000"); + x2s("\000[\000[\000:\000u\000p\000p\000e\000r\000:\000]\000]\000\000", "\000B\000\000", 0, 2); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000+\000\000", 0, 2); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000G\000H\000I\000K\000K\000-\0009\000+\000*\000\000", 12, 14); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000-\000@\000^\000+\000\000", 6, 8); + ns("\000[\000[\000:\000u\000p\000p\000e\000r\000]\000]\000\000", "\000A\000\000"); + x2s("\000[\000[\000:\000u\000p\000p\000e\000r\000]\000]\000\000", "\000:\000\000", 0, 2); + x2s("\000[\000\134\0000\0000\0000\000\134\0000\0004\0004\000-\000\134\0000\0000\0000\000\134\0000\0004\0007\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000\134\000x\0000\0000\000\134\000x\0005\000a\000-\000\134\000x\0000\0000\000\134\000x\0005\000c\000]\000\000", "\000[\000\000", 0, 2); + x2s("\000[\000\134\000x\0000\0000\000\134\000x\0006\000A\000-\000\134\000x\0000\0000\000\134\000x\0006\000D\000]\000\000", "\000l\000\000", 0, 2); + ns("\000[\000\134\000x\0000\0000\000\134\000x\0006\000A\000-\000\134\000x\0000\0000\000\134\000x\0006\000D\000]\000\000", "\000n\000\000"); + ns("\000^\000[\0000\000-\0009\000A\000-\000F\000]\000+\000 \0000\000+\000 \000U\000N\000D\000E\000F\000 \000\000", "\0007\0005\000F\000 \0000\0000\0000\0000\0000\0000\0000\0000\000 \000S\000E\000C\000T\0001\0004\000A\000 \000n\000o\000t\000y\000p\000e\000 \000(\000)\000 \000 \000 \000 \000E\000x\000t\000e\000r\000n\000a\000l\000 \000 \000 \000 \000|\000 \000_\000r\000b\000_\000a\000p\000p\000l\000y\000\000"); + x2s("\000[\000\134\000[\000]\000\000", "\000[\000\000", 0, 2); + x2s("\000[\000\134\000]\000]\000\000", "\000]\000\000", 0, 2); + x2s("\000[\000&\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000[\000a\000b\000]\000]\000\000", "\000b\000\000", 0, 2); + x2s("\000[\000[\000a\000b\000]\000c\000]\000\000", "\000c\000\000", 0, 2); + ns("\000[\000[\000^\000a\000]\000]\000\000", "\000a\000\000"); + ns("\000[\000^\000[\000a\000]\000]\000\000", "\000a\000\000"); + x2s("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000a\000\000"); + ns("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000c\000\000"); + x2s("\000[\000a\000-\000z\000&\000&\000b\000-\000y\000&\000&\000c\000-\000x\000]\000\000", "\000w\000\000", 0, 2); + ns("\000[\000^\000a\000-\000z\000&\000&\000b\000-\000y\000&\000&\000c\000-\000x\000]\000\000", "\000w\000\000"); + x2s("\000[\000[\000^\000a\000&\000&\000a\000]\000&\000&\000a\000-\000z\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000[\000^\000a\000&\000&\000a\000]\000&\000&\000a\000-\000z\000]\000\000", "\000a\000\000"); + x2s("\000[\000[\000^\000a\000-\000z\000&\000&\000b\000c\000d\000e\000f\000]\000&\000&\000[\000^\000c\000-\000g\000]\000]\000\000", "\000h\000\000", 0, 2); + ns("\000[\000[\000^\000a\000-\000z\000&\000&\000b\000c\000d\000e\000f\000]\000&\000&\000[\000^\000c\000-\000g\000]\000]\000\000", "\000c\000\000"); + x2s("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000c\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000e\000\000", 0, 2); + ns("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000f\000\000"); + x2s("\000[\000a\000-\000&\000&\000-\000a\000]\000\000", "\000-\000\000", 0, 2); + ns("\000[\000a\000\134\000-\000&\000&\000\134\000-\000a\000]\000\000", "\000&\000\000"); + ns("\000\134\000w\000a\000b\000c\000\000", "\000 \000a\000b\000c\000\000"); + x2s("\000a\000\134\000W\000b\000c\000\000", "\000a\000 \000b\000c\000\000", 0, 8); + x2s("\000a\000.\000b\000.\000c\000\000", "\000a\000a\000b\000b\000c\000\000", 0, 10); + x2s("\000.\000\134\000w\000b\000\134\000W\000.\000.\000c\000\000", "\000a\000b\000b\000 \000b\000c\000c\000\000", 0, 14); + x2s("\000\134\000s\000\134\000w\000z\000z\000z\000\000", "\000 \000z\000z\000z\000z\000\000", 0, 10); + x2s("\000a\000a\000.\000b\000\000", "\000a\000a\000b\000b\000\000", 0, 8); + ns("\000.\000a\000\000", "\000a\000b\000\000"); + x2s("\000.\000a\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000^\000a\000\000", "\000a\000\000", 0, 2); + x2s("\000^\000a\000$\000\000", "\000a\000\000", 0, 2); + x2s("\000^\000\134\000w\000$\000\000", "\000a\000\000", 0, 2); + ns("\000^\000\134\000w\000$\000\000", "\000 \000\000"); + x2s("\000^\000\134\000w\000a\000b\000$\000\000", "\000z\000a\000b\000\000", 0, 6); + x2s("\000^\000\134\000w\000a\000b\000c\000d\000e\000f\000$\000\000", "\000z\000a\000b\000c\000d\000e\000f\000\000", 0, 14); + x2s("\000^\000\134\000w\000.\000.\000.\000d\000e\000f\000$\000\000", "\000z\000a\000b\000c\000d\000e\000f\000\000", 0, 14); + x2s("\000\134\000w\000\134\000w\000\134\000s\000\134\000W\000a\000a\000a\000\134\000d\000\000", "\000a\000a\000 \000 \000a\000a\000a\0004\000\000", 0, 16); + x2s("\000\134\000A\000\134\000Z\000\000", "\000\000", 0, 0); + x2s("\000\134\000A\000x\000y\000z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000x\000y\000z\000\134\000Z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000x\000y\000z\000\134\000z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000a\000\134\000Z\000\000", "\000a\000\000", 0, 2); + x2s("\000\134\000G\000a\000z\000\000", "\000a\000z\000\000", 0, 4); + ns("\000\134\000G\000z\000\000", "\000b\000z\000a\000\000"); + ns("\000a\000z\000\134\000G\000\000", "\000a\000z\000\000"); + ns("\000a\000z\000\134\000A\000\000", "\000a\000z\000\000"); + ns("\000a\000\134\000A\000z\000\000", "\000a\000z\000\000"); + x2s("\000\134\000^\000\134\000$\000\000", "\000^\000$\000\000", 0, 4); + x2s("\000^\000x\000?\000y\000\000", "\000x\000y\000\000", 0, 4); + x2s("\000^\000(\000x\000?\000y\000)\000\000", "\000x\000y\000\000", 0, 4); + x2s("\000\134\000w\000\000", "\000_\000\000", 0, 2); + ns("\000\134\000W\000\000", "\000_\000\000"); + x2s("\000(\000?\000=\000z\000)\000z\000\000", "\000z\000\000", 0, 2); + ns("\000(\000?\000=\000z\000)\000.\000\000", "\000a\000\000"); + x2s("\000(\000?\000!\000z\000)\000a\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000!\000z\000)\000a\000\000", "\000z\000\000"); + x2s("\000(\000?\000i\000:\000a\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000a\000)\000\000", "\000A\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000A\000)\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000i\000:\000A\000)\000\000", "\000b\000\000"); + x2s("\000(\000?\000i\000:\000[\000A\000-\000Z\000]\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000H\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000h\000\000", 0, 2); + ns("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000e\000\000"); + x2s("\000(\000?\000i\000:\000[\000A\000-\000c\000]\000)\000\000", "\000D\000\000", 0, 2); + ns("\000(\000?\000i\000:\000[\000^\000a\000-\000z\000]\000)\000\000", "\000A\000\000"); + ns("\000(\000?\000i\000:\000[\000^\000a\000-\000z\000]\000)\000\000", "\000a\000\000"); + x2s("\000(\000?\000i\000:\000[\000!\000-\000k\000]\000)\000\000", "\000Z\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000!\000-\000k\000]\000)\000\000", "\0007\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000T\000-\000}\000]\000)\000\000", "\000b\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000T\000-\000}\000]\000)\000\000", "\000{\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000\134\000?\000a\000)\000\000", "\000?\000A\000\000", 0, 4); + x2s("\000(\000?\000i\000:\000\134\000*\000A\000)\000\000", "\000*\000a\000\000", 0, 4); + ns("\000.\000\000", "\000\012\000\000"); + x2s("\000(\000?\000m\000:\000.\000)\000\000", "\000\012\000\000", 0, 2); + x2s("\000(\000?\000m\000:\000a\000.\000)\000\000", "\000a\000\012\000\000", 0, 4); + x2s("\000(\000?\000m\000:\000.\000b\000)\000\000", "\000a\000\012\000b\000\000", 2, 6); + x2s("\000.\000*\000a\000b\000c\000\000", "\000d\000d\000d\000a\000b\000d\000d\000\012\000d\000d\000a\000b\000c\000\000", 16, 26); + x2s("\000(\000?\000m\000:\000.\000*\000a\000b\000c\000)\000\000", "\000d\000d\000d\000a\000b\000d\000d\000a\000b\000c\000\000", 0, 20); + ns("\000(\000?\000i\000)\000(\000?\000-\000i\000)\000a\000\000", "\000A\000\000"); + ns("\000(\000?\000i\000)\000(\000?\000-\000i\000:\000a\000)\000\000", "\000A\000\000"); + x2s("\000a\000?\000\000", "\000\000", 0, 0); + x2s("\000a\000?\000\000", "\000b\000\000", 0, 0); + x2s("\000a\000?\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000*\000\000", "\000\000", 0, 0); + x2s("\000a\000*\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000*\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000a\000*\000\000", "\000b\000a\000a\000a\000a\000\000", 0, 0); + ns("\000a\000+\000\000", "\000\000"); + x2s("\000a\000+\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000+\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000a\000+\000\000", "\000a\000a\000b\000b\000b\000\000", 0, 4); + x2s("\000a\000+\000\000", "\000b\000a\000a\000a\000a\000\000", 2, 10); + x2s("\000.\000?\000\000", "\000\000", 0, 0); + x2s("\000.\000?\000\000", "\000f\000\000", 0, 2); + x2s("\000.\000?\000\000", "\000\012\000\000", 0, 0); + x2s("\000.\000*\000\000", "\000\000", 0, 0); + x2s("\000.\000*\000\000", "\000a\000b\000c\000d\000e\000\000", 0, 10); + x2s("\000.\000+\000\000", "\000z\000\000", 0, 2); + x2s("\000.\000+\000\000", "\000z\000d\000s\000w\000e\000r\000\012\000\000", 0, 12); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000b\000f\000b\000a\000c\000\000", 0, 8); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000c\000b\000a\000b\000f\000\000", 6, 14); + x2s("\000(\000(\000.\000*\000)\000a\000\134\0002\000f\000)\000\000", "\000b\000a\000c\000b\000a\000b\000f\000\000", 6, 14); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000c\000z\000z\000z\000z\000z\000z\000\012\000b\000a\000z\000z\000\012\000z\000z\000z\000z\000b\000a\000b\000f\000\000", 38, 46); + x2s("\000a\000|\000b\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000|\000b\000\000", "\000b\000\000", 0, 2); + x2s("\000|\000a\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000|\000a\000)\000\000", "\000a\000\000", 0, 0); + x2s("\000a\000b\000|\000b\000c\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000a\000b\000|\000b\000c\000\000", "\000b\000c\000\000", 0, 4); + x2s("\000z\000(\000?\000:\000a\000b\000|\000b\000c\000)\000\000", "\000z\000b\000c\000\000", 0, 6); + x2s("\000a\000(\000?\000:\000a\000b\000|\000b\000c\000)\000c\000\000", "\000a\000a\000b\000c\000\000", 0, 8); + x2s("\000a\000b\000|\000(\000?\000:\000a\000c\000|\000a\000z\000)\000\000", "\000a\000z\000\000", 0, 4); + x2s("\000a\000|\000b\000|\000c\000\000", "\000d\000c\000\000", 2, 4); + x2s("\000a\000|\000b\000|\000c\000d\000|\000e\000f\000g\000|\000h\000|\000i\000j\000k\000|\000l\000m\000n\000|\000o\000|\000p\000q\000|\000r\000s\000t\000u\000v\000w\000x\000|\000y\000z\000\000", "\000p\000q\000r\000\000", 0, 4); + ns("\000a\000|\000b\000|\000c\000d\000|\000e\000f\000g\000|\000h\000|\000i\000j\000k\000|\000l\000m\000n\000|\000o\000|\000p\000q\000|\000r\000s\000t\000u\000v\000w\000x\000|\000y\000z\000\000", "\000m\000n\000\000"); + x2s("\000a\000|\000^\000z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000^\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000\134\000G\000z\000\000", "\000b\000z\000a\000\000", 4, 6); + x2s("\000a\000|\000\134\000G\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000\134\000A\000z\000\000", "\000b\000z\000a\000\000", 4, 6); + x2s("\000a\000|\000\134\000A\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000b\000\134\000Z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000b\000\134\000Z\000\000", "\000b\000\000", 0, 2); + x2s("\000a\000|\000b\000\134\000z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000b\000\134\000z\000\000", "\000b\000\000", 0, 2); + x2s("\000\134\000w\000|\000\134\000s\000\000", "\000 \000\000", 0, 2); + ns("\000\134\000w\000|\000\134\000w\000\000", "\000 \000\000"); + x2s("\000\134\000w\000|\000%\000\000", "\000%\000\000", 0, 2); + x2s("\000\134\000w\000|\000[\000&\000$\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000b\000-\000d\000]\000|\000[\000^\000e\000-\000z\000]\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000|\000[\000c\000-\000f\000]\000)\000|\000b\000z\000\000", "\000d\000z\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000|\000[\000c\000-\000f\000]\000)\000|\000b\000z\000\000", "\000b\000z\000\000", 0, 4); + x2s("\000a\000b\000c\000|\000(\000?\000=\000z\000z\000)\000.\000.\000f\000\000", "\000z\000z\000f\000\000", 0, 6); + x2s("\000a\000b\000c\000|\000(\000?\000!\000z\000z\000)\000.\000.\000f\000\000", "\000a\000b\000f\000\000", 0, 6); + x2s("\000(\000?\000=\000z\000a\000)\000.\000.\000a\000|\000(\000?\000=\000z\000z\000)\000.\000.\000a\000\000", "\000z\000z\000a\000\000", 0, 6); + ns("\000(\000?\000>\000a\000|\000a\000b\000d\000)\000c\000\000", "\000a\000b\000d\000c\000\000"); + x2s("\000(\000?\000>\000a\000b\000d\000|\000a\000)\000c\000\000", "\000a\000b\000d\000c\000\000", 0, 8); + x2s("\000a\000?\000|\000b\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000?\000|\000b\000\000", "\000b\000\000", 0, 0); + x2s("\000a\000?\000|\000b\000\000", "\000\000", 0, 0); + x2s("\000a\000*\000|\000b\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000a\000*\000|\000b\000*\000\000", "\000b\000a\000\000", 0, 0); + x2s("\000a\000*\000|\000b\000*\000\000", "\000a\000b\000\000", 0, 2); + x2s("\000a\000+\000|\000b\000*\000\000", "\000\000", 0, 0); + x2s("\000a\000+\000|\000b\000*\000\000", "\000b\000b\000b\000\000", 0, 6); + x2s("\000a\000+\000|\000b\000*\000\000", "\000a\000b\000b\000b\000\000", 0, 2); + ns("\000a\000+\000|\000b\000+\000\000", "\000\000"); + x2s("\000(\000a\000|\000b\000)\000?\000\000", "\000b\000\000", 0, 2); + x2s("\000(\000a\000|\000b\000)\000*\000\000", "\000b\000a\000\000", 0, 4); + x2s("\000(\000a\000|\000b\000)\000+\000\000", "\000b\000a\000b\000\000", 0, 6); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000c\000a\000a\000b\000b\000c\000\000", 0, 8); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000a\000a\000b\000c\000a\000\000", 2, 10); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000a\000b\000z\000c\000a\000\000", 0, 4); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 10); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000b\000a\000a\000a\000b\000a\000\000", 2, 8); + x2s("\000(\000?\000:\000a\000|\000b\000)\000(\000?\000:\000a\000|\000b\000)\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000(\000?\000:\000a\000*\000|\000b\000*\000)\000(\000?\000:\000a\000*\000|\000b\000*\000)\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 6); + x2s("\000(\000?\000:\000a\000*\000|\000b\000*\000)\000(\000?\000:\000a\000+\000|\000b\000+\000)\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + x2s("\000(\000?\000:\000a\000+\000|\000b\000+\000)\000{\0002\000}\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + x2s("\000h\000{\0000\000,\000}\000\000", "\000h\000h\000h\000h\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000+\000|\000b\000+\000)\000{\0001\000,\0002\000}\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + ns("\000a\000x\000{\0002\000}\000*\000a\000\000", "\0000\000a\000x\000x\000x\000a\0001\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000a\000\000", "\0000\000a\000X\000X\000X\000a\0000\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000?\000a\000\000", "\0000\000a\000X\000X\000X\000a\0000\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000?\000a\000\000", "\0000\000a\000X\000X\000X\000X\000a\0000\000\000"); + x2s("\000^\000a\000{\0002\000,\000}\000?\000a\000$\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000^\000[\000a\000-\000z\000]\000{\0002\000,\000}\000?\000$\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000(\000?\000:\000a\000+\000|\000\134\000A\000b\000*\000)\000c\000c\000\000", "\000c\000c\000\000", 0, 4); + ns("\000(\000?\000:\000a\000+\000|\000\134\000A\000b\000*\000)\000c\000c\000\000", "\000a\000b\000c\000c\000\000"); + x2s("\000(\000?\000:\000^\000a\000+\000|\000b\000+\000)\000*\000c\000\000", "\000a\000a\000b\000b\000b\000a\000b\000c\000\000", 12, 16); + x2s("\000(\000?\000:\000^\000a\000+\000|\000b\000+\000)\000*\000c\000\000", "\000a\000a\000b\000b\000b\000b\000c\000\000", 0, 14); + x2s("\000a\000|\000(\000?\000i\000)\000c\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|\000a\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|\000a\000\000", "\000A\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000c\000)\000|\000a\000\000", "\000C\000\000", 0, 2); + ns("\000(\000?\000i\000:\000c\000)\000|\000a\000\000", "\000A\000\000"); + x2s("\000[\000a\000b\000c\000]\000?\000\000", "\000a\000b\000c\000\000", 0, 2); + x2s("\000[\000a\000b\000c\000]\000*\000\000", "\000a\000b\000c\000\000", 0, 6); + x2s("\000[\000^\000a\000b\000c\000]\000*\000\000", "\000a\000b\000c\000\000", 0, 0); + ns("\000[\000^\000a\000b\000c\000]\000+\000\000", "\000a\000b\000c\000\000"); + x2s("\000a\000?\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000b\000a\000?\000?\000b\000\000", "\000b\000a\000b\000\000", 0, 6); + x2s("\000a\000*\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000b\000a\000*\000?\000\000", "\000b\000a\000a\000\000", 0, 2); + x2s("\000b\000a\000*\000?\000b\000\000", "\000b\000a\000a\000b\000\000", 0, 8); + x2s("\000a\000+\000?\000\000", "\000a\000a\000a\000\000", 0, 2); + x2s("\000b\000a\000+\000?\000\000", "\000b\000a\000a\000\000", 0, 4); + x2s("\000b\000a\000+\000?\000b\000\000", "\000b\000a\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000?\000)\000?\000?\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000?\000?\000)\000?\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000?\000)\000+\000?\000\000", "\000a\000a\000a\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000+\000)\000?\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000+\000)\000?\000?\000b\000\000", "\000a\000a\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000?\000{\0002\000}\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000b\000)\000?\000{\0002\000}\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000*\000{\0000\000}\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000b\000)\000{\0003\000,\000}\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 16); + ns("\000(\000?\000:\000a\000b\000)\000{\0003\000,\000}\000\000", "\000a\000b\000a\000b\000\000"); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000\000", "\000a\000b\000a\000b\000a\000b\000\000", 0, 12); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 16); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000?\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000{\000,\000}\000\000", "\000a\000b\000{\000,\000}\000\000", 0, 10); + x2s("\000(\000?\000:\000a\000b\000c\000)\000+\000?\000{\0002\000}\000\000", "\000a\000b\000c\000a\000b\000c\000a\000b\000c\000\000", 0, 12); + x2s("\000(\000?\000:\000X\000*\000)\000(\000?\000i\000:\000x\000a\000)\000\000", "\000X\000X\000X\000a\000\000", 0, 8); + x2s("\000(\000d\000+\000)\000(\000[\000^\000a\000b\000c\000]\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x2s("\000(\000[\000^\000a\000b\000c\000]\000*\000)\000(\000[\000^\000a\000b\000c\000]\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x2s("\000(\000\134\000w\000+\000)\000(\000\134\000w\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x3s("\000(\000a\000)\000\000", "\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000b\000)\000\000", "\000a\000b\000\000", 0, 4, 1); + x2s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4); + x3s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 1); + x3s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 2); + x3s("\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000a\000b\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 20); + x3s("\000(\000a\000b\000)\000(\000c\000d\000)\000\000", "\000a\000b\000c\000d\000\000", 0, 4, 1); + x3s("\000(\000a\000b\000)\000(\000c\000d\000)\000\000", "\000a\000b\000c\000d\000\000", 4, 8, 2); + x3s("\000(\000)\000(\000a\000)\000b\000c\000(\000d\000e\000f\000)\000g\000h\000i\000j\000k\000\000", "\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000\000", 6, 12, 3); + x3s("\000(\000(\000)\000(\000a\000)\000b\000c\000(\000d\000e\000f\000)\000g\000h\000i\000j\000k\000)\000\000", "\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000\000", 6, 12, 4); + x2s("\000(\000^\000a\000)\000\000", "\000a\000\000", 0, 2); + x3s("\000(\000a\000)\000|\000(\000a\000)\000\000", "\000b\000a\000\000", 2, 4, 1); + x3s("\000(\000^\000a\000)\000|\000(\000a\000)\000\000", "\000b\000a\000\000", 2, 4, 2); + x3s("\000(\000a\000?\000)\000\000", "\000a\000a\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000*\000)\000\000", "\000a\000a\000a\000\000", 0, 6, 1); + x3s("\000(\000a\000*\000)\000\000", "\000\000", 0, 0, 1); + x3s("\000(\000a\000+\000)\000\000", "\000a\000a\000a\000a\000a\000a\000a\000\000", 0, 14, 1); + x3s("\000(\000a\000+\000|\000b\000*\000)\000\000", "\000b\000b\000b\000a\000a\000\000", 0, 6, 1); + x3s("\000(\000a\000+\000|\000b\000?\000)\000\000", "\000b\000b\000b\000a\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000b\000c\000)\000?\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000a\000b\000c\000)\000*\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000x\000y\000z\000|\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000[\000x\000y\000z\000]\000[\000a\000b\000c\000]\000|\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000i\000:\000a\000b\000c\000)\000)\000\000", "\000A\000b\000C\000\000", 0, 6, 1); + x2s("\000(\000a\000b\000c\000)\000(\000?\000i\000:\000\134\0001\000)\000\000", "\000a\000b\000c\000A\000B\000C\000\000", 0, 12); + x3s("\000(\000(\000?\000m\000:\000a\000.\000c\000)\000)\000\000", "\000a\000\012\000c\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000=\000a\000z\000)\000a\000)\000\000", "\000a\000z\000b\000\000", 0, 2, 1); + x3s("\000a\000b\000c\000|\000(\000.\000a\000b\000d\000)\000\000", "\000z\000a\000b\000d\000\000", 0, 8, 1); + x2s("\000(\000?\000:\000a\000b\000c\000)\000|\000(\000A\000B\000C\000)\000\000", "\000a\000b\000c\000\000", 0, 6); + x3s("\000(\000?\000i\000:\000(\000a\000b\000c\000)\000)\000|\000(\000z\000z\000z\000)\000\000", "\000A\000B\000C\000\000", 0, 6, 1); + x3s("\000a\000*\000(\000.\000)\000\000", "\000a\000a\000a\000a\000z\000\000", 8, 10, 1); + x3s("\000a\000*\000?\000(\000.\000)\000\000", "\000a\000a\000a\000a\000z\000\000", 0, 2, 1); + x3s("\000a\000*\000?\000(\000c\000)\000\000", "\000a\000a\000a\000a\000c\000\000", 8, 10, 1); + x3s("\000[\000b\000c\000d\000]\000a\000*\000(\000.\000)\000\000", "\000c\000a\000a\000a\000a\000z\000\000", 10, 12, 1); + x3s("\000(\000\134\000A\000b\000b\000)\000c\000c\000\000", "\000b\000b\000c\000c\000\000", 0, 4, 1); + ns("\000(\000\134\000A\000b\000b\000)\000c\000c\000\000", "\000z\000b\000b\000c\000c\000\000"); + x3s("\000(\000^\000b\000b\000)\000c\000c\000\000", "\000b\000b\000c\000c\000\000", 0, 4, 1); + ns("\000(\000^\000b\000b\000)\000c\000c\000\000", "\000z\000b\000b\000c\000c\000\000"); + x3s("\000c\000c\000(\000b\000b\000$\000)\000\000", "\000c\000c\000b\000b\000\000", 4, 8, 1); + ns("\000c\000c\000(\000b\000b\000$\000)\000\000", "\000c\000c\000b\000b\000b\000\000"); + ns("\000(\000\134\0001\000)\000\000", "\000\000"); + ns("\000\134\0001\000(\000a\000)\000\000", "\000a\000a\000\000"); + ns("\000(\000a\000(\000b\000)\000\134\0001\000)\000\134\0002\000+\000\000", "\000a\000b\000a\000b\000b\000\000"); + ns("\000(\000?\000:\000(\000?\000:\000\134\0001\000|\000z\000)\000(\000a\000)\000)\000+\000$\000\000", "\000z\000a\000a\000\000"); + x2s("\000(\000?\000:\000(\000?\000:\000\134\0001\000|\000z\000)\000(\000a\000)\000)\000+\000$\000\000", "\000z\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000a\000)\000(\000?\000=\000\134\0001\000)\000\000", "\000a\000a\000\000", 0, 2); + ns("\000(\000a\000)\000$\000|\000\134\0001\000\000", "\000a\000z\000\000"); + x2s("\000(\000a\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + ns("\000(\000a\000)\000\134\0001\000\000", "\000a\000b\000\000"); + x2s("\000(\000a\000?\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000(\000a\000?\000?\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 0); + x2s("\000(\000a\000*\000)\000\134\0001\000\000", "\000a\000a\000a\000a\000a\000\000", 0, 8); + x3s("\000(\000a\000*\000)\000\134\0001\000\000", "\000a\000a\000a\000a\000a\000\000", 0, 4, 1); + x2s("\000a\000(\000b\000*\000)\000\134\0001\000\000", "\000a\000b\000b\000b\000b\000\000", 0, 10); + x2s("\000a\000(\000b\000*\000)\000\134\0001\000\000", "\000a\000b\000\000", 0, 2); + x2s("\000(\000a\000*\000)\000(\000b\000*\000)\000\134\0001\000\134\0002\000\000", "\000a\000a\000a\000b\000b\000a\000a\000a\000b\000b\000\000", 0, 20); + x2s("\000(\000a\000*\000)\000(\000b\000*\000)\000\134\0002\000\000", "\000a\000a\000a\000b\000b\000b\000b\000\000", 0, 14); + x2s("\000(\000(\000(\000(\000(\000(\000(\000a\000*\000)\000b\000)\000)\000)\000)\000)\000)\000c\000\134\0007\000\000", "\000a\000a\000a\000b\000c\000a\000a\000a\000\000", 0, 16); + x3s("\000(\000(\000(\000(\000(\000(\000(\000a\000*\000)\000b\000)\000)\000)\000)\000)\000)\000c\000\134\0007\000\000", "\000a\000a\000a\000b\000c\000a\000a\000a\000\000", 0, 6, 7); + x2s("\000(\000a\000)\000(\000b\000)\000(\000c\000)\000\134\0002\000\134\0001\000\134\0003\000\000", "\000a\000b\000c\000b\000a\000c\000\000", 0, 12); + x2s("\000(\000[\000a\000-\000d\000]\000)\000\134\0001\000\000", "\000c\000c\000\000", 0, 4); + x2s("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "\000f\0005\000 \000f\0005\000 \000\000", 0, 12); + ns("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "\000f\0005\000 \000f\0005\000\000"); + x2s("\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000w\000h\000o\000w\000h\000o\000\000", 0, 12); + x2s("\000.\000.\000.\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000a\000b\000c\000w\000h\000o\000w\000h\000o\000\000", 0, 18); + x2s("\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000c\000b\000c\000c\000b\000c\000\000", 0, 12); + x2s("\000(\000^\000a\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + ns("\000(\000^\000a\000)\000\134\0001\000\000", "\000b\000a\000a\000\000"); + ns("\000(\000a\000$\000)\000\134\0001\000\000", "\000a\000a\000\000"); + ns("\000(\000a\000b\000\134\000Z\000)\000\134\0001\000\000", "\000a\000b\000\000"); + x2s("\000(\000a\000*\000\134\000Z\000)\000\134\0001\000\000", "\000a\000\000", 2, 2); + x2s("\000.\000(\000a\000*\000\134\000Z\000)\000\134\0001\000\000", "\000b\000a\000\000", 2, 4); + x3s("\000(\000.\000(\000a\000b\000c\000)\000\134\0002\000)\000\000", "\000z\000a\000b\000c\000a\000b\000c\000\000", 0, 14, 1); + x3s("\000(\000.\000(\000.\000.\000\134\000d\000.\000)\000\134\0002\000)\000\000", "\000z\0001\0002\0003\0004\0001\0002\0003\0004\000\000", 0, 18, 1); + x2s("\000(\000(\000?\000i\000:\000a\000z\000)\000)\000\134\0001\000\000", "\000A\000z\000A\000z\000\000", 0, 8); + ns("\000(\000(\000?\000i\000:\000a\000z\000)\000)\000\134\0001\000\000", "\000A\000z\000a\000z\000\000"); + x2s("\000(\000?\000<\000=\000a\000)\000b\000\000", "\000a\000b\000\000", 2, 4); + ns("\000(\000?\000<\000=\000a\000)\000b\000\000", "\000b\000b\000\000"); + x2s("\000(\000?\000<\000=\000a\000|\000b\000)\000b\000\000", "\000b\000b\000\000", 2, 4); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000)\000b\000\000", "\000b\000c\000b\000\000", 4, 6); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000)\000b\000\000", "\000a\000b\000\000", 2, 4); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000|\000|\000d\000e\000f\000g\000h\000i\000j\000|\000k\000l\000m\000n\000o\000p\000q\000|\000r\000)\000z\000\000", "\000r\000z\000\000", 2, 4); + x2s("\000(\000a\000)\000\134\000g\000<\0001\000>\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000(\000?\000<\000!\000a\000)\000b\000\000", "\000c\000b\000\000", 2, 4); + ns("\000(\000?\000<\000!\000a\000)\000b\000\000", "\000a\000b\000\000"); + x2s("\000(\000?\000<\000!\000a\000|\000b\000c\000)\000b\000\000", "\000b\000b\000b\000\000", 0, 2); + ns("\000(\000?\000<\000!\000a\000|\000b\000c\000)\000z\000\000", "\000b\000c\000z\000\000"); + x2s("\000(\000?\000<\000n\000a\000m\000e\0001\000>\000a\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000<\000n\000a\000m\000e\000_\0002\000>\000a\000b\000)\000\134\000g\000<\000n\000a\000m\000e\000_\0002\000>\000\000", "\000a\000b\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000a\000m\000e\000_\0003\000>\000.\000z\000v\000.\000)\000\134\000k\000<\000n\000a\000m\000e\000_\0003\000>\000\000", "\000a\000z\000v\000b\000a\000z\000v\000b\000\000", 0, 16); + x2s("\000(\000?\000<\000=\000\134\000g\000<\000a\000b\000>\000)\000|\000-\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000a\000b\000>\000X\000y\000Z\000)\000\000", "\000X\000y\000Z\000\000", 6, 6); + x2s("\000(\000?\000<\000n\000>\000|\000a\000\134\000g\000<\000n\000>\000)\000+\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000<\000n\000>\000|\000\134\000(\000\134\000g\000<\000n\000>\000\134\000)\000)\000+\000$\000\000", "\000(\000)\000(\000(\000)\000)\000\000", 0, 12); + x3s("\000\134\000g\000<\000n\000>\000(\000?\000<\000n\000>\000.\000)\000{\0000\000}\000\000", "\000X\000\000", 0, 2, 1); + x2s("\000\134\000g\000<\000n\000>\000(\000a\000b\000c\000|\000d\000f\000(\000?\000<\000n\000>\000.\000Y\000Z\000)\000{\0002\000,\0008\000}\000)\000{\0000\000}\000\000", "\000X\000Y\000Z\000\000", 0, 6); + x2s("\000\134\000A\000(\000?\000<\000n\000>\000(\000a\000\134\000g\000<\000n\000>\000)\000|\000)\000\134\000z\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000>\000|\000\134\000g\000<\000m\000>\000\134\000g\000<\000n\000>\000)\000\134\000z\000|\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000m\000>\000a\000|\000(\000b\000)\000\134\000g\000<\000m\000>\000)\000\000", "\000b\000b\000b\000b\000a\000b\000b\000a\000\000", 0, 16); + x2s("\000(\000?\000<\000n\000a\000m\000e\0001\0002\0004\0000\000>\000\134\000w\000+\000\134\000s\000x\000)\000a\000+\000\134\000k\000<\000n\000a\000m\000e\0001\0002\0004\0000\000>\000\000", "\000 \000 \000f\000g\000 \000x\000a\000a\000a\000a\000a\000a\000a\000a\000f\000g\000 \000x\000\000", 4, 36); + x3s("\000(\000z\000)\000(\000)\000(\000)\000(\000?\000<\000_\0009\000>\000a\000)\000\134\000g\000<\000_\0009\000>\000\000", "\000z\000a\000a\000\000", 4, 6, 1); + x2s("\000(\000.\000)\000(\000(\000(\000?\000<\000_\000>\000a\000)\000)\000)\000\134\000k\000<\000_\000>\000\000", "\000z\000a\000a\000\000", 0, 6); + x2s("\000(\000(\000?\000<\000n\000a\000m\000e\0001\000>\000\134\000d\000)\000|\000(\000?\000<\000n\000a\000m\000e\0002\000>\000\134\000w\000)\000)\000(\000\134\000k\000<\000n\000a\000m\000e\0001\000>\000|\000\134\000k\000<\000n\000a\000m\000e\0002\000>\000)\000\000", "\000f\000f\000\000", 0, 4); + x2s("\000(\000?\000:\000(\000?\000<\000x\000>\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000?\000<\000x\000>\000a\000b\000c\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000a\000b\000c\000e\000f\000g\000e\000f\000g\000\000", 6, 18); + ns("\000(\000?\000:\000(\000?\000<\000x\000>\000a\000b\000c\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000a\000b\000c\000e\000f\000g\000\000"); + x2s("\000(\000?\000:\000(\000?\000<\000n\0001\000>\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000)\000\134\000k\000<\000n\0001\000>\000$\000\000", "\000a\000-\000p\000y\000u\000m\000p\000y\000u\000m\000\000", 4, 20); + x3s("\000(\000?\000:\000(\000?\000<\000n\0001\000>\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000)\000\134\000k\000<\000n\0001\000>\000$\000\000", "\000x\000x\000x\000x\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000l\000m\000n\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000l\000m\000n\000\000", 8, 36, 14); + x3s("\000(\000?\000<\000n\000a\000m\000e\0001\000>\000)\000(\000?\000<\000n\000a\000m\000e\0002\000>\000)\000(\000?\000<\000n\000a\000m\000e\0003\000>\000)\000(\000?\000<\000n\000a\000m\000e\0004\000>\000)\000(\000?\000<\000n\000a\000m\000e\0005\000>\000)\000(\000?\000<\000n\000a\000m\000e\0006\000>\000)\000(\000?\000<\000n\000a\000m\000e\0007\000>\000)\000(\000?\000<\000n\000a\000m\000e\0008\000>\000)\000(\000?\000<\000n\000a\000m\000e\0009\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0000\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0001\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0002\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0003\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0004\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0005\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0006\000>\000a\000a\000a\000)\000(\000?\000<\000n\000a\000m\000e\0001\0007\000>\000)\000$\000\000", "\000a\000a\000a\000\000", 0, 6, 16); + x2s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(\000a\000)\000)\000)\000)\000)\000)\000\000", 0, 26); + x3s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(\000(\000(\000a\000)\000)\000)\000)\000)\000)\000)\000)\000\000", 0, 34, 1); + x2s("\000\134\000g\000<\000b\000a\000r\000>\000|\000\134\000z\000E\000N\000D\000(\000?\000<\000b\000a\000r\000>\000.\000*\000a\000b\000c\000$\000)\000\000", "\000a\000b\000c\000x\000x\000x\000a\000b\000c\000\000", 0, 18); + x2s("\000\134\000g\000<\0001\000>\000|\000\134\000z\000E\000N\000D\000(\000.\000a\000.\000)\000\000", "\000b\000a\000c\000\000", 0, 6); + x3s("\000\134\000g\000<\000_\000A\000>\000\134\000g\000<\000_\000A\000>\000|\000\134\000z\000E\000N\000D\000(\000.\000a\000.\000)\000(\000?\000<\000_\000A\000>\000.\000b\000.\000)\000\000", "\000x\000b\000x\000y\000b\000y\000\000", 6, 12, 1); + x2s("\000\134\000A\000(\000?\000:\000\134\000g\000<\000p\000o\000n\000>\000|\000\134\000g\000<\000p\000a\000n\000>\000|\000\134\000z\000E\000N\000D\000 \000 \000(\000?\000<\000p\000a\000n\000>\000a\000|\000c\000\134\000g\000<\000p\000o\000n\000>\000c\000)\000(\000?\000<\000p\000o\000n\000>\000b\000|\000d\000\134\000g\000<\000p\000a\000n\000>\000d\000)\000)\000$\000\000", "\000c\000d\000c\000b\000c\000d\000c\000\000", 0, 14); + x2s("\000\134\000A\000(\000?\000<\000n\000>\000|\000a\000\134\000g\000<\000m\000>\000)\000\134\000z\000|\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000m\000>\000\134\000g\000<\000n\000>\000)\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000>\000(\000a\000|\000b\000\134\000g\000<\000n\000>\000c\000)\000{\0003\000,\0005\000}\000)\000\000", "\000b\000a\000a\000a\000a\000c\000a\000\000", 2, 10); + x2s("\000(\000?\000<\000n\000>\000(\000a\000|\000b\000\134\000g\000<\000n\000>\000c\000)\000{\0003\000,\0005\000}\000)\000\000", "\000b\000a\000a\000a\000a\000c\000a\000a\000a\000a\000a\000\000", 0, 20); + x2s("\000(\000?\000<\000p\000a\000r\000e\000>\000\134\000(\000(\000[\000^\000\134\000(\000\134\000)\000]\000+\000+\000|\000\134\000g\000<\000p\000a\000r\000e\000>\000)\000*\000+\000\134\000)\000)\000\000", "\000(\000(\000a\000)\000)\000\000", 0, 10); + x2s("\000(\000)\000*\000\134\0001\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000)\000*\000\134\0001\000\134\0002\000\000", "\000\000", 0, 0); + x3s("\000(\000?\000:\000\134\0001\000a\000|\000(\000)\000)\000*\000\000", "\000a\000\000", 0, 0, 1); + x2s("\000x\000(\000(\000.\000)\000*\000)\000*\000x\000\000", "\0000\000x\0001\000x\0002\000x\0003\000\000", 2, 12); + x2s("\000x\000(\000(\000.\000)\000*\000)\000*\000x\000(\000?\000i\000:\000\134\0001\000)\000\134\000Z\000\000", "\0000\000x\0001\000x\0002\000x\0001\000X\0002\000\000", 2, 18); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000)\000*\000\134\0002\000\134\0005\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000x\000)\000|\000(\000)\000|\000(\000)\000)\000*\000\134\0002\000b\000\134\0005\000\000", "\000b\000\000", 0, 2); + x2s("\217\372\000\000", "\217\372\000\000", 0, 2); + x2s("\000\000", "0B\000\000", 0, 0); + x2s("0B\000\000", "0B\000\000", 0, 2); + ns("0D\000\000", "0B\000\000"); + x2s("0F0F\000\000", "0F0F\000\000", 0, 4); + x2s("0B0D0F\000\000", "0B0D0F\000\000", 0, 6); + x2s("0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S\000\000", "0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S\000\000", 0, 70); + x2s("0B\000\000", "0D0B\000\000", 2, 4); + x2s("0D0F\000\000", "0B0D0F\000\000", 2, 6); + x2s("e\207\000\000", "e\207\000\000", 0, 2); + x2s("\000.\000\000", "0B\000\000", 0, 2); + x2s("\000.\000.\000\000", "0K0M\000\000", 0, 4); + x2s("\000\134\000w\000\000", "0J\000\000", 0, 2); + ns("\000\134\000W\000\000", "0B\000\000"); + x2s("\000[\000\134\000W\000]\000\000", "0F\000$\000\000", 2, 4); + x2s("\000\134\000S\000\000", "0]\000\000", 0, 2); + x2s("\000\134\000S\000\000", "o\042\000\000", 0, 2); + x2s("\000\134\000b\000\000", "l\027\000 \000\000", 0, 0); + x2s("\000\134\000b\000\000", "\000 0{\000\000", 2, 2); + x2s("\000\134\000B\000\000", "0[0]\000 \000\000", 2, 2); + x2s("\000\134\000B\000\000", "0F\000 \000\000", 4, 4); + x2s("\000\134\000B\000\000", "\000 0D\000\000", 0, 0); + x2s("\000[0_0a\000]\000\000", "0a\000\000", 0, 2); + ns("\000[0j0k\000]\000\000", "0l\000\000"); + x2s("\000[0F\000-0J\000]\000\000", "0H\000\000", 0, 2); + ns("\000[\000^0Q\000]\000\000", "0Q\000\000"); + x2s("\000[\000\134\000w\000]\000\000", "0m\000\000", 0, 2); + ns("\000[\000\134\000d\000]\000\000", "0u\000\000"); + x2s("\000[\000\134\000D\000]\000\000", "0o\000\000", 0, 2); + ns("\000[\000\134\000s\000]\000\000", "0O\000\000"); + x2s("\000[\000\134\000S\000]\000\000", "0x\000\000", 0, 2); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "0\210\000\000", 0, 2); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "\000 \000 \000 0\210\000\000", 6, 8); + ns("\000\134\000w\233<\216\312\000\000", "\000 \233<\216\312\000\000"); + x2s("\233<\000\134\000W\216\312\000\000", "\233<\000 \216\312\000\000", 0, 6); + x2s("0B\000.0D\000.0F\000\000", "0B0B0D0D0F\000\000", 0, 10); + x2s("\000.\000\134\000w0F\000\134\000W\000.\000.0^\000\000", "0H0F0F\000 0F0^0^\000\000", 0, 14); + x2s("\000\134\000s\000\134\000w0S0S0S\000\000", "\000 0S0S0S0S\000\000", 0, 10); + x2s("0B0B\000.0Q\000\000", "0B0B0Q0Q\000\000", 0, 8); + ns("\000.0D\000\000", "0D0H\000\000"); + x2s("\000.0J\000\000", "0J0J\000\000", 0, 4); + x2s("\000^0B\000\000", "0B\000\000", 0, 2); + x2s("\000^0\200\000$\000\000", "0\200\000\000", 0, 2); + x2s("\000^\000\134\000w\000$\000\000", "0k\000\000", 0, 2); + x2s("\000^\000\134\000w0K0M0O0Q0S\000$\000\000", "\000z0K0M0O0Q0S\000\000", 0, 12); + x2s("\000^\000\134\000w\000.\000.\000.0F0H0J\000$\000\000", "\000z0B0D0F0F0H0J\000\000", 0, 14); + x2s("\000\134\000w\000\134\000w\000\134\000s\000\134\000W0J0J0J\000\134\000d\000\000", "\000a0J\000 \000 0J0J0J\0004\000\000", 0, 16); + x2s("\000\134\000A0_0a0d\000\000", "0_0a0d\000\000", 0, 6); + x2s("0\2000\2010\202\000\134\000Z\000\000", "0\2000\2010\202\000\000", 0, 6); + x2s("0K0M0O\000\134\000z\000\000", "0K0M0O\000\000", 0, 6); + x2s("0K0M0O\000\134\000Z\000\000", "0K0M0O\000\012\000\000", 0, 6); + x2s("\000\134\000G0}0t\000\000", "0}0t\000\000", 0, 4); + ns("\000\134\000G0H\000\000", "0F0H0J\000\000"); + ns("0h0f\000\134\000G\000\000", "0h0f\000\000"); + ns("0~0\177\000\134\000A\000\000", "0~0\177\000\000"); + ns("0~\000\134\000A0\177\000\000", "0~0\177\000\000"); + x2s("\000(\000?\000=0[\000)0[\000\000", "0[\000\000", 0, 2); + ns("\000(\000?\000=0F\000)\000.\000\000", "0D\000\000"); + x2s("\000(\000?\000!0F\000)0K\000\000", "0K\000\000", 0, 2); + ns("\000(\000?\000!0h\000)0B\000\000", "0h\000\000"); + x2s("\000(\000?\000i\000:0B\000)\000\000", "0B\000\000", 0, 2); + x2s("\000(\000?\000i\000:0v0y\000)\000\000", "0v0y\000\000", 0, 4); + ns("\000(\000?\000i\000:0D\000)\000\000", "0F\000\000"); + x2s("\000(\000?\000m\000:0\210\000.\000)\000\000", "0\210\000\012\000\000", 0, 4); + x2s("\000(\000?\000m\000:\000.0\201\000)\000\000", "0~\000\0120\201\000\000", 2, 6); + x2s("0B\000?\000\000", "\000\000", 0, 0); + x2s("Y\011\000?\000\000", "S\026\000\000", 0, 0); + x2s("Y\011\000?\000\000", "Y\011\000\000", 0, 2); + x2s("\221\317\000*\000\000", "\000\000", 0, 0); + x2s("\221\317\000*\000\000", "\221\317\000\000", 0, 2); + x2s("[P\000*\000\000", "[P[P[P\000\000", 0, 6); + x2s("\231\254\000*\000\000", "\236\177\231\254\231\254\231\254\231\254\000\000", 0, 0); + ns("\134q\000+\000\000", "\000\000"); + x2s("l\263\000+\000\000", "l\263\000\000", 0, 2); + x2s("fB\000+\000\000", "fBfBfBfB\000\000", 0, 8); + x2s("0H\000+\000\000", "0H0H0F0F0F\000\000", 0, 4); + x2s("0F\000+\000\000", "0J0F0F0F0F\000\000", 2, 10); + x2s("\000.\000?\000\000", "0_\000\000", 0, 2); + x2s("\000.\000*\000\000", "0q0t0w0z\000\000", 0, 8); + x2s("\000.\000+\000\000", "0\215\000\000", 0, 2); + x2s("\000.\000+\000\000", "0D0F0H0K\000\012\000\000", 0, 8); + x2s("0B\000|0D\000\000", "0B\000\000", 0, 2); + x2s("0B\000|0D\000\000", "0D\000\000", 0, 2); + x2s("0B0D\000|0D0F\000\000", "0B0D\000\000", 0, 4); + x2s("0B0D\000|0D0F\000\000", "0D0F\000\000", 0, 4); + x2s("0\222\000(\000?\000:0K0M\000|0M0O\000)\000\000", "0\2220K0M\000\000", 0, 6); + x2s("0\222\000(\000?\000:0K0M\000|0M0O\000)0Q\000\000", "0\2220M0O0Q\000\000", 0, 8); + x2s("0B0D\000|\000(\000?\000:0B0F\000|0B0\222\000)\000\000", "0B0\222\000\000", 0, 4); + x2s("0B\000|0D\000|0F\000\000", "0H0F\000\000", 2, 4); + x2s("0B\000|0D\000|0F0H\000|0J0K0M\000|0O\000|0Q0S0U\000|0W0Y0[\000|0]\000|0_0a\000|0d0f0h0j0k\000|0l0m\000\000", "0W0Y0[\000\000", 0, 6); + ns("0B\000|0D\000|0F0H\000|0J0K0M\000|0O\000|0Q0S0U\000|0W0Y0[\000|0]\000|0_0a\000|0d0f0h0j0k\000|0l0m\000\000", "0Y0[\000\000"); + x2s("0B\000|\000^0\217\000\000", "0v0B\000\000", 2, 4); + x2s("0B\000|\000^0\222\000\000", "0\2220B\000\000", 0, 2); + x2s("\233<\000|\000\134\000G\216\312\000\000", "0Q\216\312\233<\000\000", 4, 6); + x2s("\233<\000|\000\134\000G\216\312\000\000", "\216\312\233<\000\000", 0, 2); + x2s("\233<\000|\000\134\000A\216\312\000\000", "\000b\216\312\233<\000\000", 4, 6); + x2s("\233<\000|\000\134\000A\216\312\000\000", "\216\312\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\233<\000\000", 2, 4); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\000\012\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000z\000\000", "\216\312\233<\000\000", 2, 4); + x2s("\233<\000|\216\312\000\134\000z\000\000", "\216\312\000\000", 0, 2); + x2s("\000\134\000w\000|\000\134\000s\000\000", "0J\000\000", 0, 2); + x2s("\000\134\000w\000|\000%\000\000", "\000%0J\000\000", 0, 2); + x2s("\000\134\000w\000|\000[\000&\000$\000]\000\000", "0F\000&\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000\000", "0F\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000|\000[\000^0K\000-0S\000]\000\000", "0B\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000|\000[\000^0K\000-0S\000]\000\000", "0K\000\000", 0, 2); + x2s("\000[\000^0B\000]\000\000", "\000\012\000\000", 0, 2); + x2s("\000(\000?\000:0B\000|\000[0F\000-0M\000]\000)\000|0D0\222\000\000", "0F0\222\000\000", 0, 2); + x2s("\000(\000?\000:0B\000|\000[0F\000-0M\000]\000)\000|0D0\222\000\000", "0D0\222\000\000", 0, 4); + x2s("0B0D0F\000|\000(\000?\000=0Q0Q\000)\000.\000.0{\000\000", "0Q0Q0{\000\000", 0, 6); + x2s("0B0D0F\000|\000(\000?\000!0Q0Q\000)\000.\000.0{\000\000", "0B0D0{\000\000", 0, 6); + x2s("\000(\000?\000=0\2220B\000)\000.\000.0B\000|\000(\000?\000=0\2220\222\000)\000.\000.0B\000\000", "0\2220\2220B\000\000", 0, 6); + x2s("\000(\000?\000<\000=0B\000|0D0F\000)0D\000\000", "0D0F0D\000\000", 4, 6); + ns("\000(\000?\000>0B\000|0B0D0H\000)0F\000\000", "0B0D0H0F\000\000"); + x2s("\000(\000?\000>0B0D0H\000|0B\000)0F\000\000", "0B0D0H0F\000\000", 0, 8); + x2s("0B\000?\000|0D\000\000", "0B\000\000", 0, 2); + x2s("0B\000?\000|0D\000\000", "0D\000\000", 0, 0); + x2s("0B\000?\000|0D\000\000", "\000\000", 0, 0); + x2s("0B\000*\000|0D\000\000", "0B0B\000\000", 0, 4); + x2s("0B\000*\000|0D\000*\000\000", "0D0B\000\000", 0, 0); + x2s("0B\000*\000|0D\000*\000\000", "0B0D\000\000", 0, 2); + x2s("\000[\000a0B\000]\000*\000|0D\000*\000\000", "\000a0B0D0D0D\000\000", 0, 4); + x2s("0B\000+\000|0D\000*\000\000", "\000\000", 0, 0); + x2s("0B\000+\000|0D\000*\000\000", "0D0D0D\000\000", 0, 6); + x2s("0B\000+\000|0D\000*\000\000", "0B0D0D0D\000\000", 0, 2); + x2s("0B\000+\000|0D\000*\000\000", "\000a0B0D0D0D\000\000", 0, 0); + ns("0B\000+\000|0D\000+\000\000", "\000\000"); + x2s("\000(0B\000|0D\000)\000?\000\000", "0D\000\000", 0, 2); + x2s("\000(0B\000|0D\000)\000*\000\000", "0D0B\000\000", 0, 4); + x2s("\000(0B\000|0D\000)\000+\000\000", "0D0B0D\000\000", 0, 6); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0F0B0B0D0F0H\000\000", 0, 8); + x2s("\000(0B0D\000|0F0H\000)\000+\000\000", "0F0B0B0D0F0H\000\000", 4, 12); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0B0B0D0F0B\000\000", 2, 10); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0B0D0\2220F0B\000\000", 0, 4); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "\000$\000$\000z\000z\000z\000z0B0D0\2220F0B\000\000", 12, 16); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0B0D0B0D0B\000\000", 0, 10); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0D0B\000\000", 2, 4); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0D0B0B0B0D0B\000\000", 2, 8); + x2s("\000(\000?\000:0B\000|0D\000)\000(\000?\000:0B\000|0D\000)\000\000", "0B0D\000\000", 0, 4); + x2s("\000(\000?\000:0B\000*\000|0D\000*\000)\000(\000?\000:0B\000*\000|0D\000*\000)\000\000", "0B0B0B0D0D0D\000\000", 0, 6); + x2s("\000(\000?\000:0B\000*\000|0D\000*\000)\000(\000?\000:0B\000+\000|0D\000+\000)\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|0D\000+\000)\000{\0002\000}\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|0D\000+\000)\000{\0001\000,\0002\000}\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|\000\134\000A0D\000*\000)0F0F\000\000", "0F0F\000\000", 0, 4); + ns("\000(\000?\000:0B\000+\000|\000\134\000A0D\000*\000)0F0F\000\000", "0B0D0F0F\000\000"); + x2s("\000(\000?\000:\000^0B\000+\000|0D\000+\000)\000*0F\000\000", "0B0B0D0D0D0B0D0F\000\000", 12, 16); + x2s("\000(\000?\000:\000^0B\000+\000|0D\000+\000)\000*0F\000\000", "0B0B0D0D0D0D0F\000\000", 0, 14); + x2s("0F\000{\0000\000,\000}\000\000", "0F0F0F0F\000\000", 0, 8); + x2s("0B\000|\000(\000?\000i\000)\000c\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|0B\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000:0B\000)\000|\000a\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000i\000:0B\000)\000|\000a\000\000", "\000A\000\000"); + x2s("\000[0B0D0F\000]\000?\000\000", "0B0D0F\000\000", 0, 2); + x2s("\000[0B0D0F\000]\000*\000\000", "0B0D0F\000\000", 0, 6); + x2s("\000[\000^0B0D0F\000]\000*\000\000", "0B0D0F\000\000", 0, 0); + ns("\000[\000^0B0D0F\000]\000+\000\000", "0B0D0F\000\000"); + x2s("0B\000?\000?\000\000", "0B0B0B\000\000", 0, 0); + x2s("0D0B\000?\000?0D\000\000", "0D0B0D\000\000", 0, 6); + x2s("0B\000*\000?\000\000", "0B0B0B\000\000", 0, 0); + x2s("0D0B\000*\000?\000\000", "0D0B0B\000\000", 0, 2); + x2s("0D0B\000*\000?0D\000\000", "0D0B0B0D\000\000", 0, 8); + x2s("0B\000+\000?\000\000", "0B0B0B\000\000", 0, 2); + x2s("0D0B\000+\000?\000\000", "0D0B0B\000\000", 0, 4); + x2s("0D0B\000+\000?0D\000\000", "0D0B0B0D\000\000", 0, 8); + x2s("\000(\000?\000:Y)\000?\000)\000?\000?\000\000", "Y)\000\000", 0, 0); + x2s("\000(\000?\000:Y)\000?\000?\000)\000?\000\000", "Y)\000\000", 0, 0); + x2s("\000(\000?\000:Y\042\000?\000)\000+\000?\000\000", "Y\042Y\042Y\042\000\000", 0, 2); + x2s("\000(\000?\000:\230\250\000+\000)\000?\000?\000\000", "\230\250\230\250\230\250\000\000", 0, 0); + x2s("\000(\000?\000:\226\352\000+\000)\000?\000?\227\034\000\000", "\226\352\226\352\226\352\227\034\000\000", 0, 8); + x2s("\000(\000?\000:0B0D\000)\000?\000{\0002\000}\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\233<\216\312\000)\000?\000{\0002\000}\000\000", "\233<\216\312\233<\216\312\233<\000\000", 0, 8); + x2s("\000(\000?\000:\233<\216\312\000)\000*\000{\0000\000}\000\000", "\233<\216\312\233<\216\312\233<\000\000", 0, 0); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0003\000,\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 16); + ns("\000(\000?\000:\233<\216\312\000)\000{\0003\000,\000}\000\000", "\233<\216\312\233<\216\312\000\000"); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 12); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 16); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000?\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 8); + x2s("\000(\000?\000:\233<\216\312\000)\000{\000,\000}\000\000", "\233<\216\312\000{\000,\000}\000\000", 0, 10); + x2s("\000(\000?\000:0K0M0O\000)\000+\000?\000{\0002\000}\000\000", "0K0M0O0K0M0O0K0M0O\000\000", 0, 12); + x3s("\000(pk\000)\000\000", "pk\000\000", 0, 2, 1); + x3s("\000(pkl4\000)\000\000", "pkl4\000\000", 0, 4, 1); + x2s("\000(\000(fB\225\223\000)\000)\000\000", "fB\225\223\000\000", 0, 4); + x3s("\000(\000(\230\250l4\000)\000)\000\000", "\230\250l4\000\000", 0, 4, 1); + x3s("\000(\000(f(e\345\000)\000)\000\000", "f(e\345\000\000", 0, 4, 2); + x3s("\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\221\317[P\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000\000", "\221\317[P\000\000", 0, 4, 20); + x3s("\000(0B0D\000)\000(0F0H\000)\000\000", "0B0D0F0H\000\000", 0, 4, 1); + x3s("\000(0B0D\000)\000(0F0H\000)\000\000", "0B0D0F0H\000\000", 4, 8, 2); + x3s("\000(\000)\000(0B\000)0D0F\000(0H0J0K\000)0M0O0Q0S\000\000", "0B0D0F0H0J0K0M0O0Q0S\000\000", 6, 12, 3); + x3s("\000(\000(\000)\000(0B\000)0D0F\000(0H0J0K\000)0M0O0Q0S\000)\000\000", "0B0D0F0H0J0K0M0O0Q0S\000\000", 6, 12, 4); + x3s("\000.\000*\000(0\3250\251\000)0\3630\3730\336\000(0\363\000(\000)0\2670\3450\277\000)0\2440\363\000\000", "0\3250\2510\3630\3730\3360\3630\2670\3450\2770\2440\363\000\000", 10, 18, 2); + x2s("\000(\000^0B\000)\000\000", "0B\000\000", 0, 2); + x3s("\000(0B\000)\000|\000(0B\000)\000\000", "0D0B\000\000", 2, 4, 1); + x3s("\000(\000^0B\000)\000|\000(0B\000)\000\000", "0D0B\000\000", 2, 4, 2); + x3s("\000(0B\000?\000)\000\000", "0B0B0B\000\000", 0, 2, 1); + x3s("\000(0~\000*\000)\000\000", "0~0~0~\000\000", 0, 6, 1); + x3s("\000(0h\000*\000)\000\000", "\000\000", 0, 0, 1); + x3s("\000(0\213\000+\000)\000\000", "0\2130\2130\2130\2130\2130\2130\213\000\000", 0, 14, 1); + x3s("\000(0u\000+\000|0x\000*\000)\000\000", "0u0u0u0x0x\000\000", 0, 6, 1); + x3s("\000(0B\000+\000|0D\000?\000)\000\000", "0D0D0D0B0B\000\000", 0, 2, 1); + x3s("\000(0B0D0F\000)\000?\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0B0D0F\000)\000*\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0B0D0F\000)\000+\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0U0W0Y\000|0B0D0F\000)\000+\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(\000[0j0k0l\000]\000[0K0M0O\000]\000|0K0M0O\000)\000+\000\000", "0K0M0O\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000i\000:0B0D0F\000)\000)\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000m\000:0B\000.0F\000)\000)\000\000", "0B\000\0120F\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000=0B0\223\000)0B\000)\000\000", "0B0\2230D\000\000", 0, 2, 1); + x3s("0B0D0F\000|\000(\000.0B0D0H\000)\000\000", "0\2230B0D0H\000\000", 0, 8, 1); + x3s("0B\000*\000(\000.\000)\000\000", "0B0B0B0B0\223\000\000", 8, 10, 1); + x3s("0B\000*\000?\000(\000.\000)\000\000", "0B0B0B0B0\223\000\000", 0, 2, 1); + x3s("0B\000*\000?\000(0\223\000)\000\000", "0B0B0B0B0\223\000\000", 8, 10, 1); + x3s("\000[0D0F0H\000]0B\000*\000(\000.\000)\000\000", "0H0B0B0B0B0\223\000\000", 10, 12, 1); + x3s("\000(\000\134\000A0D0D\000)0F0F\000\000", "0D0D0F0F\000\000", 0, 4, 1); + ns("\000(\000\134\000A0D0D\000)0F0F\000\000", "0\2230D0D0F0F\000\000"); + x3s("\000(\000^0D0D\000)0F0F\000\000", "0D0D0F0F\000\000", 0, 4, 1); + ns("\000(\000^0D0D\000)0F0F\000\000", "0\2230D0D0F0F\000\000"); + x3s("0\2150\215\000(0\2130\213\000$\000)\000\000", "0\2150\2150\2130\213\000\000", 4, 8, 1); + ns("0\2150\215\000(0\2130\213\000$\000)\000\000", "0\2150\2150\2130\2130\213\000\000"); + x2s("\000(q!\000)\000\134\0001\000\000", "q!q!\000\000", 0, 4); + ns("\000(q!\000)\000\134\0001\000\000", "q!kf\000\000"); + x2s("\000(zz\000?\000)\000\134\0001\000\000", "zzzz\000\000", 0, 4); + x2s("\000(zz\000?\000?\000)\000\134\0001\000\000", "zzzz\000\000", 0, 0); + x2s("\000(zz\000*\000)\000\134\0001\000\000", "zzzzzzzzzz\000\000", 0, 8); + x3s("\000(zz\000*\000)\000\134\0001\000\000", "zzzzzzzzzz\000\000", 0, 4, 1); + x2s("0B\000(0D\000*\000)\000\134\0001\000\000", "0B0D0D0D0D\000\000", 0, 10); + x2s("0B\000(0D\000*\000)\000\134\0001\000\000", "0B0D\000\000", 0, 2); + x2s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0001\000\134\0002\000\000", "0B0B0B0D0D0B0B0B0D0D\000\000", 0, 20); + x2s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0002\000\000", "0B0B0B0D0D0D0D\000\000", 0, 14); + x3s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0002\000\000", "0B0B0B0D0D0D0D\000\000", 6, 10, 2); + x2s("\000(\000(\000(\000(\000(\000(\000(0}\000*\000)0z\000)\000)\000)\000)\000)\000)0t\000\134\0007\000\000", "0}0}0}0z0t0}0}0}\000\000", 0, 16); + x3s("\000(\000(\000(\000(\000(\000(\000(0}\000*\000)0z\000)\000)\000)\000)\000)\000)0t\000\134\0007\000\000", "0}0}0}0z0t0}0}0}\000\000", 0, 6, 7); + x2s("\000(0o\000)\000(0r\000)\000(0u\000)\000\134\0002\000\134\0001\000\134\0003\000\000", "0o0r0u0r0o0u\000\000", 0, 12); + x2s("\000(\000[0M\000-0Q\000]\000)\000\134\0001\000\000", "0O0O\000\000", 0, 4); + x2s("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "0B\0005\000 0B\0005\000 \000\000", 0, 12); + ns("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "0B\0005\000 0B\0005\000\000"); + x2s("\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "\212\260\377\037\212\260\377\037\000\000", 0, 8); + x2s("\000.\000.\000.\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "0B\000a0B\212\260\377\037\212\260\377\037\000\000", 0, 14); + x2s("\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "0F0D0F0F0D0F\000\000", 0, 12); + x2s("\000(\000^0S\000)\000\134\0001\000\000", "0S0S\000\000", 0, 4); + ns("\000(\000^0\200\000)\000\134\0001\000\000", "0\2010\2000\200\000\000"); + ns("\000(0B\000$\000)\000\134\0001\000\000", "0B0B\000\000"); + ns("\000(0B0D\000\134\000Z\000)\000\134\0001\000\000", "0B0D\000\000"); + x2s("\000(0B\000*\000\134\000Z\000)\000\134\0001\000\000", "0B\000\000", 2, 2); + x2s("\000.\000(0B\000*\000\134\000Z\000)\000\134\0001\000\000", "0D0B\000\000", 2, 4); + x3s("\000(\000.\000(0\2040D0\206\000)\000\134\0002\000)\000\000", "\000z0\2040D0\2060\2040D0\206\000\000", 0, 14, 1); + x3s("\000(\000.\000(\000.\000.\000\134\000d\000.\000)\000\134\0002\000)\000\000", "0B\0001\0002\0003\0004\0001\0002\0003\0004\000\000", 0, 18, 1); + x2s("\000(\000(\000?\000i\000:0B\000v0Z\000)\000)\000\134\0001\000\000", "0B\000v0Z0B\000v0Z\000\000", 0, 12); + x2s("\000(\000?\000Y\011\000|\000\134\000(\000\134\000g\000\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(Y\011\000)\000)\000)\000)\000)\000)\000\000", 0, 26); + x2s("\000\134\000A\000(\000?\000:\000\134\000g\000<\226?\000_\0001\000>\000|\000\134\000g\000\000|\000\134\000z}BN\206\000 \000 \000(\000?\000<\226?\000_\0001\000>\211\263\000|\201\352\000\134\000g\000\201\352\000)\000(\000?\000W(\000|\203\351\205\251\000\134\000g\000<\226?\000_\0001\000>\203\351\205\251\000)\000)\000$\000\000", "\203\351\205\251\201\352\203\351\205\251\201\352W(\201\352\203\351\205\251\201\352\203\351\205\251\000\000", 0, 26); + x2s("\000[\000[0r0u\000]\000]\000\000", "0u\000\000", 0, 2); + x2s("\000[\000[0D0J0F\000]0K\000]\000\000", "0K\000\000", 0, 2); + ns("\000[\000[\000^0B\000]\000]\000\000", "0B\000\000"); + ns("\000[\000^\000[0B\000]\000]\000\000", "0B\000\000"); + x2s("\000[\000^\000[\000^0B\000]\000]\000\000", "0B\000\000", 0, 2); + x2s("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0O\000\000", 0, 2); + ns("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0K\000\000"); + ns("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0Q\000\000"); + x2s("\000[0B\000-0\223\000&\000&0D\000-0\222\000&\000&0F\000-0\221\000]\000\000", "0\221\000\000", 0, 2); + ns("\000[\000^0B\000-0\223\000&\000&0D\000-0\222\000&\000&0F\000-0\221\000]\000\000", "0\221\000\000"); + x2s("\000[\000[\000^0B\000&\000&0B\000]\000&\000&0B\000-0\223\000]\000\000", "0D\000\000", 0, 2); + ns("\000[\000[\000^0B\000&\000&0B\000]\000&\000&0B\000-0\223\000]\000\000", "0B\000\000"); + x2s("\000[\000[\000^0B\000-0\223\000&\000&0D0F0H0J\000]\000&\000&\000[\000^0F\000-0K\000]\000]\000\000", "0M\000\000", 0, 2); + ns("\000[\000[\000^0B\000-0\223\000&\000&0D0F0H0J\000]\000&\000&\000[\000^0F\000-0K\000]\000]\000\000", "0D\000\000"); + x2s("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0F\000\000", 0, 2); + x2s("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0H\000\000", 0, 2); + ns("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0K\000\000"); + x2s("\000[0B\000-\000&\000&\000-0B\000]\000\000", "\000-\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000q\000-\000w\000]\000\000", "0H\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\000f\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\000g\000\000", 0, 2); + ns("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\0002\000\000"); + x2s("\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000\134\000/\000b\000>\000\000", "\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000/\000b\000>\000\000", 0, 40); + x2s("\000.\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000\134\000/\000b\000>\000\000", "\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000/\000b\000>\000\000", 0, 40); + + // Unicode case folding tests + // common case folding: u+0041 and u+0061 + x2s("\u0000\u0041\000\000", "\u0000\u0061\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0000\u0061\000\000", "\u0000\u0041\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+00C0 and u+00E0 + x2s("\u0000\u00C0\000\000", "\u0000\u00E0\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0000\u00E0\000\000", "\u0000\u00C0\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+00B5 and u+03BC + x2s("\u0000\u00B5\000\000", "\u0003\u00BC\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0003\u00BC\000\000", "\u0000\u00B5\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+0073 and u+017F + x2s("\u0000\u0073\000\000", "\u0001\u007F\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0001\u007F\000\000", "\u0000\u0073\000\000", 0, 2, Option.IGNORECASE); + // full case folding: u+1FA0 and u+1F60 u+03B9 + x2s("\u001F\u00A0\000\000", "\u001F\u0060\u0003\u00B9\000\000", 0, 4, Option.IGNORECASE); + x2s("\u001F\u0060\u0003\u00B9\000\000", "\u001F\u00A0\000\000", 0, 2, Option.IGNORECASE); + // full case folding: u+1FA8 and u+1F60 u+03B9 + x2s("\u001F\u00A8\000\000", "\u001F\u0060\u0003\u00B9\000\000", 0, 4, Option.IGNORECASE); + x2s("\u001F\u0060\u0003\u00B9\000\000", "\u001F\u00A8\000\000", 0, 2, Option.IGNORECASE); + // simple case folding: u+1FA8 and u+1FA0 + x2s("\u001F\u00A8\000\000", "\u001F\u00A0\000\000", 0, 2, Option.IGNORECASE); + x2s("\u001F\u00A0\000\000", "\u001F\u00A8\000\000", 0, 2, Option.IGNORECASE); + // FIXME: Case folding for 'LATIN CAPITAL LETTER SHARP S' not supported + // full case folding: u+1E9E and u+0073 u+0073 + x2s("\u001E\u009E\000\000", "\u0000\u0073\u0000\u0073\000\000", 0, 4, Option.IGNORECASE); + // simple case folding: u+1E9E and u+00DF + x2s("\u001E\u009E\000\000", "\u0000\u00DF\000\000", 0, 2, Option.IGNORECASE); + + // Case fold exceeding Analyser#THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION (= 8) + x2s("\u0000\u0041\u0000\u0041\u0000\u0041\u0000\u0041\000\000", "\u0000\u0061\u0000\u0061\u0000\u0061\u0000\u0061\000\000", 0, 8, Option.IGNORECASE); + + x2s("\000[\000\134\000x\000{\0000\000}\000-\000X\000]\000\000", "\0000\000\000", 0, 2, Option.IGNORECASE); + } +} diff --git a/third_party/joni/test/org/joni/test/TestU8.java b/third_party/joni/test/org/joni/test/TestU8.java new file mode 100755 index 000000000..e243a4b49 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestU8.java @@ -0,0 +1,359 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestU8 extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + xx("^\\d\\d\\d-".getBytes(), new byte []{-30, -126, -84, 48, 45}, 0, 0, 0, true); + x2s("x{2}", "xx", 0, 2, Option.IGNORECASE); + x2s("x{2}", "XX", 0, 2, Option.IGNORECASE); + x2s("x{3}", "XxX", 0, 3, Option.IGNORECASE); + ns("x{2}", "x", Option.IGNORECASE); + ns("x{2}", "X", Option.IGNORECASE); + + byte[] pat = new byte[] {(byte)227, (byte)131, (byte)160, (byte)40, (byte)46, (byte)41}; + byte[] str = new byte[]{(byte)227, (byte)130, (byte)185, (byte)227, (byte)131, (byte)145, (byte)227, (byte)131, (byte)160, (byte)227, (byte)131, (byte)143, (byte)227, (byte)131, (byte)179, (byte)227, (byte)130, (byte)175}; + + x2(pat, str, 6, 12); + + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35, Option.IGNORECASE); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 0, 35, Option.IGNORECASE); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAA", 0, 35, Option.IGNORECASE); + + pat = new byte[]{94, 40, (byte)239, (byte)188, (byte)161, 41, 92, 49, 36}; + str = new byte[]{(byte)239, (byte)188, (byte)161, 65}; + + n(pat, str, Option.IGNORECASE); + + pat = new byte[]{94, (byte)195, (byte)159, 123, 50, 125, 36}; + str = new byte[]{(byte)195, (byte)159, 115, 115}; + + x2(pat, str, 0, 4, Option.IGNORECASE); + + String str2 = new String(new byte[]{-61, -123, -61, -123}); + String pat2 = new String(new byte[]{'^', -61, -123, '{', '2', '}', '$'}); + + // x2s(pat2, str2, 4, 4); + // x2s(pat2, str2, 4, 4, Option.IGNORECASE); + + ns("(?i-mx:ak)a", "ema"); + + x2s("(?i:!\\[CDAT)", "![CDAT", 0, 6); + x2s("(?i:\\!\\[CDAa)", "\\![CDAa", 1, 7); + x2s("(?i:\\!\\[CDAb)", "\\![CDAb", 1, 7); + + x2s("\\R", "\u0085", 0, 2); + x2s("\\R", "\u2028", 0, 3); + x2s("\\R", "\u2029", 0, 3); + + x2s("\\A\\R\\z", "\r", 0, 1); + x2s("\\A\\R\\z", "\n", 0, 1); + x2s("\\A\\R\\z", "\r\n", 0, 2); + + x2s("foo\\b", "foo", 0, 3); + + x2s("(x?)x*\\1", "x", 0, 1, Option.IGNORECASE); + x2s("(x?)x*\\k<1+0>", "x", 0, 1, Option.IGNORECASE); + x2s("(?x?)(?x?)\\k", "x", 0, 1, Option.IGNORECASE); + + x2s("(?=((?)(\\k)))", "", 0, 0); + + x2s("a\\g<0>*z", "aaazzz", 0, 6); + + x2s("ab\\Kcd", "abcd", 2, 4); + x2s("ab\\Kc(\\Kd|z)", "abcd", 3, 4); + x2s("ab\\Kc(\\Kz|d)", "abcd", 2, 4); + x2s("(a\\K)*", "aaab", 3, 3); + x3s("(a\\K)*", "aaab", 2, 3, 1); + // x2s("a\\K?a", "aa", 0, 2); // error: differ from perl + x2s("ab(?=c\\Kd)", "abcd", 2, 2); // This behaviour is currently not well defined. (see: perlre) + x2s("(?<=a\\Kb|aa)cd", "abcd", 1, 4); // ... + x2s("(?<=ab|a\\Ka)cd", "abcd", 2, 4); // ... + + x2s("\\X", "\n", 0, 1); + x2s("\\X", "\r", 0, 1); + x2s("\\X{3}", "\r\r\n\n", 0, 4); + x2s("\\X", "\u306F\u309A\n", 0, 6); + x2s("\\A\\X\\z", "\u0020\u200d", 0, 4); + x2s("\\A\\X\\z", "\u0600\u0600", 0, 4); + x2s("\\A\\X\\z", "\u0600\u0020", 0, 3); + + x2s("\\A\\X\\z", " ‍", 0, 4); + x2s("\\A\\X\\z", "؀؀", 0, 4); + x2s("\\A\\X\\z", "؀", 0, 2); + x2s("\\A\\X\\z", "☝🏻", 0, 7); + x2s("\\A\\X\\z", "😀", 0, 4); + x2s("\\A\\X\\z", " ̈", 0, 3); // u{1f600} + + // u{20 200d} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)32, (byte)226, (byte)128, (byte)141}, 0, 4); + // u{600 600} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)216, (byte)128, (byte)216, (byte)128}, 0, 4); + // u{600 20} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)216, (byte)128, (byte)32}, 0, 3); + // u{261d 1F3FB} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)226, (byte)152, (byte)157, (byte)240, (byte)159, (byte)143, (byte)187}, 0, 7); + // u{1f600} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)152, (byte)128}, 0, 4); + // u{20 308} + x2s("\\A\\X\\z", " \u0308", 0, 3); + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)32, (byte)204, (byte)136}, 0, 3); + // u{a 308} + x2s("\\A\\X\\z", "a\u0308", 0, 3); + x2("\\A\\X\\X\\z".getBytes(), new byte[] {(byte)10, (byte)204, (byte)136}, 0, 3); + // u{d 308} + x2s("\\A\\X\\z", "d\u0308", 0, 3); + x2("\\A\\X\\X\\z".getBytes(), new byte[] {(byte)13, (byte)204, (byte)136}, 0, 3); + // u{1F477 1F3FF 200D 2640 FE0F} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)183, (byte)240, (byte)159, (byte)143, (byte)191, (byte)226, (byte)128, (byte)141, (byte)226, (byte)153, (byte)128, (byte)239, (byte)184, (byte)143}, 0, 17); + // u{1F468 200D 1F393} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)168, (byte)226, (byte)128, (byte)141, (byte)240, (byte)159, (byte)142, (byte)147}, 0, 11); + // u{1F46F 200D 2642 FE0F} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)175, (byte)226, (byte)128, (byte)141, (byte)226, (byte)153, (byte)130, (byte)239, (byte)184, (byte)143}, 0, 13); + // u{1f469 200d 2764 fe0f 200d 1f469} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)169, (byte)226, (byte)128, (byte)141, (byte)226, (byte)157, (byte)164, (byte)239, (byte)184, (byte)143, (byte)226, (byte)128, (byte)141, (byte)240, (byte)159, (byte)145, (byte)169}, 0, 20); + + x2s("\\A\\X\\X\\z", "\r\u0308", 0, 3); + x2s("\\A\\X\\X\\z", "\n\u0308", 0, 3); + + x2s("[0-9-a]+", " 0123456789-a ", 1, 13); + x2s("[0-9-\\s]+", " 0123456789-a ", 0, 12); + x2s("[0-9-あ\\\\/\u0001]+", " 0123456789-あ\\/\u0001 ", 1, 18); + x2s("[a-b-]+", "ab-", 0, 3); + x2s("[a-b-&&-]+", "ab-", 2, 3); + x2s("(?i)[a[b-あ]]+", "abあ", 0, 5); + x2s("(?i)[\\d[:^graph:]]+", "0あ", 0, 1); + x2s("(?ia)[\\d[:^print:]]+", "0あ", 0, 4); + + x2s("(?i:a) B", "a B", 0, 3); + x2s("(?i:a )B", "a B", 0, 3); + x2s("B (?i:a)", "B a", 0, 3); + x2s("B(?i: a)", "B a", 0, 3); + + x2s("(?a)[\\p{Space}\\d]", "\u00a0", 0, 2); + x2s("(?a)[\\d\\p{Space}]", "\u00a0", 0, 2); + ns("(?a)[^\\p{Space}\\d]", "\u00a0"); + ns("(?a)[^\\d\\p{Space}]", "\u00a0"); + x2s("(?d)[[:space:]\\d]", "\u00a0", 0, 2); + ns("(?d)[^\\d[:space:]]", "\u00a0"); + + x2s("\\p{In_Unified_Canadian_Aboriginal_Syllabics_Extended}+", "\u18B0\u18FF", 0, 6); + x2s("(?i)\u1ffc", "\u2126\u1fbe", 0, 6); + x2s("(?i)\u1ffc", "\u1ff3", 0, 3); + x2s("(?i)\u0390", "\u03b9\u0308\u0301", 0, 6); + x2s("(?i)\u03b9\u0308\u0301", "\u0390", 0, 2); + x2s("(?i)ff", "\ufb00", 0, 3); + x2s("(?i)\ufb01", "fi", 0, 2); + + x2s("(?i)\u0149\u0149", "\u0149\u0149", 0, 4); + x2s("(?i)(?<=\u0149)a", "\u02bcna", 3, 4); + + x2s("(?m:.*abc)", "dddabdd\nddabc", 0, 13); + x2s("(?m:.+abc)", "dddabdd\nddabc", 0, 13); + x2s("(?-m:.*abc)", "dddabdd\nddabc", 8, 13); + ns("(?-m:.*ab[x-z])", "dddabdd\nddabc"); + x2s("(?-m:.*(?:abc|\\Gabc))", "dddabdd\nddabc", 8, 13); + x2s("(?-m:.+abc)", "dddabdd\nddabc", 8, 13); + x2s("(?-m:.*abc)", "dddabdd\nabc", 8, 11); + ns("(?-m:.+abc)", "dddabdd\nabc"); + x2s("(?m:.*\\Z)", "dddabdd\nddabc", 0, 13); + x2s("(?-m:.*\\Z)", "dddabdd\nddabc", 8, 13); + x2s("(.*)X\\1", "1234X2345", 1, 8); + + x2s("(?<=(?i)ab)cd", "ABcd", 2, 4); + x2s("(?<=(?i:ab))cd", "ABcd", 2, 4); + ns("(?<=(?i)ab)cd", "ABCD"); + ns("(?<=(?i:ab))cd", "ABCD"); + x2s("(?)->", "<- ->->", 0, 5); + x2s("<-(?~->)->\n", "<-1->2<-3->\n", 6, 12); + x2s("<-(?~->)->.*<-(?~->)->", "<-1->2<-3->4<-5->", 0, 17); + x2s("<-(?~->)->.*?<-(?~->)->", "<-1->2<-3->4<-5->", 0, 11); + x2s("(?~abc)c", "abc", 0, 3); + x2s("(?~abc)bc", "abc", 0, 3); + x2s("(?~abc)abc", "abc", 0, 3); + + // ns("(?~)", " "); + ns("(?~)", ""); + ns(" (?~)", " "); + ns(" (?~)", " "); + // x2s("(?~(?~))", "abc", 0, 3); + x2s("(?~a)", "", 0, 0); + x2s("(?~a)a", "a", 0, 1); + x2s("(?~a)", "x", 0, 1); + x2s("(?~a)a", "xa", 0, 2); + x2s("(?~.)", "", 0, 0); + x2s("(?~.)a", "a", 0, 1); + x2s("(?~.)", "x", 0, 0); + x2s("(?~.)a", "xa", 1, 2); + x2s("(?~abc)", "abc", 0, 2); + x2s("(?~b)", "abc", 0, 1); + x2s("(?~abc|b)", "abc", 0, 1); + // ns("(?~|abc)", "abc"); // ? + x2s("(?~abc|)", "abc", 0, 1); // ? + x2s("(?~abc|def)x", "abcx", 1, 4); + x2s("(?~abc|def)x", "defx", 1, 4); + x2s("^(?~\\S+)TEST", "TEST", 0, 4); + x3s("(?~(a)c)", "aab", -1, -1, 1); // # $1 should not match. + + x2s("𠜎𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎?𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎*𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎{3}", "𠜎𠜎𠜎", 0, 12); + + x2s("[^a\\x{80}]", "x", 0, 1); + ns("[^a\\x{80}]", "a"); + ns("[a\\x{80}]", "x", Option.CR_7_BIT); + x2s("[a\\x{80}]", "a", 0, 1, Option.CR_7_BIT); + x2s("[^a\\x{80}]", "x", 0, 1, Option.CR_7_BIT); + ns("[^a\\x{80}]", "a", Option.CR_7_BIT); + + ns("(\\2)(\\1)", ""); + + x2s("(?<=fo).*", "foo", 2, 3); + x2s("(?m)(?<=fo).*", "foo", 2, 3); + x2s("(?m)(?<=fo).+", "foo", 2, 3); + + x3s("\\(((?:[^(]|\\g<0>)*)\\)", "(abc)(abc)", 1, 4, 1); + x3s("\\(((?:[^(]|\\g<0>)*)\\)", "((abc)(abc))", 1, 11, 1); + x3s("\\(((?:[^(]|(\\g<0>))*)\\)", "((abc)(abc))", 6, 11, 2); + + x2s("^.+$", "a\n", 0, 1); + x2s("^.+$", "\na\n", 1, 2); + ns("^.+$", "\n"); + + ns("💌", "aa"); + ns("aa", "💌"); + + x2s("\\P{In_Supplemental_Symbols_and_Pictographs}?", "", 0, 0); + x2s("\\P{In_Transport_and_Map_Symbols}?", "", 0, 0); + + x2s("^(\"|)(.*)\\1$", "X6", 0, 2); + } +} diff --git a/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java b/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java new file mode 100644 index 000000000..69796af5b --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java @@ -0,0 +1,56 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestUtf8CaseFoldingLatin1Supplement extends Test { + + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "utf-8"; + } + + @Override + public Syntax syntax() { + return Syntax.Java; + } + + @Override + public void test() throws InterruptedException { + // test ignorecase for Latin-1 Supplement + x2s("[\\u00e0-\\u00e5]", "\u00c2", 0, 2, Option.IGNORECASE); + x2s("[\\u00e2]", "\u00c2", 0, 2, Option.IGNORECASE); + x2s("\\u00e2", "\u00c2", 0, 2, Option.IGNORECASE); + } +} diff --git a/third_party/licenses/jcodings-LICENSE.txt b/third_party/licenses/jcodings-LICENSE.txt new file mode 100644 index 000000000..ed2a963f7 --- /dev/null +++ b/third_party/licenses/jcodings-LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2025 JRuby Team + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + From 839ab72863c033ba88ac381ee3ef75edb23dc773 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 19:58:01 +0200 Subject: [PATCH 2/2] feat(regex): execute embedded callbacks in Joni Implement phases 36.2 through 36.4 of the vendored Joni callout design. Phase 36.2 represents literal (?{...}) bodies and callback conditions as parser-created lexical closures. Structured regex templates preserve those closures and callback tables across interpolation, qr embedding, JVM codegen, and interpreter bytecode without exposing internal callout syntax to runtime strings. Phase 36.3 makes every structured executable callback select Joni, including constant-bodied callbacks. The runtime bridge publishes provisional captures, executes closures in scalar context, maintains $^R, checkpoints regex and dynamic-local state, and restores state exactly once on backtracking, failure, exceptions, or successful completion. Phase 36.4 adds callback-selected conditional branches and callback-table ID remapping for embedded regex values. Preserve literal CALL-like pattern text, retain the compatibility no-op for eval groups introduced only as runtime text, and keep dynamic (??{...}) patterns gated until nested alternatives can participate in outer backtracking. Stabilize literal match-target scalar identity for /g call sites and fix undefining a typeglob scalar slot that aliases a read-only callback value while preserving immediate DESTROY behavior for mutable referents. Track runtime regex interpolation explicitly so compile-time transforms such as \Q...\E keep eager qr validation. Preserve a lone interpolated value's runtime type for qr overloading and already-compiled callback regexes. Keep Perl source policy and Java-only rewrites in PerlOnJava rather than moving the whole RegexPreprocessor into Joni. Move only matcher semantics incrementally and use the missing-regex feature matrix as the Joni routing roadmap. Original Joni copyright, authorship, license, and modification notices remain intact. Verification: - full make passes, including Joni tests, five unit shards, packaging, notices, relocation, and SBOM checks - executable_callbacks.t passes 18/18 under system Perl, the JVM backend, and the interpreter backend - perl5_t/t/re completes 80 files with zero timeouts and 50458/94827 passing - versus test_20260815_080000_958.log: 17 files improve, 63 are unchanged, none regress, and the net gain is 185 passing assertions Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- docs/design/joni-callout-fork.md | 100 ++++++++++-- docs/reference/feature-matrix.md | 8 +- .../backend/bytecode/BytecodeCompiler.java | 21 +++ .../backend/bytecode/BytecodeInterpreter.java | 15 ++ .../backend/bytecode/CompileOperator.java | 29 +++- .../perlonjava/backend/bytecode/Opcodes.java | 6 + .../backend/jvm/EmitOperatorNode.java | 2 + .../org/perlonjava/backend/jvm/EmitRegex.java | 34 +++- .../frontend/parser/StringDoubleQuoted.java | 2 +- .../frontend/parser/StringSegmentParser.java | 140 +++++++--------- .../runtime/regex/JoniRegexPattern.java | 150 +++++++++++++++++- .../runtime/regex/RuntimeRegex.java | 61 +++++-- .../runtime/regex/RuntimeRegexCallback.java | 24 +++ .../runtime/regex/RuntimeRegexTemplate.java | 92 +++++++++++ .../runtime/runtimetypes/RuntimeGlob.java | 13 +- .../runtimetypes/RuntimeRegexState.java | 2 + .../unit/regex/executable_callbacks.t | 56 +++++++ third_party/joni/src/org/joni/Analyser.java | 2 +- .../joni/src/org/joni/ArrayCompiler.java | 9 +- .../joni/src/org/joni/ByteCodeMachine.java | 20 ++- .../joni/src/org/joni/CalloutHandler.java | 5 + third_party/joni/src/org/joni/Parser.java | 13 +- .../joni/src/org/joni/StackMachine.java | 12 ++ .../joni/src/org/joni/ast/EncloseNode.java | 2 + .../org/joni/constants/internal/OPCode.java | 3 + .../org/joni/constants/internal/OPSize.java | 1 + 26 files changed, 692 insertions(+), 130 deletions(-) create mode 100644 src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java create mode 100644 src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java create mode 100644 src/test/resources/unit/regex/executable_callbacks.t diff --git a/docs/design/joni-callout-fork.md b/docs/design/joni-callout-fork.md index a36c38425..d019672c4 100644 --- a/docs/design/joni-callout-fork.md +++ b/docs/design/joni-callout-fork.md @@ -45,6 +45,7 @@ The fork exposes a runtime-neutral API in `org.joni`: interface CalloutHandler { CalloutResult execute(int calloutId, MatchView match); void unwind(Object backtrackToken); + default void complete(Object successfulToken) { unwind(successfulToken); } } interface MatchView { @@ -70,15 +71,49 @@ runtime classes. 2. A non-null token is stored in a matcher stack frame before execution continues or fails. 3. Backtracking across that frame calls `unwind(token)` exactly once. -4. Successful completion, failed completion, interruption, timeout, and handler - exceptions unwind every still-active token in reverse execution order. -5. A `FAIL` result enters normal matcher backtracking after installing the frame, +4. Successful completion calls `complete(token)` for every still-active token in + reverse execution order. Its default implementation delegates to `unwind`, + preserving the cleanup-only contract for engine-neutral handlers. +5. Failed completion, interruption, timeout, and handler exceptions unwind every + still-active token in reverse execution order. +6. A `FAIL` result enters normal matcher backtracking after installing the frame, so cleanup follows the same path as later-pattern failure. -6. Patterns without callout nodes allocate no handler state or callout frames. - -The first engine slice supports plain callouts. Conditional callbacks and -dynamic nested patterns extend the action set only after their Perl semantics -and outer-backtracking behavior are proven independently. +7. Patterns without callout nodes allocate no handler state or callout frames. + +Callback conditions use an internal conditional-callout opcode. `CONTINUE` +selects the yes branch and `FAIL` selects the no branch without treating the +condition itself as a failed match. Dynamic nested patterns remain outside this +contract until their alternatives can participate in outer backtracking. + +## Structured Perl frontend + +Literal callback bodies are represented as lexical `SubroutineNode` closures. +The parser emits a `regexTemplate` operation whose ordered parts contain normal +interpolation values and explicit `regexCallback` wrappers. Runtime template +construction assigns callback IDs and produces the engine-facing skeleton. + +An interpolated coderef remains ordinary interpolation because only the parser +can create a callback wrapper. Runtime strings containing Perl eval groups never +become trusted callback skeletons. They retain the existing security checks and, +where compatibility mode permits unsupported eval groups, the existing +zero-width no-op behavior. + +At execution, the bridge publishes provisional numbered captures and offsets, +runs the closure in scalar context, and updates `$^R` for plain callbacks only. +Each token checkpoints regex state, `$^R`, and the dynamic-local stack. Backtrack +unwind restores all three; successful completion restores dynamic scope while +retaining the last successful plain callback result. + +Every structured executable callback template selects Joni, including a +callback whose body happens to return a constant: `(?{ 1 })` still has match-time +side effects and unwind semantics. Constant dynamic-pattern expressions +`(??{ ... })` may retain the existing compile-time fold path; a runtime-dependent +dynamic-pattern closure selects Joni only after the nested-pattern backtracking +contract is implemented. + +Literal match targets have one stable scalar identity per compiled call site so +`pos()` and `/g` survive repeated loop execution without allowing two identical +literal occurrences to share state. ## Capture view @@ -104,6 +139,40 @@ separate PerlOnJava modification notice under `META-INF/licenses`. The SBOM must identify the vendored Joni component, its upstream version, MIT license, and JCodings dependency. +## Regex preprocessing boundary + +Do not move the entire `RegexPreprocessor` into the Joni fork. The ownership +boundary is: + +- PerlOnJava owns Perl source policy and runtime integration: trusted callback + templates, `use re 'eval'` security, lexical warnings, user-defined Unicode + properties, Perl diagnostics, named/branch-reset capture mapping, and backend + selection. +- A backend-neutral Perl normalization layer may own syntax aliases shared by + all engines, such as accepted named-group spellings and modifier validation. +- Joni should own matcher semantics required by Perl, including condition + execution, backtracking-visible state, native capture behavior, case-folding + differences, and constructs that cannot be represented faithfully by textual + rewrites. +- The Java-only adapter retains Java `Pattern` syntax rewrites and stack-safety + workarounds while the Java fast path exists. + +Migration is incremental. First inventory each preprocessing rule as source +policy, backend-neutral syntax, Java adaptation, or matcher semantics. Move only +the matcher-semantic category into focused Joni changes with differential Perl +tests. A future decision to make Joni the default backend requires corpus and +performance evidence; it is not implied by executable-callback support. + +The Joni routing roadmap is the missing-regex feature list in +`docs/reference/feature-matrix.md`: dynamically scoped regex state, recursive +and dynamic patterns, backtracking verbs and definitions, variable lookbehind, +branch reset, subpattern calls, conditions, extended Unicode and graphemes, +embedded code, regex debugging, runtime eval, lexical default flags, and named +capture behavior. As each feature becomes executable, patterns containing it +select Joni. The current selector routes declarative subpattern calls and every +structured executable callback template; constant `(??{...})` expressions keep +their established fold-to-pattern path. + ## Implementation stages 1. Import and build the pinned upstream snapshot with its original tests through @@ -112,10 +181,12 @@ JCodings dependency. 3. Add the callout parser node, opcode, handler API, provisional match view, and forward execution. 4. Add matcher-owned callout frames and exact-once cleanup for every exit path. -5. Connect structured Perl callback templates without enabling unsupported - callback forms. -6. Add conditional and nested-program actions only after focused differential - tests establish their contracts. +5. Connect structured Perl callback templates and lexical closures. +6. Publish provisional match state for plain callbacks and preserve `$^R` across + successful completion and backtracking. +7. Add dynamic-local checkpoints and callback conditions. +8. Add dynamic nested patterns only after focused differential tests establish + their outer-backtracking contract. ## Verification @@ -128,6 +199,11 @@ JCodings dependency. both relocated trees, and includes all required notices. - An embedding smoke test can load stock Joni and PerlOnJava together. - Full `make` and the focused direct/thread regex matrix pass. +- The executable-callback unit test passes under standard Perl and both + PerlOnJava execution backends. +- `dev/tools/perl_test_runner.pl perl5_t/t/re/` is compared file-by-file with + `../PerlOnJava/logs/test_20260815_080000_958.log`; no previously passing regex + file may regress, and changed pass counts or blocked-test totals are reported. ## Stop conditions diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 758aa4e3f..fd4463671 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -390,15 +390,15 @@ my @copy = @{$z}; # ERROR ### Missing Regular Expression Features - ❌ **Dynamically-scoped regex variables**: Regex variables are not dynamically-scoped. -- ❌ **Recursive Patterns**: Features like `(?R)`, `(?0)` or `(??{ code })` for recursive matching are not supported. +- 🟡 **Recursive Patterns**: `(?R)` and `(?0)` execute through Joni. Constant `(??{ code })` expressions fold to a pattern; runtime-dependent dynamic patterns remain unsupported. - ❌ **Backtracking Control Verbs and Definitions**: Features such as `(*PRUNE)`, `(*SKIP)`, and `(?(DEFINE)...)` are not supported. Atomic groups `(?>...)` are supported as noted above. - ❌ **Lookbehind Assertions**: Variable-length negative or positive lookbehind assertions, e.g., `(?<=...)` or `(? cp > 255); + if (node.forceByteString || isAsciiOnly(node.value) + || (!hasWideChars && emitterContext != null && emitterContext.symbolTable != null + && !emitterContext.symbolTable.isStrictOptionEnabled(Strict.HINT_UTF8))) { + literal.type = RuntimeScalarType.BYTE_STRING; + } + // Do not value-deduplicate this entry with ordinary read-only literals. + // Its identity is the pos() storage for this one regex call site. + int constant = constants.size(); + constants.add(literal); + int register = allocateOutputRegister(); + emit(Opcodes.LOAD_CONST); + emitReg(register); + emit(constant); + lastResultReg = register; + return register; + } + @Override public void visit(IdentifierNode node) { // Variable reference diff --git a/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java b/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java index abadb91f1..021700a65 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java +++ b/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java @@ -1036,6 +1036,21 @@ private static RuntimeList execute(SuspendedInterpreterFrame frame) { registers[rd] = RuntimeCode.copyDoBlockListResult(registers[rs]); } + case Opcodes.REGEX_CALLBACK -> { + int rd = bytecode[pc++]; + int codeReg = bytecode[pc++]; + int kindIdx = bytecode[pc++]; + registers[rd] = org.perlonjava.runtime.regex.RuntimeRegexCallback.wrap( + registers[codeReg].scalar(), code.stringPool[kindIdx]); + } + + case Opcodes.REGEX_TEMPLATE -> { + int rd = bytecode[pc++]; + int partsReg = bytecode[pc++]; + registers[rd] = org.perlonjava.runtime.regex.RuntimeRegexTemplate.build( + (RuntimeList) registers[partsReg]); + } + case Opcodes.ASSIGN_LEXICAL_SCALAR -> { int rd = bytecode[pc++]; int rs = bytecode[pc++]; diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java index aae2f1019..059827ddf 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java @@ -323,8 +323,13 @@ private static void visitMatchRegex(BytecodeCompiler bc, OperatorNode node) { } int stringReg; if (args.elements.size() > 2) { - bc.compileNode(args.elements.get(2), -1, RuntimeContextType.SCALAR); - stringReg = bc.lastResultReg; + Node target = args.elements.get(2); + if (target instanceof StringNode literal) { + stringReg = bc.compileStableRegexLiteral(literal); + } else { + bc.compileNode(target, -1, RuntimeContextType.SCALAR); + stringReg = bc.lastResultReg; + } } else { stringReg = loadDefaultUnderscore(bc); } @@ -813,6 +818,26 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode case "getppid" -> { int rd = bytecodeCompiler.allocateOutputRegister(); bytecodeCompiler.emitWithToken(Opcodes.GETPPID, node.getIndex()); bytecodeCompiler.emitReg(rd); bytecodeCompiler.lastResultReg = rd; } case "open" -> visitOpen(bytecodeCompiler, node); case "matchRegex" -> visitMatchRegex(bytecodeCompiler, node); + case "regexCallback" -> { + bytecodeCompiler.compileNode(node.operand, -1, RuntimeContextType.SCALAR); + int codeReg = bytecodeCompiler.lastResultReg; + int rd = bytecodeCompiler.allocateOutputRegister(); + bytecodeCompiler.emit(Opcodes.REGEX_CALLBACK); + bytecodeCompiler.emitReg(rd); + bytecodeCompiler.emitReg(codeReg); + bytecodeCompiler.emit(bytecodeCompiler.addToStringPool( + (String) node.getAnnotation("regexCallbackKind"))); + bytecodeCompiler.lastResultReg = rd; + } + case "regexTemplate" -> { + bytecodeCompiler.compileNode(node.operand, -1, RuntimeContextType.LIST); + int partsReg = bytecodeCompiler.lastResultReg; + int rd = bytecodeCompiler.allocateOutputRegister(); + bytecodeCompiler.emit(Opcodes.REGEX_TEMPLATE); + bytecodeCompiler.emitReg(rd); + bytecodeCompiler.emitReg(partsReg); + bytecodeCompiler.lastResultReg = rd; + } case "replaceRegex" -> visitReplaceRegex(bytecodeCompiler, node); case "substr" -> visitSubstr(bytecodeCompiler, node); case "chomp" -> visitChomp(bytecodeCompiler, node); diff --git a/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java b/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java index 21a7a53f1..63a82d5ff 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java +++ b/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java @@ -2461,6 +2461,12 @@ public class Opcodes { /** Copy a do-block scalar/list result so loose lvalues do not escape. Format: rd, rs. */ public static final short COPY_DO_BLOCK_RESULT = 519; + /** Wrap a lexical closure as an executable regex callback. Format: rd codeReg kindStringIdx. */ + public static final short REGEX_CALLBACK = 520; + + /** Build an interpolated regex template while preserving callback objects. Format: rd partsReg. */ + public static final short REGEX_TEMPLATE = 521; + /** Return the mutable {@code $#array} cell. Format: ARRAY_LAST_INDEX_LVALUE rd arrayReg. */ public static final short ARRAY_LAST_INDEX_LVALUE = 518; diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java b/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java index 95658498d..9f9b3c52c 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java @@ -159,6 +159,8 @@ public static void emitOperatorNode(EmitterVisitor emitterVisitor, OperatorNode // Regular expression operations case "matchRegex" -> EmitRegex.handleMatchRegex(emitterVisitor, node); case "quoteRegex" -> EmitRegex.handleQuoteRegex(emitterVisitor, node); + case "regexCallback" -> EmitRegex.handleRegexCallback(emitterVisitor, node); + case "regexTemplate" -> EmitRegex.handleRegexTemplate(emitterVisitor, node); case "replaceRegex" -> EmitRegex.handleReplaceRegex(emitterVisitor, node); case "tr", "y" -> EmitRegex.handleTransliterate(emitterVisitor, node); case "qx" -> EmitRegex.handleSystemCommand(emitterVisitor, node); diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java index c9229435e..3cb446718 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java @@ -20,6 +20,25 @@ * transliteration and replacement. */ public class EmitRegex { + static void handleRegexCallback(EmitterVisitor emitterVisitor, OperatorNode node) { + node.operand.accept(emitterVisitor.with(RuntimeContextType.SCALAR)); + emitterVisitor.ctx.mv.visitLdcInsn((String) node.getAnnotation("regexCallbackKind")); + emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegexCallback", "wrap", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Ljava/lang/String;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + } + + static void handleRegexTemplate(EmitterVisitor emitterVisitor, OperatorNode node) { + node.operand.accept(emitterVisitor.with(RuntimeContextType.LIST)); + emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegexTemplate", "build", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeList;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + if (emitterVisitor.ctx.contextType == RuntimeContextType.VOID) { + emitterVisitor.ctx.mv.visitInsn(Opcodes.POP); + } + } // Callsite ID counter for /o modifier support (unique across all JVM compilations) private static final AtomicInteger nextCallsiteId = new AtomicInteger(100000); @@ -359,7 +378,7 @@ static void handleMatchRegex(EmitterVisitor emitterVisitor, OperatorNode node) { emitterVisitor.ctx.mv.visitVarInsn(Opcodes.ASTORE, regexSlot); // Use default variable $_ if none specified - handleVariableBinding(operand, 2, scalarVisitor); + handleVariableBinding(operand, 2, scalarVisitor, true); emitterVisitor.ctx.mv.visitVarInsn(Opcodes.ALOAD, regexSlot); emitterVisitor.ctx.mv.visitInsn(Opcodes.SWAP); @@ -421,6 +440,12 @@ private static void emitMatchRegexRuntimeCall(EmitterVisitor emitterVisitor, Str * @param scalarVisitor The visitor used to emit scalar context bytecode */ private static void handleVariableBinding(ListNode operand, int variableIndex, EmitterVisitor scalarVisitor) { + handleVariableBinding(operand, variableIndex, scalarVisitor, false); + } + + private static void handleVariableBinding(ListNode operand, int variableIndex, + EmitterVisitor scalarVisitor, + boolean stabilizeLiteral) { // Check if a variable was provided in the operand list Node variable = null; if (operand.elements.size() > variableIndex) { @@ -434,5 +459,12 @@ private static void handleVariableBinding(ListNode operand, int variableIndex, E // Generate bytecode for the variable access variable.accept(scalarVisitor); + if (stabilizeLiteral && variable instanceof StringNode) { + scalarVisitor.ctx.mv.visitLdcInsn(nextCallsiteId.getAndIncrement()); + scalarVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegex", "stabilizeLiteralTarget", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;I)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + } } } diff --git a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java index d7e031064..6ad7839e6 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java @@ -275,7 +275,7 @@ public Node parse() { applyCaseModifier(caseModifiers.pop()); } - return createJoinNode(segments); + return needsStructuredRegexTemplate() ? buildResult() : createJoinNode(segments); } /** diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index 9293208d5..81de96e26 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -8,12 +8,10 @@ import org.perlonjava.frontend.lexer.LexerToken; import org.perlonjava.frontend.lexer.LexerTokenType; import org.perlonjava.runtime.operators.PerlUtfString; -import org.perlonjava.runtime.regex.CaptureNameEncoder; import org.perlonjava.runtime.regex.RegexMarkers; import org.perlonjava.runtime.regex.UnicodeResolver; import org.perlonjava.runtime.runtimetypes.PerlCompilerException; import org.perlonjava.runtime.runtimetypes.RuntimeScalar; -import org.perlonjava.runtime.runtimetypes.RuntimeScalarCache; import org.perlonjava.runtime.runtimetypes.ScalarUtils; import java.math.BigInteger; @@ -51,7 +49,6 @@ public abstract class StringSegmentParser { * Must be static to ensure names don't collide across different patterns that share * the same pendingCodeBlockConstants map */ - private static int codeBlockCaptureCounter = 0; /** * The emitter context for logging and error handling */ @@ -85,6 +82,8 @@ public abstract class StringSegmentParser { * List of AST nodes representing string segments (literals and interpolated expressions) */ protected final List segments; + protected boolean hasExecutableRegexCallbacks; + protected boolean hasRuntimeInterpolation; protected final boolean interpolateVariable; protected final boolean parseEscapes; /** @@ -253,6 +252,7 @@ private boolean shouldForceByteStringLiteral(String value) { */ protected void parseVariableInterpolation(String sigil) { flushCurrentSegment(); + hasRuntimeInterpolation = true; if (CompilerOptions.DEBUG_ENABLED) ctx.logDebug("str sigil"); @@ -641,6 +641,10 @@ private Node parseSimpleVariableInterpolation(String sigil) { protected Node buildResult() { flushCurrentSegment(); + if (needsStructuredRegexTemplate()) { + return new OperatorNode("regexTemplate", new ListNode(segments, tokenIndex), tokenIndex); + } + return switch (segments.size()) { case 0 -> new StringNode("", tokenIndex); case 1 -> { @@ -663,6 +667,11 @@ yield new BinaryOperatorNode("join", }; } + /** Preserve regex-valued interpolation so embedded callback tables are not stringified away. */ + protected boolean needsStructuredRegexTemplate() { + return hasExecutableRegexCallbacks || (isRegex && hasRuntimeInterpolation); + } + /** * Abstract method for parsing escape sequences. * @@ -787,7 +796,10 @@ protected boolean handleSpecialToken(String text) { } case "(" -> { // Check for (?{...}) and (??{...}) regex code blocks - only in regex context - if (isRegex && regexCodeBlocksAreActive() && isRegexCodeBlock()) { + if (isRegex && regexCodeBlocksAreActive() && isRegexCallbackCondition()) { + parseRegexCallbackCondition(); + yield true; + } else if (isRegex && regexCodeBlocksAreActive() && isRegexCodeBlock()) { parseRegexCodeBlock(false); // (?{...}) - code execution yield true; } else if (isRegex && regexCodeBlocksAreActive() && isRegexRecursiveBlock()) { @@ -819,6 +831,29 @@ private boolean isRegexCodeBlock() { return false; } + private boolean isRegexCallbackCondition() { + int currentPos = parser.tokenIndex; + return currentPos + 3 < parser.tokens.size() + && "?".equals(parser.tokens.get(currentPos).text) + && "(".equals(parser.tokens.get(currentPos + 1).text) + && "?".equals(parser.tokens.get(currentPos + 2).text) + && "{".equals(parser.tokens.get(currentPos + 3).text); + } + + private void parseRegexCallbackCondition() { + flushCurrentSegment(); + int start = tokenIndex; + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "("); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "{"); + Node block = parseBlock(parser); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "}"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, ")"); + segments.add(new StringNode("(?(", start)); + segments.add(regexCallback(block, "CONDITION", start)); + } + /** * Checks if the current tokens form a (??{...}) recursive regex pattern. * This is similar to (?{...}) but uses the result as a regex pattern. @@ -883,92 +918,27 @@ private void parseRegexCodeBlock(boolean isRecursive) { // Consume the closing ")" that completes the (?{...}) construct TokenUtils.consume(parser, LexerTokenType.OPERATOR, ")"); - // Try to apply constant folding to the block - Node folded = ConstantFoldingVisitor.foldConstants(block); - - // If it's a BlockNode with a single element, extract it - // This handles both empty blocks (BlockNode with empty ListNode) and single-expression blocks - if (folded instanceof BlockNode blockNode) { - if (blockNode.elements.size() == 1) { - folded = blockNode.elements.get(0); - } - } - - // Check if the result is a simple constant using the visitor pattern - RuntimeScalar constantValue = - ConstantFoldingVisitor.getConstantValue(folded); - - if (constantValue != null) { - if (isRecursive) { - // For (??{...}), the constant becomes a pattern to match - // Extract the string value and insert it directly as a pattern - String patternString = constantValue.toString(); - // Insert the pattern string directly - it will be compiled as a regex - segments.add(new StringNode(patternString, savedTokenIndex)); - } else { - // For (?{...}), encode the value in a capture group for $^R - String captureName; - - // Check if it's undef (needs special encoding) - if (constantValue == RuntimeScalarCache.scalarUndef) { - captureName = String.format("cb%03du", codeBlockCaptureCounter++); - } else { - // Use CaptureNameEncoder to encode the value in the capture name - captureName = CaptureNameEncoder.encodeCodeBlockValue( - codeBlockCaptureCounter++, constantValue - ); - } - - if (captureName == null) { - // Encoding failed (e.g., name too long) - use fallback - segments.add(new StringNode(RegexMarkers.CODE_BLOCK, savedTokenIndex)); - } else { - // Encoding succeeded - create capture group - StringNode captureNode = new StringNode("(?<" + captureName + ">)", savedTokenIndex); - segments.add(captureNode); - } + if (isRecursive) { + // Preserve the existing constant (??{...}) support. Runtime-dependent + // nested patterns remain deferred to Stage 36.5. + Node folded = ConstantFoldingVisitor.foldConstants(block); + if (folded instanceof BlockNode blockNode && blockNode.elements.size() == 1) { + folded = blockNode.elements.getFirst(); } + RuntimeScalar constant = ConstantFoldingVisitor.getConstantValue(folded); + segments.add(new StringNode(constant == null + ? RegexMarkers.RECURSIVE_PATTERN : constant.toString(), savedTokenIndex)); } else { - if (isRecursive) { - segments.add(new StringNode(RegexMarkers.RECURSIVE_PATTERN, savedTokenIndex)); - } else if (isSimpleRegexCodeBlockSideEffect(block)) { - List sideEffectElements = new ArrayList<>(); - if (block instanceof BlockNode blockNode) { - sideEffectElements.addAll(blockNode.elements); - } else { - sideEffectElements.add(block); - } - sideEffectElements.add(new StringNode("", savedTokenIndex)); - segments.add(new BlockNode(sideEffectElements, savedTokenIndex)); - } else { - segments.add(new StringNode(RegexMarkers.CODE_BLOCK, savedTokenIndex)); - } + segments.add(regexCallback(block, "BLOCK", savedTokenIndex)); } } - private boolean isSimpleRegexCodeBlockSideEffect(Node block) { - if (!isRegexQuoteConstruction) { - return false; - } - Node node = block; - if (node instanceof BlockNode blockNode) { - if (blockNode.elements.size() != 1) { - return false; - } - node = blockNode.elements.get(0); - } - if (!(node instanceof OperatorNode operatorNode)) { - return false; - } - if (!operatorNode.operator.equals("++") - && !operatorNode.operator.equals("++postfix") - && !operatorNode.operator.equals("--") - && !operatorNode.operator.equals("--postfix")) { - return false; - } - return operatorNode.operand instanceof OperatorNode scalarOp - && scalarOp.operator.equals("$") - && scalarOp.operand instanceof IdentifierNode; + private Node regexCallback(Node block, String kind, int index) { + SubroutineNode closure = new SubroutineNode(null, null, null, block, false, index); + OperatorNode callback = new OperatorNode("regexCallback", closure, index); + callback.setAnnotation("regexCallbackKind", kind); + hasExecutableRegexCallbacks = true; + return callback; } /** diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 08047737c..5a02de43a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -2,6 +2,9 @@ import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; +import org.joni.CalloutHandler; +import org.joni.CalloutResult; +import org.joni.MatchView; import org.joni.NameEntry; import org.joni.Option; import org.joni.Regex; @@ -12,9 +15,13 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.List; + +import org.perlonjava.runtime.runtimetypes.*; /** - * Stack-based regex backend for declarative recursive subpatterns. + * Stack-based regex backend for Perl constructs that require matcher semantics + * beyond the Java Pattern fast path. */ final class JoniRegexPattern { private final Regex regex; @@ -29,8 +36,8 @@ final class JoniRegexPattern { namedGroups = collectNamedGroups(regex); } - RegexMatcher matcher(String input) { - return new JoniRegexMatcher(regex, sourcePattern, namedGroups, input); + RegexMatcher matcher(String input, List callbacks) { + return new JoniRegexMatcher(regex, sourcePattern, namedGroups, input, callbacks); } String patternDescription() { @@ -53,9 +60,11 @@ private static int toJoniOptions(RegexFlags flags) { return options; } - static boolean requiresRecursiveBackend(String pattern) { + static boolean requiresJoniBackend(String pattern) { if (pattern == null) return false; - return pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*" ) + return pattern.contains("(?{=CALL:") + || pattern.contains("(?(?{=CALL:") + || pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*" ) || pattern.contains("(?&") || pattern.contains("(?P>"); } @@ -91,6 +100,20 @@ private static String translatePattern(String pattern, RegexFlags flags) { out.append(ch); continue; } + if (!inClass && pattern.startsWith("(?{", i) + && !pattern.startsWith("(?{=CALL:", i)) { + // A callback introduced as text by runtime interpolation has + // no parser-created lexical closure to invoke. Preserve the + // historical compatibility behavior for that unsupported + // case: treat it as a zero-width no-op. Structured callbacks + // remain match-time Joni callouts and are handled below. + int end = findCodeBlockEnd(pattern, i); + if (end >= 0) { + out.append("(?:)"); + i = end; + continue; + } + } if (!inClass && pattern.startsWith("(?[", i)) { StringBuilder translatedClass = new StringBuilder(); int end = ExtendedCharClass.handleExtendedCharacterClass( @@ -153,6 +176,22 @@ private static String translatePattern(String pattern, RegexFlags flags) { return out.toString(); } + private static int findCodeBlockEnd(String pattern, int offset) { + int depth = 1; + for (int i = offset + 3; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + } else if (ch == '{') { + depth++; + } else if (ch == '}' && --depth == 0) { + return i + 1 < pattern.length() && pattern.charAt(i + 1) == ')' + ? i + 1 : -1; + } + } + return -1; + } + /** * Joni's Ruby syntax does not understand Java's {@code &&} character-class * intersection. For an explicitly ASCII-bounded Perl extended class, @@ -277,13 +316,16 @@ private static final class JoniRegexMatcher implements RegexMatcher { private int regionEnd; private int nextStart; private boolean matched; + private final List callbacks; + private PerlCalloutHandler calloutHandler; JoniRegexMatcher(Regex regex, String sourcePattern, Map namedGroups, - String input) { + String input, List callbacks) { this.regex = regex; this.sourcePattern = sourcePattern; this.namedGroups = namedGroups; this.input = input; + this.callbacks = callbacks; this.bytes = input.getBytes(StandardCharsets.UTF_8); this.charToByte = buildCharToByte(input); this.byteToChar = buildByteToChar(input, bytes.length, charToByte); @@ -297,8 +339,13 @@ public boolean find() { return false; } matcher = regex.matcher(bytes); + if (!callbacks.isEmpty()) { + calloutHandler = new PerlCalloutHandler(input, byteToChar, callbacks); + matcher.setCalloutHandler(calloutHandler); + } int result = matcher.search(charToByte[nextStart], charToByte[regionEnd], Option.NONE); matched = result >= 0; + if (calloutHandler != null) calloutHandler.finish(matched); if (!matched) return false; captures = matcher.getEagerRegion(); int start = start(); @@ -404,4 +451,95 @@ private static int[] buildByteToChar(String input, int byteLength, int[] charToB return offsets; } } + + private static final class PerlCalloutHandler implements CalloutHandler { + private record Token(int localLevel, RegexState regexState, RuntimeScalar previousR, + RuntimeScalar result, boolean block) {} + + private final String input; + private final int[] byteToChar; + private final List callbacks; + private RuntimeScalar completedResult; + + PerlCalloutHandler(String input, int[] byteToChar, List callbacks) { + this.input = input; + this.byteToChar = byteToChar; + this.callbacks = callbacks; + } + + @Override + public CalloutResult execute(int id, MatchView match) { + RuntimeRegexCallback callback = callbacks.get(id); + int localLevel = DynamicVariableManager.getLocalLevel(); + RegexState savedRegex = new RegexState(); + RuntimeScalar rVariable = GlobalVariable.getGlobalVariable( + GlobalContext.encodeSpecialVar("R")); + RuntimeScalar previousR = rVariable.clone(); + publishProvisional(match); + + RuntimeScalar result = RuntimeCode.apply(new RuntimeScalar(callback.code), + new RuntimeArray(), RuntimeContextType.SCALAR).scalar(); + boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK; + if (block) rVariable.set(result); + Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block); + return callback.kind == RuntimeRegexCallback.Kind.CONDITION && !result.getBoolean() + ? CalloutResult.failWith(token) : CalloutResult.continueWith(token); + } + + @Override + public void unwind(Object value) { + restore((Token) value, false); + } + + @Override + public void complete(Object value) { + restore((Token) value, true); + } + + void finish(boolean matched) { + if (matched && completedResult != null) { + GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) + .set(completedResult); + } + } + + private void restore(Token token, boolean completed) { + DynamicVariableManager.popToLocalLevel(token.localLevel()); + token.regexState().restore(); + GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) + .set(token.previousR()); + if (completed && token.block() && completedResult == null) { + completedResult = token.result(); + } + } + + private void publishProvisional(MatchView match) { + RuntimeRegexState state = PerlRuntime.current().regexState; + int count = match.captureCount(); + state.globalMatchString = input; + state.lastMatchedString = input; + state.lastMatchStart = charOffset(match.captureBegin(0)); + state.lastMatchEnd = charOffset(match.captureEnd(0)); + state.lastCaptureGroups = new String[count]; + state.manualCaptureStarts = new int[count]; + state.manualCaptureEnds = new int[count]; + for (int group = 1; group <= count; group++) { + int begin = charOffset(match.captureBegin(group)); + int end = charOffset(match.captureEnd(group)); + if (begin < 0 || end < begin) { + state.manualCaptureStarts[group - 1] = -1; + state.manualCaptureEnds[group - 1] = -1; + state.lastCaptureGroups[group - 1] = null; + } else { + state.manualCaptureStarts[group - 1] = begin; + state.manualCaptureEnds[group - 1] = end; + state.lastCaptureGroups[group - 1] = input.substring(begin, end); + } + } + } + + private int charOffset(int byteOffset) { + return byteOffset < 0 || byteOffset >= byteToChar.length ? -1 : byteToChar[byteOffset]; + } + } } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 742dc53e7..6c3b32657 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -35,6 +35,23 @@ */ public class RuntimeRegex extends RuntimeBase implements RuntimeScalarReference { + /** + * Keep a literal target's SV stable across repeated execution of one regex call site. + * Perl stores pos() on the literal SV in the compiled optree; rematerializing it on + * every loop iteration would restart /g at zero forever. + */ + public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int callsiteId) { + RuntimeRegexState state = PerlRuntime.current().regexState; + RuntimeScalar stable = state.literalRegexTargets.get(callsiteId); + if (stable == null) { + stable = new RuntimeScalar(literal.toString()); + stable.type = literal.type; + stable.tainted = literal.isTainted(); + state.literalRegexTargets.put(callsiteId, stable); + } + return stable; + } + /** Private AST/runtime modifier markers; removed before Perl modifier parsing. */ public static final char INTERNAL_DEBUG_MARKER = '\u0001'; public static final char INTERNAL_DEBUGCOLOR_MARKER = '\u0002'; @@ -143,6 +160,7 @@ private static RuntimeRegexState state() { Pattern notemptyPattern; Pattern notemptyPatternUnicode; JoniRegexPattern recursivePattern; + List executableCallbacks = List.of(); int[] branchResetCaptureMap; int patternFlags; int patternFlagsUnicode; @@ -173,7 +191,7 @@ private static RuntimeRegexState state() { // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site. private int lexicalDebugMode; private static final String DYNAMIC_PATTERN_ERROR = - "\u0000(??{...}) recursive/dynamic regex patterns not implemented"; + "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)"; public RuntimeRegex() { this.regexFlags = null; @@ -195,6 +213,7 @@ public RuntimeRegex cloneTracked() { copy.notemptyPattern = this.notemptyPattern; copy.notemptyPatternUnicode = this.notemptyPatternUnicode; copy.recursivePattern = this.recursivePattern; + copy.executableCallbacks = this.executableCallbacks; copy.branchResetCaptureMap = this.branchResetCaptureMap; copy.patternFlags = this.patternFlags; copy.patternFlagsUnicode = this.patternFlagsUnicode; @@ -421,12 +440,15 @@ private static synchronized RuntimeRegex compileSynchronized( GlobalVariable.getGlobalHash("main::ENV") .get("JPERL_UNIMPLEMENTED").toString()); boolean hasDeferredDynamicPattern = hasDynamicPattern && !warnOnUnimplemented; - if (hasDeferredDynamicPattern) { - // Perl permits qr// construction before the dynamic callback is - // needed. Compile a never-matching placeholder and retain a hard - // error for the first use instead of rejecting module loading. + boolean hasWarnDynamicFallback = hasDynamicPattern && warnOnUnimplemented; + if (hasDeferredDynamicPattern || hasWarnDynamicFallback) { + // Perl permits qr// construction before the dynamic callback is needed. + // Default mode keeps a never-matching placeholder and reports the hard + // error on use. Warn mode retains the historical compatibility fallback + // that ignores the unsupported dynamic component after warning. compilePatternString = compilePatternString.replace( - RegexMarkers.RECURSIVE_PATTERN, "(?!)"); + RegexMarkers.RECURSIVE_PATTERN, + hasWarnDynamicFallback ? "(?:)" : "(?!)"); } List quoteMetaWarningsOnUse = new ArrayList<>(); if (compilePatternString != null && compilePatternString.contains("\\Q")) { @@ -471,7 +493,7 @@ private static synchronized RuntimeRegex compileSynchronized( String javaPattern = null; try { - boolean usesRecursiveBackend = JoniRegexPattern.requiresRecursiveBackend(compilePatternString); + boolean usesRecursiveBackend = JoniRegexPattern.requiresJoniBackend(compilePatternString); if (usesRecursiveBackend && compilePatternString.contains("(?&") && (compilePatternString.contains("(?<=") @@ -497,6 +519,9 @@ private static synchronized RuntimeRegex compileSynchronized( regex.warningsOnUse = new ArrayList<>(quoteMetaWarningsOnUse); if (hasDeferredDynamicPattern) { regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); + } else if (hasWarnDynamicFallback) { + regex.warningsOnUse.add( + "(??{...}) recursive/dynamic regex patterns not implemented\n"); } if (usesRecursiveBackend) { regex.recursivePattern = new JoniRegexPattern(compilePatternString, regex.regexFlags); @@ -1123,6 +1148,12 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS validateTaintedPatternSecurity(patternString); + if (patternString.value instanceof RuntimeRegexTemplate template) { + RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode).cloneTracked(); + regex.executableCallbacks = template.callbacks(); + return new RuntimeScalar(regex).propagateTaint(patternString); + } + // Check if patternString is already a compiled regex if (patternString.type == RuntimeScalarType.REGEX) { RuntimeRegex originalRegex = (RuntimeRegex) patternString.value; @@ -1137,6 +1168,7 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS regex.pattern = originalRegex.pattern; regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; + regex.executableCallbacks = originalRegex.executableCallbacks; regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1175,6 +1207,7 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS regex.pattern = originalRegex.pattern; regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; + regex.executableCallbacks = originalRegex.executableCallbacks; regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1298,6 +1331,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.pattern = resolvedRegex.pattern; regex.patternUnicode = resolvedRegex.patternUnicode; regex.recursivePattern = resolvedRegex.recursivePattern; + regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.branchResetCaptureMap = resolvedRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = resolvedRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = resolvedRegex.patternUnicodeNoInternalMarkers; @@ -1335,6 +1369,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; + regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.branchResetCaptureMap = recompiledRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = recompiledRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiledRegex.patternUnicodeNoInternalMarkers; @@ -1549,6 +1584,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc tempRegex.pattern = pattern; tempRegex.patternUnicode = regexState.lastSuccessfulPattern.patternUnicode; tempRegex.recursivePattern = regexState.lastSuccessfulPattern.recursivePattern; + tempRegex.executableCallbacks = regexState.lastSuccessfulPattern.executableCallbacks; tempRegex.branchResetCaptureMap = regexState.lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = regexState.lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = regexState.lastSuccessfulPattern.patternUnicodeNoInternalMarkers; @@ -1594,7 +1630,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc CharSequence matchInput = new RegexTimeoutCharSequence(inputStr); RegexMatcher matcher; if (regex.recursivePattern != null) { - matcher = regex.recursivePattern.matcher(inputStr); + matcher = regex.recursivePattern.matcher(inputStr, regex.executableCallbacks); } else { Pattern pattern = regex.selectPattern(string, inputStr); @@ -2400,6 +2436,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar tempRegex.pattern = pattern; tempRegex.patternUnicode = state().lastSuccessfulPattern.patternUnicode; tempRegex.recursivePattern = state().lastSuccessfulPattern.recursivePattern; + tempRegex.executableCallbacks = state().lastSuccessfulPattern.executableCallbacks; tempRegex.branchResetCaptureMap = state().lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = state().lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = state().lastSuccessfulPattern.patternUnicodeNoInternalMarkers; @@ -2432,7 +2469,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar Pattern pattern = null; RegexMatcher matcher; if (regex.recursivePattern != null) { - matcher = regex.recursivePattern.matcher(inputStr); + matcher = regex.recursivePattern.matcher(inputStr, regex.executableCallbacks); } else { pattern = regex.selectPattern(inputValue, inputStr); if (inputStr.isEmpty() && (pattern.flags() & Pattern.MULTILINE) != 0) { @@ -2980,6 +3017,12 @@ private static ResolvedRegex resolveRegexWithOrigin(RuntimeScalar quotedRegex) { // Unwrap readonly scalar if (quotedRegex.type == RuntimeScalarType.READONLY_SCALAR) quotedRegex = (RuntimeScalar) quotedRegex.value; + if (quotedRegex.value instanceof RuntimeRegexTemplate) { + RuntimeScalar compiled = getQuotedRegex( + quotedRegex, RuntimeScalarCache.scalarEmptyString); + return new ResolvedRegex((RuntimeRegex) compiled.value, false); + } + if (quotedRegex.type == RuntimeScalarType.REGEX) { return new ResolvedRegex((RuntimeRegex) quotedRegex.value, true); } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java new file mode 100644 index 000000000..bff85141c --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java @@ -0,0 +1,24 @@ +package org.perlonjava.runtime.regex; + +import org.perlonjava.runtime.runtimetypes.RuntimeCode; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; + +/** A parser-created executable segment in a regex template. */ +public final class RuntimeRegexCallback { + public enum Kind { BLOCK, CONDITION } + + final RuntimeCode code; + final Kind kind; + + private RuntimeRegexCallback(RuntimeCode code, Kind kind) { + this.code = code; + this.kind = kind; + } + + public static RuntimeScalar wrap(RuntimeScalar codeRef, String kindName) { + if (!(codeRef.value instanceof RuntimeCode code)) { + throw new IllegalArgumentException("regex callback is not a code reference"); + } + return new RuntimeScalar(new RuntimeRegexCallback(code, Kind.valueOf(kindName))); + } +} diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java new file mode 100644 index 000000000..0b9d56cb1 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java @@ -0,0 +1,92 @@ +package org.perlonjava.runtime.regex; + +import org.perlonjava.runtime.runtimetypes.RuntimeBase; +import org.perlonjava.runtime.runtimetypes.RuntimeList; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Runtime interpolation result that keeps executable regex callbacks out of strings. */ +public final class RuntimeRegexTemplate { + private static final Pattern CALLOUT_ID = Pattern.compile("\\(\\?\\{=CALL:(\\d+)\\}\\)"); + private final String pattern; + private final List callbacks; + + private RuntimeRegexTemplate(String pattern, List callbacks) { + this.pattern = pattern; + this.callbacks = List.copyOf(callbacks); + } + + public static RuntimeScalar build(RuntimeList parts) { + if (parts.elements.size() == 1) { + RuntimeScalar only = parts.elements.getFirst().scalar(); + // A lone interpolation must retain its runtime type so qr + // overloading and an already-compiled regex remain observable. + // Only a parser-created callback needs a new template skeleton. + if (!(only.value instanceof RuntimeRegexCallback)) return only; + } + StringBuilder pattern = new StringBuilder(); + List callbacks = new ArrayList<>(); + boolean tainted = false; + for (RuntimeBase part : parts.elements) { + RuntimeScalar scalar = part.scalar(); + tainted |= scalar.isTainted(); + if (scalar.value instanceof RuntimeRegexCallback callback) { + int id = callbacks.size(); + callbacks.add(callback); + if (callback.kind == RuntimeRegexCallback.Kind.CONDITION) { + pattern.append("?{=CALL:").append(id).append("})"); + } else { + pattern.append("(?{=CALL:").append(id).append("})"); + } + } else if (scalar.value instanceof RuntimeRegex regex + && !regex.executableCallbacks.isEmpty()) { + appendEmbeddedRegex(pattern, callbacks, regex.toString(), regex.executableCallbacks); + } else if (scalar.value instanceof RuntimeRegexTemplate template) { + appendEmbeddedRegex(pattern, callbacks, template.pattern, template.callbacks); + } else { + pattern.append(scalar); + } + } + RuntimeScalar result = callbacks.isEmpty() + ? new RuntimeScalar(pattern.toString()) + : new RuntimeScalar(new RuntimeRegexTemplate(pattern.toString(), callbacks)); + result.tainted = tainted; + return result; + } + + private static void appendEmbeddedRegex(StringBuilder pattern, + List callbacks, + String embeddedPattern, + List embeddedCallbacks) { + int offset = callbacks.size(); + Matcher matcher = CALLOUT_ID.matcher(embeddedPattern); + StringBuilder remapped = new StringBuilder(); + while (matcher.find()) { + int oldId = Integer.parseInt(matcher.group(1)); + if (oldId < 0 || oldId >= embeddedCallbacks.size()) { + throw new IllegalArgumentException("Invalid embedded regex callout ID " + oldId); + } + matcher.appendReplacement(remapped, "(?{=CALL:" + (offset + oldId) + "})"); + } + matcher.appendTail(remapped); + pattern.append(remapped); + callbacks.addAll(embeddedCallbacks); + } + + String pattern() { + return pattern; + } + + List callbacks() { + return callbacks; + } + + @Override + public String toString() { + return pattern; + } +} diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java index df5e6f739..1fef6b84d 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java @@ -1444,8 +1444,17 @@ public RuntimeGlob undefine() { // Undefine FORMAT GlobalVariable.getGlobalFormatRef(this.globName).undefineFormat(); - // Undefine SCALAR - GlobalVariable.getGlobalVariable(this.globName).set(new RuntimeScalar()); + // Release a mutable scalar's current value first so tracked objects run + // DESTROY immediately. A typeglob scalar slot can also directly alias + // a read-only constant (for example after `*a = "a"`); that referent + // cannot and need not be cleared. + RuntimeScalar oldScalar = GlobalVariable.globalVariables.get(this.globName); + if (oldScalar != null && !(oldScalar instanceof RuntimeScalarReadOnly)) { + oldScalar.undefine(); + } + // Replace the slot in either case. Undefining the glob severs aliases; + // it must not leave a read-only constant installed as the glob's SV. + GlobalVariable.aliasGlobalVariable(this.globName, new RuntimeScalar()); // Undefine ARRAY - clear the existing array (fires DESTROY on blessed // elements via MortalList) before replacing with an empty one. Just diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java index 9cabdebdf..e96fedb0f 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java @@ -40,6 +40,8 @@ public final class RuntimeRegexState { /** Per-runtime callsite state for {@code /o} and {@code m?PAT?}. */ public final Map optimizedRegexCache = new LinkedHashMap<>(); + /** Stable scalar identities for literal regex targets, keyed by compiled call site. */ + public final Map literalRegexTargets = new LinkedHashMap<>(); public final Map userUnicodePropertyCache = new LinkedHashMap<>(); /** diff --git a/src/test/resources/unit/regex/executable_callbacks.t b/src/test/resources/unit/regex/executable_callbacks.t new file mode 100644 index 000000000..217a6f8cf --- /dev/null +++ b/src/test/resources/unit/regex/executable_callbacks.t @@ -0,0 +1,56 @@ +use strict; +use warnings; +use Test::More; + +our $localized = 'outer'; +my $executions = 0; +my @seen; +my $rx = qr/(a)(?{ + ++$executions; + push @seen, "$1:$localized"; + local $localized = 'inside'; + 17; +})(b|c)/; + +is($executions, 0, 'plain callback is not executed while qr is constructed'); +ok('ac' =~ $rx, 'plain callback pattern matches'); +is($executions, 1, 'plain callback executes at match time'); +is_deeply(\@seen, ['a:outer'], 'callback sees provisional captures and lexical state'); +is($^R, 17, 'last successful plain callback publishes $^R'); +is($localized, 'outer', 'callback local scope unwinds after a successful match'); +my $embedded = qr/(?:)$rx(?:)/; +is($executions, 1, 'embedding a callback regex does not execute its closure'); +ok('ab' =~ $embedded, 'embedded callback regex matches'); +is($executions, 2, 'embedded regex retains its callback table'); + +my $literal_call_id = qr/CALL:999(?{ ++$executions })/; +my $embedded_literal_call_id = qr/$literal_call_id/; +ok('CALL:999' =~ $embedded_literal_call_id, + 'single-segment embedding remaps only internal callout markers'); +is($executions, 3, 'literal callout-like text does not corrupt the callback table'); + +@seen = (); +my $backtracking = qr/ + a(?{ local $localized = 'branch'; push @seen, 'first'; 11 })b + | + a(?{ push @seen, $localized; 23 })c +/x; +ok('ac' =~ $backtracking, 'engine backtracks across callbacks'); +is_deeply(\@seen, ['first', 'outer'], 'backtracking restores callback local scope'); +is($^R, 23, 'backtracked callback result does not replace successful $^R'); + +my $condition_calls = 0; +ok('ab' =~ /a(?(?{ ++$condition_calls; 1 })b|c)/, + 'true callback condition selects yes branch'); +ok('ac' =~ /a(?(?{ ++$condition_calls; 0 })b|c)/, + 'false callback condition selects no branch'); +is($condition_calls, 2, 'each callback condition executes once'); + +my $literal_iterations = 0; +while ("\x{100}bc" =~ /(..?)(?{ $^N })/g) { + ++$literal_iterations; + last if $literal_iterations > 2; +} +is($literal_iterations, 2, 'literal target preserves /g position across callback matches'); + +done_testing(); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index f7190c6b8..fe61398a4 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -277,7 +277,7 @@ private void renumberByMap(Node node, int[]map) { case NodeType.ENCLOSE: EncloseNode en = (EncloseNode)node; - if (en.type == EncloseType.CONDITION) { + if (en.type == EncloseType.CONDITION && en.calloutConditionId < 0) { en.regNum = map[en.regNum]; } renumberByMap(en.target, map); diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java index 5bd5a4996..7b527c743 100644 --- a/third_party/joni/src/org/joni/ArrayCompiler.java +++ b/third_party/joni/src/org/joni/ArrayCompiler.java @@ -875,7 +875,7 @@ private int compileLengthEncloseNode(EncloseNode node) { break; case EncloseType.CONDITION: - len = OPSize.CONDITION; + len = node.calloutConditionId >= 0 ? OPSize.CALLOUT_CONDITION : OPSize.CONDITION; if (node.target.getType() == NodeType.ALT) { ListNode x = (ListNode)node.target; tlen = compileLengthTree(x.value); /* yes-node */ @@ -975,8 +975,11 @@ protected void compileEncloseNode(EncloseNode node) { break; case EncloseType.CONDITION: - addOpcode(OPCode.CONDITION); - addMemNum(node.regNum); + if (node.calloutConditionId >= 0) regex.requireStack = true; + addOpcode(node.calloutConditionId >= 0 + ? OPCode.CALLOUT_CONDITION : OPCode.CONDITION); + addMemNum(node.calloutConditionId >= 0 + ? node.calloutConditionId : node.regNum); if (node.target.getType() == NodeType.ALT) { ListNode x = (ListNode)node.target; len = compileLengthTree(x.value); /* yes-node */ diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index e7d1d903d..db5a27739 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -168,11 +168,14 @@ protected final int matchAt(int _range, int _sstart, int _sprev, boolean interru bestLen = -1; s = _sstart; pkeep = _sstart; + int result = -1; try { - return enc.isSingleByte() || (msaOptions & Option.CR_7_BIT) != 0 + result = enc.isSingleByte() || (msaOptions & Option.CR_7_BIT) != 0 ? executeSb(interrupt) : execute(interrupt); + return result; } finally { - unwindActiveCallouts(); + if (result >= 0) completeActiveCallouts(); + else unwindActiveCallouts(); } } @@ -299,6 +302,7 @@ private final int execute(final boolean checkThreadInterrupt) throws Interrupted case OPCode.FINISH: return finish(); case OPCode.FAIL: opFail(); continue; case OPCode.CALLOUT: opCallout(); continue; + case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; case OPCode.STATE_CHECK_ANYCHAR_STAR: if (USE_CEC) {opStateCheckAnyCharStar(); break;} case OPCode.STATE_CHECK_ANYCHAR_ML_STAR:if (USE_CEC) {opStateCheckAnyCharMLStar();break;} @@ -440,6 +444,7 @@ private final int executeSb(final boolean checkThreadInterrupt) throws Interrupt case OPCode.FINISH: return finish(); case OPCode.FAIL: opFail(); continue; case OPCode.CALLOUT: opCallout(); continue; + case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; case OPCode.EXACT1_IC_SB: opExact1ICSb(); break; case OPCode.EXACTN_IC_SB: opExactNICSb(); continue; @@ -1936,6 +1941,17 @@ private void opCallout() { if (result.getAction() == CalloutAction.FAIL) opFail(); } + private void opCalloutCondition() { + int calloutId = code[ip++]; + int addr = code[ip++]; + CalloutHandler handler = getCalloutHandler(); + if (handler == null) throw new IllegalStateException("callout pattern has no handler"); + CalloutResult result = handler.execute(calloutId, this); + if (result == null) throw new NullPointerException("callout result"); + if (result.getBacktrackToken() != null) pushCallout(result.getBacktrackToken()); + if (result.getAction() == CalloutAction.FAIL) ip += addr; + } + @Override public int currentBytePosition() { return s - str; diff --git a/third_party/joni/src/org/joni/CalloutHandler.java b/third_party/joni/src/org/joni/CalloutHandler.java index 3722b2caa..8ab64a7a1 100644 --- a/third_party/joni/src/org/joni/CalloutHandler.java +++ b/third_party/joni/src/org/joni/CalloutHandler.java @@ -24,4 +24,9 @@ public interface CalloutHandler { CalloutResult execute(int calloutId, MatchView match); void unwind(Object backtrackToken); + + /** Complete a token on a successful path. The default preserves the original cleanup contract. */ + default void complete(Object backtrackToken) { + unwind(backtrackToken); + } } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 6077ae08d..cd2be3dda 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -523,8 +523,12 @@ private Node parseEnclose(TokenType term) { if (left() && syntax.op2QMarkLParenCondition()) { int num = -1; int name = -1; + int calloutConditionId = -1; fetch(); - if (enc.isDigit(c)) { /* (n) */ + if (c == '?' && left() && peekIs('{')) { + fetch(); + calloutConditionId = parseInternalCalloutId(); + } else if (enc.isDigit(c)) { /* (n) */ unfetch(); num = fetchName('(', true); if (syntax.strictCheckBackref()) { @@ -544,6 +548,7 @@ private Node parseEnclose(TokenType term) { } EncloseNode en = new EncloseNode(EncloseType.CONDITION); en.regNum = num; + en.calloutConditionId = calloutConditionId; if (name != -1) en.setNameRef(); node = en; } else { @@ -716,6 +721,10 @@ private Node parseEnclose(TokenType term) { } private Node parseInternalCallout() { + return new CalloutNode(parseInternalCalloutId()); + } + + private int parseInternalCalloutId() { final String prefix = "=CALL:"; for (int i = 0; i < prefix.length(); i++) { if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); @@ -743,7 +752,7 @@ private Node parseInternalCallout() { if (c != ')') newSyntaxException(UNMATCHED_CLOSE_PARENTHESIS); returnCode = 0; - return new CalloutNode(calloutId); + return calloutId; } private Node parseEncloseNamedGroup2(boolean listCapture) { diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java index 18a9f7657..d3fb30433 100644 --- a/third_party/joni/src/org/joni/StackMachine.java +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -402,6 +402,18 @@ protected final void unwindActiveCallouts() { } } + protected final void completeActiveCallouts() { + if (stack == null) return; + for (int i = stk - 1; i >= 0; i--) { + if (stack[i].type == CALLOUT) completeCallout(stack[i]); + } + } + + private void completeCallout(StackEntry entry) { + Object token = entry.takeCalloutToken(); + if (token != null) getCalloutHandler().complete(token); + } + private void unwindCallout(StackEntry entry) { Object token = entry.takeCalloutToken(); if (token != null) getCalloutHandler().unwind(token); diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java index 680ded2b8..4e791d164 100644 --- a/third_party/joni/src/org/joni/ast/EncloseNode.java +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -33,6 +33,7 @@ public final class EncloseNode extends StateNode implements EncloseType { public int maxLength; // OnigDistance public int charLength; public int optCount; // referenced count in optimize_node_left() + public int calloutConditionId = -1; public Node containingAnchor; // // node_new_enclose / onig_node_new_enclose @@ -86,6 +87,7 @@ public String toString(int level) { value.append(", maxLength: " + maxLength); value.append(", charLength: " + charLength); value.append(", optCount: " + optCount); + value.append(", calloutConditionId: " + calloutConditionId); value.append("\n target: " + pad(target, level + 1)); return value.toString(); } diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java index 4029d2d20..9a382b6a9 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPCode.java +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -142,6 +142,7 @@ public interface OPCode { int EXACT1_IC_SB = 98; /* single byte, N = 1, ignore case */ int EXACTN_IC_SB = 99; /* single byte, ignore case */ int CALLOUT = 100; /* generic match-time callout */ + int CALLOUT_CONDITION = 101; /* callback-selected conditional */ String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { "finish", /*OP_FINISH*/ @@ -246,6 +247,7 @@ public interface OPCode { "exact1-ic-sb", /*OP_EXACT1_IC*/ "exactn-ic-sb", /*OP_EXACTN_IC*/ "callout", /*OP_CALLOUT*/ + "callout-condition", /*OP_CALLOUT_CONDITION*/ } : null; int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { @@ -351,5 +353,6 @@ public interface OPCode { Arguments.SPECIAL, /*OP_EXACT1_IC*/ Arguments.SPECIAL, /*OP_EXACTN_IC*/ Arguments.MEMNUM, /*OP_CALLOUT*/ + Arguments.SPECIAL, /*OP_CALLOUT_CONDITION*/ } : null; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java index de452b5fc..d5a411b69 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPSize.java +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -72,6 +72,7 @@ public interface OPSize { int ABSENT = (OPCODE + RELADDR); int ABSENT_END = OPCODE; int CALLOUT = (OPCODE + MEMNUM); + int CALLOUT_CONDITION = (OPCODE + MEMNUM + RELADDR); // #ifdef USE_COMBINATION_EXPLOSION_CHECK int STATE_CHECK = (OPCODE + STATE_CHECK_NUM);