diff --git a/build.gradle b/build.gradle index f065399dc..bee88bff1 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,9 @@ version = '5.44.0' // CycloneDX SBOM generation configuration cyclonedxBom { + // Production runtime dependencies are the contents of the standalone JAR; + // test and annotation configurations do not belong in the shipped SBOM. + includeConfigs = ["runtimeClasspath"] projectType = "application" schemaVersion = "1.5" includeLicenseText = false @@ -229,7 +232,7 @@ dependencies { implementation libs.bcpkix // Bouncy Castle PEM/PKCS parsing implementation libs.snappy.java // Official Sereal Java codec compression implementation libs.zstd.jni // Official Sereal Java codec compression - implementation 'org.jruby.joni:joni:2.2.7' // Stack-safe recursive regex backend + implementation 'org.jruby.jcodings:jcodings:1.0.64' // Encoding support for vendored Joni implementation 'io.netty:netty-codec-http:4.1.115.Final' // Netty HTTP codec for PSGI server // Testing dependencies @@ -343,6 +346,23 @@ shadowJar { } exclude 'module-info.class' exclude 'META-INF/MANIFEST.MF' + + // Keep the embedded regex engine private when PerlOnJava shares a + // classpath with JRuby or another Joni/JCodings version. + relocate 'org.joni', 'org.perlonjava.internal.joni' + relocate 'org.jcodings', 'org.perlonjava.internal.jcodings' + + from('third_party/joni/LICENSE') { + into 'META-INF/licenses' + rename { 'joni-LICENSE.txt' } + } + from('third_party/licenses/jcodings-LICENSE.txt') { + into 'META-INF/licenses' + } + from('third_party/joni/PERLONJAVA-NOTICE.md') { + into 'META-INF/licenses' + rename { 'joni-PERLONJAVA-NOTICE.md' } + } // Include combined SBOM in JAR's META-INF/sbom/ directory from("$buildDir/reports/sbom.json") { @@ -350,6 +370,17 @@ shadowJar { } } +tasks.register('verifyJoniPackaging', Exec) { + description = 'Verifies Joni namespace isolation, notices, and SBOM metadata' + group = 'verification' + dependsOn shadowJar + inputs.file("dev/tools/verify-joni-packaging.pl") + inputs.file("target/perlonjava-${project.version}.jar") + inputs.file("build/reports/sbom.json") + commandLine 'perl', 'dev/tools/verify-joni-packaging.pl', + "target/perlonjava-${project.version}.jar", 'build/reports/sbom.json' +} + // Some Perl unit tests spawn a child interpreter via $^X. Under Gradle those // children go through the repository launcher, so the target jar must be built // before tests run; otherwise a stale target/perlonjava-*.jar can be executed. @@ -392,6 +423,7 @@ tasks.register('mergeSbom', Exec) { // Declare inputs so Gradle knows dependencies inputs.file("build/reports/bom.json") inputs.file("build/reports/perl-bom.json") + inputs.file("dev/tools/merge-sbom.pl") workingDir = projectDir commandLine 'perl', 'dev/tools/merge-sbom.pl', @@ -420,6 +452,11 @@ tasks.named('build') { // Source sets configuration for including Perl resources sourceSets { main { + java { + // Start below Joni's optional Java 9 module descriptor so the + // otherwise classpath-based PerlOnJava build remains unnamed. + srcDir 'third_party/joni/src/org' + } resources { srcDir 'src/main/perl' srcDir 'src/main/resources' @@ -437,6 +474,11 @@ sourceSets { include 'META-INF/services/**' } } + joniTest { + java.srcDir 'third_party/joni/test' + compileClasspath += sourceSets.main.output + configurations.testRuntimeClasspath + runtimeClasspath += output + compileClasspath + } test { resources { srcDir 'src/test/resources' @@ -444,6 +486,23 @@ sourceSets { } } +configurations { + joniTestImplementation.extendsFrom testImplementation + joniTestRuntimeOnly.extendsFrom testRuntimeOnly +} + +dependencies { + joniTestImplementation 'junit:junit:4.13.2' +} + +tasks.register('testJoni', Test) { + description = 'Runs the imported upstream Joni test suite' + group = 'verification' + testClassesDirs = sourceSets.joniTest.output.classesDirs + classpath = sourceSets.joniTest.runtimeClasspath + useJUnit() +} + // Resource processing configuration tasks.named('processResources', Copy) { from(sourceSets.main.resources.srcDirs) { @@ -498,6 +557,8 @@ tasks.register('testUnitParallel') { description = 'Runs unit tests in parallel across multiple JVMs. Usage: gradle testUnitParallel --parallel' def shardTasks = (0..{components}) { for my $comp (@{$java_bom->{components}}) { @@ -97,6 +100,43 @@ } } +# Joni is compiled from the pinned source snapshot in third_party/joni rather +# than resolved as a binary dependency, so CycloneDX cannot discover it from +# runtimeClasspath. Record the vendored component explicitly. +push @all_components, { + type => 'library', + 'bom-ref' => $joni_ref, + group => 'org.jruby.joni', + name => 'joni', + version => '2.2.7', + description => 'Vendored Java port of Oniguruma with PerlOnJava callout extensions', + licenses => [ + { + license => { + id => 'MIT' + } + } + ], + purl => $joni_ref, + externalReferences => [ + { + type => 'website', + url => 'https://github.com/jruby/joni' + }, + { + type => 'vcs', + url => 'https://github.com/jruby/joni.git' + } + ], + properties => [ + { + name => 'perlonjava:vendored', + value => 'true' + } + ] +}; +push @root_deps, $joni_ref; + # Add Perl components if ($perl_bom->{components}) { for my $comp (@{$perl_bom->{components}}) { @@ -112,6 +152,10 @@ { ref => 'perlonjava', dependsOn => \@root_deps + }, + { + ref => $joni_ref, + dependsOn => [$jcodings_ref] } ]; diff --git a/dev/tools/verify-joni-packaging.pl b/dev/tools/verify-joni-packaging.pl new file mode 100644 index 000000000..27e6760c9 --- /dev/null +++ b/dev/tools/verify-joni-packaging.pl @@ -0,0 +1,49 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use JSON::PP; + +die "Usage: $0 \n" unless @ARGV == 2; +my ($jar_file, $sbom_file) = @ARGV; + +my %entries; +open my $jar_fh, '-|', 'jar', 'tf', $jar_file + or die "Cannot list $jar_file: $!\n"; +while (my $entry = <$jar_fh>) { + chomp $entry; + $entries{$entry} = 1; +} +close $jar_fh or die "Cannot list $jar_file: jar exited with status $?\n"; + +die "Standalone JAR contains unrelocated Joni classes\n" + if grep { m{^org/joni/} } keys %entries; +die "Standalone JAR contains unrelocated JCodings classes\n" + if grep { m{^org/jcodings/} } keys %entries; +die "Standalone JAR does not contain relocated Joni classes\n" + unless grep { m{^org/perlonjava/internal/joni/} } keys %entries; +die "Standalone JAR does not contain relocated JCodings classes\n" + unless grep { m{^org/perlonjava/internal/jcodings/} } keys %entries; + +for my $notice (qw( + joni-LICENSE.txt + jcodings-LICENSE.txt + joni-PERLONJAVA-NOTICE.md +)) { + die "Standalone JAR is missing $notice\n" + unless $entries{"META-INF/licenses/$notice"}; +} + +open my $sbom_fh, '<', $sbom_file or die "Cannot open $sbom_file: $!\n"; +local $/; +my $sbom = JSON::PP->new->utf8->decode(<$sbom_fh>); +close $sbom_fh; + +my ($joni) = grep { + ($_->{group} // '') eq 'org.jruby.joni' + && ($_->{name} // '') eq 'joni' + && ($_->{version} // '') eq '2.2.7' +} @{$sbom->{components} // []}; +die "Combined SBOM is missing vendored Joni 2.2.7\n" unless $joni; + +print "Joni packaging verification passed\n"; diff --git a/docs/README.md b/docs/README.md index 90946cc1e..7db719c4e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,7 @@ Looking to contribute? See: - **[CONTRIBUTING.md](../CONTRIBUTING.md)** - Contribution guidelines - **[dev/](../dev/)** - Developer documentation and internal architecture +- **[Joni Callout Fork](design/joni-callout-fork.md)** - Vendored regex engine design and implementation contract ## Finding What You Need diff --git a/docs/design/joni-callout-fork.md b/docs/design/joni-callout-fork.md new file mode 100644 index 000000000..d019672c4 --- /dev/null +++ b/docs/design/joni-callout-fork.md @@ -0,0 +1,214 @@ +# Vendored Joni Callout Engine + +## Purpose + +Perl executable regex constructs must run while the matcher owns provisional +captures and its backtracking stack. Stock Joni provides the required +stack-based matcher but does not expose match-time callbacks or unwind +notifications. PerlOnJava therefore vendors a minimally extended Joni engine. + +This document defines the current architecture and acceptance contract. Change +history, imported revision details, and completed-work records belong in Git +commit messages, not here. + +## Repository and dependency model + +- Joni is a pinned source snapshot in `third_party/joni`. Its production sources + and upstream tests are dedicated source sets in the root Gradle project. This + keeps the code boundary explicit without exposing the vendored engine as a + separately resolved project dependency. +- The source retains the upstream `org.joni` packages to keep changes reviewable + and suitable for upstream submission. +- The vendored project is the only Joni implementation on PerlOnJava's compile + and runtime classpaths. +- JCodings remains an upstream binary dependency because PerlOnJava does not + modify its encoding implementation. +- Ordinary Java-regex patterns keep their existing fast path. Declarative + recursive patterns and executable patterns use the vendored Joni engine. + +## Internal callout syntax + +The structured Perl regex frontend emits `(?{=CALL:})` only in an +engine-facing skeleton. `` is a non-negative decimal index into the callback +table owned by that regex value. The fork parses this representation into a +dedicated callout node; it never parses Perl source. + +Runtime-interpolated pattern text must not be promoted to trusted skeleton +syntax. The PerlOnJava frontend remains responsible for preserving source +origin and rejecting injected internal tokens. + +## Engine API + +The fork exposes a runtime-neutral API in `org.joni`: + +```java +interface CalloutHandler { + CalloutResult execute(int calloutId, MatchView match); + void unwind(Object backtrackToken); + default void complete(Object successfulToken) { unwind(successfulToken); } +} + +interface MatchView { + int currentBytePosition(); + int captureCount(); + int captureBegin(int capture); + int captureEnd(int capture); +} +``` + +The handler is installed on each `Matcher`, not on the compiled `Regex`, so a +shared compiled pattern can execute concurrently with runtime-local callback +state. + +`CalloutResult` selects `CONTINUE` or `FAIL` and may carry an opaque unwind +token. The engine neither interprets that token nor depends on PerlOnJava +runtime classes. + +## Execution and unwind contract + +1. The callout opcode invokes the handler with the callout ID and a read-only + view of the current byte position and provisional capture boundaries. +2. A non-null token is stored in a matcher stack frame before execution + continues or fails. +3. Backtracking across that frame calls `unwind(token)` exactly once. +4. Successful completion calls `complete(token)` for every still-active token in + reverse execution order. Its default implementation delegates to `unwind`, + preserving the cleanup-only contract for engine-neutral handlers. +5. Failed completion, interruption, timeout, and handler exceptions unwind every + still-active token in reverse execution order. +6. A `FAIL` result enters normal matcher backtracking after installing the frame, + so cleanup follows the same path as later-pattern failure. +7. Patterns without callout nodes allocate no handler state or callout frames. + +Callback conditions use an internal conditional-callout opcode. `CONTINUE` +selects the yes branch and `FAIL` selects the no branch without treating the +condition itself as a failed match. Dynamic nested patterns remain outside this +contract until their alternatives can participate in outer backtracking. + +## Structured Perl frontend + +Literal callback bodies are represented as lexical `SubroutineNode` closures. +The parser emits a `regexTemplate` operation whose ordered parts contain normal +interpolation values and explicit `regexCallback` wrappers. Runtime template +construction assigns callback IDs and produces the engine-facing skeleton. + +An interpolated coderef remains ordinary interpolation because only the parser +can create a callback wrapper. Runtime strings containing Perl eval groups never +become trusted callback skeletons. They retain the existing security checks and, +where compatibility mode permits unsupported eval groups, the existing +zero-width no-op behavior. + +At execution, the bridge publishes provisional numbered captures and offsets, +runs the closure in scalar context, and updates `$^R` for plain callbacks only. +Each token checkpoints regex state, `$^R`, and the dynamic-local stack. Backtrack +unwind restores all three; successful completion restores dynamic scope while +retaining the last successful plain callback result. + +Every structured executable callback template selects Joni, including a +callback whose body happens to return a constant: `(?{ 1 })` still has match-time +side effects and unwind semantics. Constant dynamic-pattern expressions +`(??{ ... })` may retain the existing compile-time fold path; a runtime-dependent +dynamic-pattern closure selects Joni only after the nested-pattern backtracking +contract is implemented. + +Literal match targets have one stable scalar identity per compiled call site so +`pos()` and `/g` survive repeated loop execution without allowing two identical +literal occurrences to share state. + +## Capture view + +Capture offsets are byte offsets relative to the matcher's input buffer, matching +Joni's native representation. Capture zero describes the provisional overall +match from the current match start to the current instruction position. Unset +captures return `Region.REGION_NOTPOS`. The bridge converts offsets to Perl +character positions when publishing match variables. + +The view is valid only during `execute`; handlers must copy any data they retain. +The engine must not allocate a `Region` snapshot for each callout. + +## Packaging and namespace isolation + +Source code remains in `org.joni`, but the standalone JAR relocates it to +`org.perlonjava.internal.joni`. JCodings is relocated to +`org.perlonjava.internal.jcodings` because Joni's API and implementation refer +to its classes. This lets an embedding application load PerlOnJava beside JRuby +or stock Joni without linkage collisions. + +The standalone JAR contains the unmodified Joni and JCodings license texts and a +separate PerlOnJava modification notice under `META-INF/licenses`. The SBOM must +identify the vendored Joni component, its upstream version, MIT license, and +JCodings dependency. + +## Regex preprocessing boundary + +Do not move the entire `RegexPreprocessor` into the Joni fork. The ownership +boundary is: + +- PerlOnJava owns Perl source policy and runtime integration: trusted callback + templates, `use re 'eval'` security, lexical warnings, user-defined Unicode + properties, Perl diagnostics, named/branch-reset capture mapping, and backend + selection. +- A backend-neutral Perl normalization layer may own syntax aliases shared by + all engines, such as accepted named-group spellings and modifier validation. +- Joni should own matcher semantics required by Perl, including condition + execution, backtracking-visible state, native capture behavior, case-folding + differences, and constructs that cannot be represented faithfully by textual + rewrites. +- The Java-only adapter retains Java `Pattern` syntax rewrites and stack-safety + workarounds while the Java fast path exists. + +Migration is incremental. First inventory each preprocessing rule as source +policy, backend-neutral syntax, Java adaptation, or matcher semantics. Move only +the matcher-semantic category into focused Joni changes with differential Perl +tests. A future decision to make Joni the default backend requires corpus and +performance evidence; it is not implied by executable-callback support. + +The Joni routing roadmap is the missing-regex feature list in +`docs/reference/feature-matrix.md`: dynamically scoped regex state, recursive +and dynamic patterns, backtracking verbs and definitions, variable lookbehind, +branch reset, subpattern calls, conditions, extended Unicode and graphemes, +embedded code, regex debugging, runtime eval, lexical default flags, and named +capture behavior. As each feature becomes executable, patterns containing it +select Joni. The current selector routes declarative subpattern calls and every +structured executable callback template; constant `(??{...})` expressions keep +their established fold-to-pattern path. + +## Implementation stages + +1. Import and build the pinned upstream snapshot with its original tests through + the dedicated root source sets. +2. Replace the external dependency and prove declarative regex parity. +3. Add the callout parser node, opcode, handler API, provisional match view, and + forward execution. +4. Add matcher-owned callout frames and exact-once cleanup for every exit path. +5. Connect structured Perl callback templates and lexical closures. +6. Publish provisional match state for plain callbacks and preserve `$^R` across + successful completion and backtracking. +7. Add dynamic-local checkpoints and callback conditions. +8. Add dynamic nested patterns only after focused differential tests establish + their outer-backtracking contract. + +## Verification + +- The complete imported Joni suite passes through the root build. +- Existing recursive-regex tests match stock Joni behavior. +- Focused tests cover provisional captures, repeated execution after + backtracking, `FAIL`, nested frames, handler exceptions, interruption, and + exact-once reverse-order unwind. +- The standalone JAR contains no `org/joni` or `org/jcodings` classes, contains + both relocated trees, and includes all required notices. +- An embedding smoke test can load stock Joni and PerlOnJava together. +- Full `make` and the focused direct/thread regex matrix pass. +- The executable-callback unit test passes under standard Perl and both + PerlOnJava execution backends. +- `dev/tools/perl_test_runner.pl perl5_t/t/re/` is compared file-by-file with + `../PerlOnJava/logs/test_20260815_080000_958.log`; no previously passing regex + file may regress, and changed pass counts or blocked-test totals are reported. + +## Stop conditions + +- Do not approximate callbacks as post-match hooks. +- Do not enable a callback form if provisional state or unwind is incomplete. +- Do not implement dynamic patterns as atomic nested matches; their alternatives + must participate in outer backtracking. +- Keep unsupported syntax fatal until its complete semantic gate passes. diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 758aa4e3f..fd4463671 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -390,15 +390,15 @@ my @copy = @{$z}; # ERROR ### Missing Regular Expression Features - ❌ **Dynamically-scoped regex variables**: Regex variables are not dynamically-scoped. -- ❌ **Recursive Patterns**: Features like `(?R)`, `(?0)` or `(??{ code })` for recursive matching are not supported. +- 🟡 **Recursive Patterns**: `(?R)` and `(?0)` execute through Joni. Constant `(??{ code })` expressions fold to a pattern; runtime-dependent dynamic patterns remain unsupported. - ❌ **Backtracking Control Verbs and Definitions**: Features such as `(*PRUNE)`, `(*SKIP)`, and `(?(DEFINE)...)` are not supported. Atomic groups `(?>...)` are supported as noted above. - ❌ **Lookbehind Assertions**: Variable-length negative or positive lookbehind assertions, e.g., `(?<=...)` or `(? cp > 255); + if (node.forceByteString || isAsciiOnly(node.value) + || (!hasWideChars && emitterContext != null && emitterContext.symbolTable != null + && !emitterContext.symbolTable.isStrictOptionEnabled(Strict.HINT_UTF8))) { + literal.type = RuntimeScalarType.BYTE_STRING; + } + // Do not value-deduplicate this entry with ordinary read-only literals. + // Its identity is the pos() storage for this one regex call site. + int constant = constants.size(); + constants.add(literal); + int register = allocateOutputRegister(); + emit(Opcodes.LOAD_CONST); + emitReg(register); + emit(constant); + lastResultReg = register; + return register; + } + @Override public void visit(IdentifierNode node) { // Variable reference diff --git a/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java b/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java index abadb91f1..021700a65 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java +++ b/src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java @@ -1036,6 +1036,21 @@ private static RuntimeList execute(SuspendedInterpreterFrame frame) { registers[rd] = RuntimeCode.copyDoBlockListResult(registers[rs]); } + case Opcodes.REGEX_CALLBACK -> { + int rd = bytecode[pc++]; + int codeReg = bytecode[pc++]; + int kindIdx = bytecode[pc++]; + registers[rd] = org.perlonjava.runtime.regex.RuntimeRegexCallback.wrap( + registers[codeReg].scalar(), code.stringPool[kindIdx]); + } + + case Opcodes.REGEX_TEMPLATE -> { + int rd = bytecode[pc++]; + int partsReg = bytecode[pc++]; + registers[rd] = org.perlonjava.runtime.regex.RuntimeRegexTemplate.build( + (RuntimeList) registers[partsReg]); + } + case Opcodes.ASSIGN_LEXICAL_SCALAR -> { int rd = bytecode[pc++]; int rs = bytecode[pc++]; diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java index aae2f1019..059827ddf 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java @@ -323,8 +323,13 @@ private static void visitMatchRegex(BytecodeCompiler bc, OperatorNode node) { } int stringReg; if (args.elements.size() > 2) { - bc.compileNode(args.elements.get(2), -1, RuntimeContextType.SCALAR); - stringReg = bc.lastResultReg; + Node target = args.elements.get(2); + if (target instanceof StringNode literal) { + stringReg = bc.compileStableRegexLiteral(literal); + } else { + bc.compileNode(target, -1, RuntimeContextType.SCALAR); + stringReg = bc.lastResultReg; + } } else { stringReg = loadDefaultUnderscore(bc); } @@ -813,6 +818,26 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode case "getppid" -> { int rd = bytecodeCompiler.allocateOutputRegister(); bytecodeCompiler.emitWithToken(Opcodes.GETPPID, node.getIndex()); bytecodeCompiler.emitReg(rd); bytecodeCompiler.lastResultReg = rd; } case "open" -> visitOpen(bytecodeCompiler, node); case "matchRegex" -> visitMatchRegex(bytecodeCompiler, node); + case "regexCallback" -> { + bytecodeCompiler.compileNode(node.operand, -1, RuntimeContextType.SCALAR); + int codeReg = bytecodeCompiler.lastResultReg; + int rd = bytecodeCompiler.allocateOutputRegister(); + bytecodeCompiler.emit(Opcodes.REGEX_CALLBACK); + bytecodeCompiler.emitReg(rd); + bytecodeCompiler.emitReg(codeReg); + bytecodeCompiler.emit(bytecodeCompiler.addToStringPool( + (String) node.getAnnotation("regexCallbackKind"))); + bytecodeCompiler.lastResultReg = rd; + } + case "regexTemplate" -> { + bytecodeCompiler.compileNode(node.operand, -1, RuntimeContextType.LIST); + int partsReg = bytecodeCompiler.lastResultReg; + int rd = bytecodeCompiler.allocateOutputRegister(); + bytecodeCompiler.emit(Opcodes.REGEX_TEMPLATE); + bytecodeCompiler.emitReg(rd); + bytecodeCompiler.emitReg(partsReg); + bytecodeCompiler.lastResultReg = rd; + } case "replaceRegex" -> visitReplaceRegex(bytecodeCompiler, node); case "substr" -> visitSubstr(bytecodeCompiler, node); case "chomp" -> visitChomp(bytecodeCompiler, node); diff --git a/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java b/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java index 21a7a53f1..63a82d5ff 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java +++ b/src/main/java/org/perlonjava/backend/bytecode/Opcodes.java @@ -2461,6 +2461,12 @@ public class Opcodes { /** Copy a do-block scalar/list result so loose lvalues do not escape. Format: rd, rs. */ public static final short COPY_DO_BLOCK_RESULT = 519; + /** Wrap a lexical closure as an executable regex callback. Format: rd codeReg kindStringIdx. */ + public static final short REGEX_CALLBACK = 520; + + /** Build an interpolated regex template while preserving callback objects. Format: rd partsReg. */ + public static final short REGEX_TEMPLATE = 521; + /** Return the mutable {@code $#array} cell. Format: ARRAY_LAST_INDEX_LVALUE rd arrayReg. */ public static final short ARRAY_LAST_INDEX_LVALUE = 518; diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java b/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java index 95658498d..9f9b3c52c 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitOperatorNode.java @@ -159,6 +159,8 @@ public static void emitOperatorNode(EmitterVisitor emitterVisitor, OperatorNode // Regular expression operations case "matchRegex" -> EmitRegex.handleMatchRegex(emitterVisitor, node); case "quoteRegex" -> EmitRegex.handleQuoteRegex(emitterVisitor, node); + case "regexCallback" -> EmitRegex.handleRegexCallback(emitterVisitor, node); + case "regexTemplate" -> EmitRegex.handleRegexTemplate(emitterVisitor, node); case "replaceRegex" -> EmitRegex.handleReplaceRegex(emitterVisitor, node); case "tr", "y" -> EmitRegex.handleTransliterate(emitterVisitor, node); case "qx" -> EmitRegex.handleSystemCommand(emitterVisitor, node); diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java index c9229435e..3cb446718 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java @@ -20,6 +20,25 @@ * transliteration and replacement. */ public class EmitRegex { + static void handleRegexCallback(EmitterVisitor emitterVisitor, OperatorNode node) { + node.operand.accept(emitterVisitor.with(RuntimeContextType.SCALAR)); + emitterVisitor.ctx.mv.visitLdcInsn((String) node.getAnnotation("regexCallbackKind")); + emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegexCallback", "wrap", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Ljava/lang/String;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + } + + static void handleRegexTemplate(EmitterVisitor emitterVisitor, OperatorNode node) { + node.operand.accept(emitterVisitor.with(RuntimeContextType.LIST)); + emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegexTemplate", "build", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeList;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + if (emitterVisitor.ctx.contextType == RuntimeContextType.VOID) { + emitterVisitor.ctx.mv.visitInsn(Opcodes.POP); + } + } // Callsite ID counter for /o modifier support (unique across all JVM compilations) private static final AtomicInteger nextCallsiteId = new AtomicInteger(100000); @@ -359,7 +378,7 @@ static void handleMatchRegex(EmitterVisitor emitterVisitor, OperatorNode node) { emitterVisitor.ctx.mv.visitVarInsn(Opcodes.ASTORE, regexSlot); // Use default variable $_ if none specified - handleVariableBinding(operand, 2, scalarVisitor); + handleVariableBinding(operand, 2, scalarVisitor, true); emitterVisitor.ctx.mv.visitVarInsn(Opcodes.ALOAD, regexSlot); emitterVisitor.ctx.mv.visitInsn(Opcodes.SWAP); @@ -421,6 +440,12 @@ private static void emitMatchRegexRuntimeCall(EmitterVisitor emitterVisitor, Str * @param scalarVisitor The visitor used to emit scalar context bytecode */ private static void handleVariableBinding(ListNode operand, int variableIndex, EmitterVisitor scalarVisitor) { + handleVariableBinding(operand, variableIndex, scalarVisitor, false); + } + + private static void handleVariableBinding(ListNode operand, int variableIndex, + EmitterVisitor scalarVisitor, + boolean stabilizeLiteral) { // Check if a variable was provided in the operand list Node variable = null; if (operand.elements.size() > variableIndex) { @@ -434,5 +459,12 @@ private static void handleVariableBinding(ListNode operand, int variableIndex, E // Generate bytecode for the variable access variable.accept(scalarVisitor); + if (stabilizeLiteral && variable instanceof StringNode) { + scalarVisitor.ctx.mv.visitLdcInsn(nextCallsiteId.getAndIncrement()); + scalarVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, + "org/perlonjava/runtime/regex/RuntimeRegex", "stabilizeLiteralTarget", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;I)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", + false); + } } } diff --git a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java index d7e031064..6ad7839e6 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java @@ -275,7 +275,7 @@ public Node parse() { applyCaseModifier(caseModifiers.pop()); } - return createJoinNode(segments); + return needsStructuredRegexTemplate() ? buildResult() : createJoinNode(segments); } /** diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index 9293208d5..81de96e26 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -8,12 +8,10 @@ import org.perlonjava.frontend.lexer.LexerToken; import org.perlonjava.frontend.lexer.LexerTokenType; import org.perlonjava.runtime.operators.PerlUtfString; -import org.perlonjava.runtime.regex.CaptureNameEncoder; import org.perlonjava.runtime.regex.RegexMarkers; import org.perlonjava.runtime.regex.UnicodeResolver; import org.perlonjava.runtime.runtimetypes.PerlCompilerException; import org.perlonjava.runtime.runtimetypes.RuntimeScalar; -import org.perlonjava.runtime.runtimetypes.RuntimeScalarCache; import org.perlonjava.runtime.runtimetypes.ScalarUtils; import java.math.BigInteger; @@ -51,7 +49,6 @@ public abstract class StringSegmentParser { * Must be static to ensure names don't collide across different patterns that share * the same pendingCodeBlockConstants map */ - private static int codeBlockCaptureCounter = 0; /** * The emitter context for logging and error handling */ @@ -85,6 +82,8 @@ public abstract class StringSegmentParser { * List of AST nodes representing string segments (literals and interpolated expressions) */ protected final List segments; + protected boolean hasExecutableRegexCallbacks; + protected boolean hasRuntimeInterpolation; protected final boolean interpolateVariable; protected final boolean parseEscapes; /** @@ -253,6 +252,7 @@ private boolean shouldForceByteStringLiteral(String value) { */ protected void parseVariableInterpolation(String sigil) { flushCurrentSegment(); + hasRuntimeInterpolation = true; if (CompilerOptions.DEBUG_ENABLED) ctx.logDebug("str sigil"); @@ -641,6 +641,10 @@ private Node parseSimpleVariableInterpolation(String sigil) { protected Node buildResult() { flushCurrentSegment(); + if (needsStructuredRegexTemplate()) { + return new OperatorNode("regexTemplate", new ListNode(segments, tokenIndex), tokenIndex); + } + return switch (segments.size()) { case 0 -> new StringNode("", tokenIndex); case 1 -> { @@ -663,6 +667,11 @@ yield new BinaryOperatorNode("join", }; } + /** Preserve regex-valued interpolation so embedded callback tables are not stringified away. */ + protected boolean needsStructuredRegexTemplate() { + return hasExecutableRegexCallbacks || (isRegex && hasRuntimeInterpolation); + } + /** * Abstract method for parsing escape sequences. * @@ -787,7 +796,10 @@ protected boolean handleSpecialToken(String text) { } case "(" -> { // Check for (?{...}) and (??{...}) regex code blocks - only in regex context - if (isRegex && regexCodeBlocksAreActive() && isRegexCodeBlock()) { + if (isRegex && regexCodeBlocksAreActive() && isRegexCallbackCondition()) { + parseRegexCallbackCondition(); + yield true; + } else if (isRegex && regexCodeBlocksAreActive() && isRegexCodeBlock()) { parseRegexCodeBlock(false); // (?{...}) - code execution yield true; } else if (isRegex && regexCodeBlocksAreActive() && isRegexRecursiveBlock()) { @@ -819,6 +831,29 @@ private boolean isRegexCodeBlock() { return false; } + private boolean isRegexCallbackCondition() { + int currentPos = parser.tokenIndex; + return currentPos + 3 < parser.tokens.size() + && "?".equals(parser.tokens.get(currentPos).text) + && "(".equals(parser.tokens.get(currentPos + 1).text) + && "?".equals(parser.tokens.get(currentPos + 2).text) + && "{".equals(parser.tokens.get(currentPos + 3).text); + } + + private void parseRegexCallbackCondition() { + flushCurrentSegment(); + int start = tokenIndex; + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "("); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "{"); + Node block = parseBlock(parser); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, "}"); + TokenUtils.consume(parser, LexerTokenType.OPERATOR, ")"); + segments.add(new StringNode("(?(", start)); + segments.add(regexCallback(block, "CONDITION", start)); + } + /** * Checks if the current tokens form a (??{...}) recursive regex pattern. * This is similar to (?{...}) but uses the result as a regex pattern. @@ -883,92 +918,27 @@ private void parseRegexCodeBlock(boolean isRecursive) { // Consume the closing ")" that completes the (?{...}) construct TokenUtils.consume(parser, LexerTokenType.OPERATOR, ")"); - // Try to apply constant folding to the block - Node folded = ConstantFoldingVisitor.foldConstants(block); - - // If it's a BlockNode with a single element, extract it - // This handles both empty blocks (BlockNode with empty ListNode) and single-expression blocks - if (folded instanceof BlockNode blockNode) { - if (blockNode.elements.size() == 1) { - folded = blockNode.elements.get(0); - } - } - - // Check if the result is a simple constant using the visitor pattern - RuntimeScalar constantValue = - ConstantFoldingVisitor.getConstantValue(folded); - - if (constantValue != null) { - if (isRecursive) { - // For (??{...}), the constant becomes a pattern to match - // Extract the string value and insert it directly as a pattern - String patternString = constantValue.toString(); - // Insert the pattern string directly - it will be compiled as a regex - segments.add(new StringNode(patternString, savedTokenIndex)); - } else { - // For (?{...}), encode the value in a capture group for $^R - String captureName; - - // Check if it's undef (needs special encoding) - if (constantValue == RuntimeScalarCache.scalarUndef) { - captureName = String.format("cb%03du", codeBlockCaptureCounter++); - } else { - // Use CaptureNameEncoder to encode the value in the capture name - captureName = CaptureNameEncoder.encodeCodeBlockValue( - codeBlockCaptureCounter++, constantValue - ); - } - - if (captureName == null) { - // Encoding failed (e.g., name too long) - use fallback - segments.add(new StringNode(RegexMarkers.CODE_BLOCK, savedTokenIndex)); - } else { - // Encoding succeeded - create capture group - StringNode captureNode = new StringNode("(?<" + captureName + ">)", savedTokenIndex); - segments.add(captureNode); - } + if (isRecursive) { + // Preserve the existing constant (??{...}) support. Runtime-dependent + // nested patterns remain deferred to Stage 36.5. + Node folded = ConstantFoldingVisitor.foldConstants(block); + if (folded instanceof BlockNode blockNode && blockNode.elements.size() == 1) { + folded = blockNode.elements.getFirst(); } + RuntimeScalar constant = ConstantFoldingVisitor.getConstantValue(folded); + segments.add(new StringNode(constant == null + ? RegexMarkers.RECURSIVE_PATTERN : constant.toString(), savedTokenIndex)); } else { - if (isRecursive) { - segments.add(new StringNode(RegexMarkers.RECURSIVE_PATTERN, savedTokenIndex)); - } else if (isSimpleRegexCodeBlockSideEffect(block)) { - List sideEffectElements = new ArrayList<>(); - if (block instanceof BlockNode blockNode) { - sideEffectElements.addAll(blockNode.elements); - } else { - sideEffectElements.add(block); - } - sideEffectElements.add(new StringNode("", savedTokenIndex)); - segments.add(new BlockNode(sideEffectElements, savedTokenIndex)); - } else { - segments.add(new StringNode(RegexMarkers.CODE_BLOCK, savedTokenIndex)); - } + segments.add(regexCallback(block, "BLOCK", savedTokenIndex)); } } - private boolean isSimpleRegexCodeBlockSideEffect(Node block) { - if (!isRegexQuoteConstruction) { - return false; - } - Node node = block; - if (node instanceof BlockNode blockNode) { - if (blockNode.elements.size() != 1) { - return false; - } - node = blockNode.elements.get(0); - } - if (!(node instanceof OperatorNode operatorNode)) { - return false; - } - if (!operatorNode.operator.equals("++") - && !operatorNode.operator.equals("++postfix") - && !operatorNode.operator.equals("--") - && !operatorNode.operator.equals("--postfix")) { - return false; - } - return operatorNode.operand instanceof OperatorNode scalarOp - && scalarOp.operator.equals("$") - && scalarOp.operand instanceof IdentifierNode; + private Node regexCallback(Node block, String kind, int index) { + SubroutineNode closure = new SubroutineNode(null, null, null, block, false, index); + OperatorNode callback = new OperatorNode("regexCallback", closure, index); + callback.setAnnotation("regexCallbackKind", kind); + hasExecutableRegexCallbacks = true; + return callback; } /** diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 08047737c..5a02de43a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -2,6 +2,9 @@ import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; +import org.joni.CalloutHandler; +import org.joni.CalloutResult; +import org.joni.MatchView; import org.joni.NameEntry; import org.joni.Option; import org.joni.Regex; @@ -12,9 +15,13 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.List; + +import org.perlonjava.runtime.runtimetypes.*; /** - * Stack-based regex backend for declarative recursive subpatterns. + * Stack-based regex backend for Perl constructs that require matcher semantics + * beyond the Java Pattern fast path. */ final class JoniRegexPattern { private final Regex regex; @@ -29,8 +36,8 @@ final class JoniRegexPattern { namedGroups = collectNamedGroups(regex); } - RegexMatcher matcher(String input) { - return new JoniRegexMatcher(regex, sourcePattern, namedGroups, input); + RegexMatcher matcher(String input, List callbacks) { + return new JoniRegexMatcher(regex, sourcePattern, namedGroups, input, callbacks); } String patternDescription() { @@ -53,9 +60,11 @@ private static int toJoniOptions(RegexFlags flags) { return options; } - static boolean requiresRecursiveBackend(String pattern) { + static boolean requiresJoniBackend(String pattern) { if (pattern == null) return false; - return pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*" ) + return pattern.contains("(?{=CALL:") + || pattern.contains("(?(?{=CALL:") + || pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*" ) || pattern.contains("(?&") || pattern.contains("(?P>"); } @@ -91,6 +100,20 @@ private static String translatePattern(String pattern, RegexFlags flags) { out.append(ch); continue; } + if (!inClass && pattern.startsWith("(?{", i) + && !pattern.startsWith("(?{=CALL:", i)) { + // A callback introduced as text by runtime interpolation has + // no parser-created lexical closure to invoke. Preserve the + // historical compatibility behavior for that unsupported + // case: treat it as a zero-width no-op. Structured callbacks + // remain match-time Joni callouts and are handled below. + int end = findCodeBlockEnd(pattern, i); + if (end >= 0) { + out.append("(?:)"); + i = end; + continue; + } + } if (!inClass && pattern.startsWith("(?[", i)) { StringBuilder translatedClass = new StringBuilder(); int end = ExtendedCharClass.handleExtendedCharacterClass( @@ -153,6 +176,22 @@ private static String translatePattern(String pattern, RegexFlags flags) { return out.toString(); } + private static int findCodeBlockEnd(String pattern, int offset) { + int depth = 1; + for (int i = offset + 3; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + } else if (ch == '{') { + depth++; + } else if (ch == '}' && --depth == 0) { + return i + 1 < pattern.length() && pattern.charAt(i + 1) == ')' + ? i + 1 : -1; + } + } + return -1; + } + /** * Joni's Ruby syntax does not understand Java's {@code &&} character-class * intersection. For an explicitly ASCII-bounded Perl extended class, @@ -277,13 +316,16 @@ private static final class JoniRegexMatcher implements RegexMatcher { private int regionEnd; private int nextStart; private boolean matched; + private final List callbacks; + private PerlCalloutHandler calloutHandler; JoniRegexMatcher(Regex regex, String sourcePattern, Map namedGroups, - String input) { + String input, List callbacks) { this.regex = regex; this.sourcePattern = sourcePattern; this.namedGroups = namedGroups; this.input = input; + this.callbacks = callbacks; this.bytes = input.getBytes(StandardCharsets.UTF_8); this.charToByte = buildCharToByte(input); this.byteToChar = buildByteToChar(input, bytes.length, charToByte); @@ -297,8 +339,13 @@ public boolean find() { return false; } matcher = regex.matcher(bytes); + if (!callbacks.isEmpty()) { + calloutHandler = new PerlCalloutHandler(input, byteToChar, callbacks); + matcher.setCalloutHandler(calloutHandler); + } int result = matcher.search(charToByte[nextStart], charToByte[regionEnd], Option.NONE); matched = result >= 0; + if (calloutHandler != null) calloutHandler.finish(matched); if (!matched) return false; captures = matcher.getEagerRegion(); int start = start(); @@ -404,4 +451,95 @@ private static int[] buildByteToChar(String input, int byteLength, int[] charToB return offsets; } } + + private static final class PerlCalloutHandler implements CalloutHandler { + private record Token(int localLevel, RegexState regexState, RuntimeScalar previousR, + RuntimeScalar result, boolean block) {} + + private final String input; + private final int[] byteToChar; + private final List callbacks; + private RuntimeScalar completedResult; + + PerlCalloutHandler(String input, int[] byteToChar, List callbacks) { + this.input = input; + this.byteToChar = byteToChar; + this.callbacks = callbacks; + } + + @Override + public CalloutResult execute(int id, MatchView match) { + RuntimeRegexCallback callback = callbacks.get(id); + int localLevel = DynamicVariableManager.getLocalLevel(); + RegexState savedRegex = new RegexState(); + RuntimeScalar rVariable = GlobalVariable.getGlobalVariable( + GlobalContext.encodeSpecialVar("R")); + RuntimeScalar previousR = rVariable.clone(); + publishProvisional(match); + + RuntimeScalar result = RuntimeCode.apply(new RuntimeScalar(callback.code), + new RuntimeArray(), RuntimeContextType.SCALAR).scalar(); + boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK; + if (block) rVariable.set(result); + Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block); + return callback.kind == RuntimeRegexCallback.Kind.CONDITION && !result.getBoolean() + ? CalloutResult.failWith(token) : CalloutResult.continueWith(token); + } + + @Override + public void unwind(Object value) { + restore((Token) value, false); + } + + @Override + public void complete(Object value) { + restore((Token) value, true); + } + + void finish(boolean matched) { + if (matched && completedResult != null) { + GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) + .set(completedResult); + } + } + + private void restore(Token token, boolean completed) { + DynamicVariableManager.popToLocalLevel(token.localLevel()); + token.regexState().restore(); + GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) + .set(token.previousR()); + if (completed && token.block() && completedResult == null) { + completedResult = token.result(); + } + } + + private void publishProvisional(MatchView match) { + RuntimeRegexState state = PerlRuntime.current().regexState; + int count = match.captureCount(); + state.globalMatchString = input; + state.lastMatchedString = input; + state.lastMatchStart = charOffset(match.captureBegin(0)); + state.lastMatchEnd = charOffset(match.captureEnd(0)); + state.lastCaptureGroups = new String[count]; + state.manualCaptureStarts = new int[count]; + state.manualCaptureEnds = new int[count]; + for (int group = 1; group <= count; group++) { + int begin = charOffset(match.captureBegin(group)); + int end = charOffset(match.captureEnd(group)); + if (begin < 0 || end < begin) { + state.manualCaptureStarts[group - 1] = -1; + state.manualCaptureEnds[group - 1] = -1; + state.lastCaptureGroups[group - 1] = null; + } else { + state.manualCaptureStarts[group - 1] = begin; + state.manualCaptureEnds[group - 1] = end; + state.lastCaptureGroups[group - 1] = input.substring(begin, end); + } + } + } + + private int charOffset(int byteOffset) { + return byteOffset < 0 || byteOffset >= byteToChar.length ? -1 : byteToChar[byteOffset]; + } + } } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 742dc53e7..6c3b32657 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -35,6 +35,23 @@ */ public class RuntimeRegex extends RuntimeBase implements RuntimeScalarReference { + /** + * Keep a literal target's SV stable across repeated execution of one regex call site. + * Perl stores pos() on the literal SV in the compiled optree; rematerializing it on + * every loop iteration would restart /g at zero forever. + */ + public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int callsiteId) { + RuntimeRegexState state = PerlRuntime.current().regexState; + RuntimeScalar stable = state.literalRegexTargets.get(callsiteId); + if (stable == null) { + stable = new RuntimeScalar(literal.toString()); + stable.type = literal.type; + stable.tainted = literal.isTainted(); + state.literalRegexTargets.put(callsiteId, stable); + } + return stable; + } + /** Private AST/runtime modifier markers; removed before Perl modifier parsing. */ public static final char INTERNAL_DEBUG_MARKER = '\u0001'; public static final char INTERNAL_DEBUGCOLOR_MARKER = '\u0002'; @@ -143,6 +160,7 @@ private static RuntimeRegexState state() { Pattern notemptyPattern; Pattern notemptyPatternUnicode; JoniRegexPattern recursivePattern; + List executableCallbacks = List.of(); int[] branchResetCaptureMap; int patternFlags; int patternFlagsUnicode; @@ -173,7 +191,7 @@ private static RuntimeRegexState state() { // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site. private int lexicalDebugMode; private static final String DYNAMIC_PATTERN_ERROR = - "\u0000(??{...}) recursive/dynamic regex patterns not implemented"; + "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)"; public RuntimeRegex() { this.regexFlags = null; @@ -195,6 +213,7 @@ public RuntimeRegex cloneTracked() { copy.notemptyPattern = this.notemptyPattern; copy.notemptyPatternUnicode = this.notemptyPatternUnicode; copy.recursivePattern = this.recursivePattern; + copy.executableCallbacks = this.executableCallbacks; copy.branchResetCaptureMap = this.branchResetCaptureMap; copy.patternFlags = this.patternFlags; copy.patternFlagsUnicode = this.patternFlagsUnicode; @@ -421,12 +440,15 @@ private static synchronized RuntimeRegex compileSynchronized( GlobalVariable.getGlobalHash("main::ENV") .get("JPERL_UNIMPLEMENTED").toString()); boolean hasDeferredDynamicPattern = hasDynamicPattern && !warnOnUnimplemented; - if (hasDeferredDynamicPattern) { - // Perl permits qr// construction before the dynamic callback is - // needed. Compile a never-matching placeholder and retain a hard - // error for the first use instead of rejecting module loading. + boolean hasWarnDynamicFallback = hasDynamicPattern && warnOnUnimplemented; + if (hasDeferredDynamicPattern || hasWarnDynamicFallback) { + // Perl permits qr// construction before the dynamic callback is needed. + // Default mode keeps a never-matching placeholder and reports the hard + // error on use. Warn mode retains the historical compatibility fallback + // that ignores the unsupported dynamic component after warning. compilePatternString = compilePatternString.replace( - RegexMarkers.RECURSIVE_PATTERN, "(?!)"); + RegexMarkers.RECURSIVE_PATTERN, + hasWarnDynamicFallback ? "(?:)" : "(?!)"); } List quoteMetaWarningsOnUse = new ArrayList<>(); if (compilePatternString != null && compilePatternString.contains("\\Q")) { @@ -471,7 +493,7 @@ private static synchronized RuntimeRegex compileSynchronized( String javaPattern = null; try { - boolean usesRecursiveBackend = JoniRegexPattern.requiresRecursiveBackend(compilePatternString); + boolean usesRecursiveBackend = JoniRegexPattern.requiresJoniBackend(compilePatternString); if (usesRecursiveBackend && compilePatternString.contains("(?&") && (compilePatternString.contains("(?<=") @@ -497,6 +519,9 @@ private static synchronized RuntimeRegex compileSynchronized( regex.warningsOnUse = new ArrayList<>(quoteMetaWarningsOnUse); if (hasDeferredDynamicPattern) { regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); + } else if (hasWarnDynamicFallback) { + regex.warningsOnUse.add( + "(??{...}) recursive/dynamic regex patterns not implemented\n"); } if (usesRecursiveBackend) { regex.recursivePattern = new JoniRegexPattern(compilePatternString, regex.regexFlags); @@ -1123,6 +1148,12 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS validateTaintedPatternSecurity(patternString); + if (patternString.value instanceof RuntimeRegexTemplate template) { + RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode).cloneTracked(); + regex.executableCallbacks = template.callbacks(); + return new RuntimeScalar(regex).propagateTaint(patternString); + } + // Check if patternString is already a compiled regex if (patternString.type == RuntimeScalarType.REGEX) { RuntimeRegex originalRegex = (RuntimeRegex) patternString.value; @@ -1137,6 +1168,7 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS regex.pattern = originalRegex.pattern; regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; + regex.executableCallbacks = originalRegex.executableCallbacks; regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1175,6 +1207,7 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS regex.pattern = originalRegex.pattern; regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; + regex.executableCallbacks = originalRegex.executableCallbacks; regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1298,6 +1331,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.pattern = resolvedRegex.pattern; regex.patternUnicode = resolvedRegex.patternUnicode; regex.recursivePattern = resolvedRegex.recursivePattern; + regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.branchResetCaptureMap = resolvedRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = resolvedRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = resolvedRegex.patternUnicodeNoInternalMarkers; @@ -1335,6 +1369,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; + regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.branchResetCaptureMap = recompiledRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = recompiledRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiledRegex.patternUnicodeNoInternalMarkers; @@ -1549,6 +1584,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc tempRegex.pattern = pattern; tempRegex.patternUnicode = regexState.lastSuccessfulPattern.patternUnicode; tempRegex.recursivePattern = regexState.lastSuccessfulPattern.recursivePattern; + tempRegex.executableCallbacks = regexState.lastSuccessfulPattern.executableCallbacks; tempRegex.branchResetCaptureMap = regexState.lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = regexState.lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = regexState.lastSuccessfulPattern.patternUnicodeNoInternalMarkers; @@ -1594,7 +1630,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc CharSequence matchInput = new RegexTimeoutCharSequence(inputStr); RegexMatcher matcher; if (regex.recursivePattern != null) { - matcher = regex.recursivePattern.matcher(inputStr); + matcher = regex.recursivePattern.matcher(inputStr, regex.executableCallbacks); } else { Pattern pattern = regex.selectPattern(string, inputStr); @@ -2400,6 +2436,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar tempRegex.pattern = pattern; tempRegex.patternUnicode = state().lastSuccessfulPattern.patternUnicode; tempRegex.recursivePattern = state().lastSuccessfulPattern.recursivePattern; + tempRegex.executableCallbacks = state().lastSuccessfulPattern.executableCallbacks; tempRegex.branchResetCaptureMap = state().lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = state().lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = state().lastSuccessfulPattern.patternUnicodeNoInternalMarkers; @@ -2432,7 +2469,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar Pattern pattern = null; RegexMatcher matcher; if (regex.recursivePattern != null) { - matcher = regex.recursivePattern.matcher(inputStr); + matcher = regex.recursivePattern.matcher(inputStr, regex.executableCallbacks); } else { pattern = regex.selectPattern(inputValue, inputStr); if (inputStr.isEmpty() && (pattern.flags() & Pattern.MULTILINE) != 0) { @@ -2980,6 +3017,12 @@ private static ResolvedRegex resolveRegexWithOrigin(RuntimeScalar quotedRegex) { // Unwrap readonly scalar if (quotedRegex.type == RuntimeScalarType.READONLY_SCALAR) quotedRegex = (RuntimeScalar) quotedRegex.value; + if (quotedRegex.value instanceof RuntimeRegexTemplate) { + RuntimeScalar compiled = getQuotedRegex( + quotedRegex, RuntimeScalarCache.scalarEmptyString); + return new ResolvedRegex((RuntimeRegex) compiled.value, false); + } + if (quotedRegex.type == RuntimeScalarType.REGEX) { return new ResolvedRegex((RuntimeRegex) quotedRegex.value, true); } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java new file mode 100644 index 000000000..bff85141c --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexCallback.java @@ -0,0 +1,24 @@ +package org.perlonjava.runtime.regex; + +import org.perlonjava.runtime.runtimetypes.RuntimeCode; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; + +/** A parser-created executable segment in a regex template. */ +public final class RuntimeRegexCallback { + public enum Kind { BLOCK, CONDITION } + + final RuntimeCode code; + final Kind kind; + + private RuntimeRegexCallback(RuntimeCode code, Kind kind) { + this.code = code; + this.kind = kind; + } + + public static RuntimeScalar wrap(RuntimeScalar codeRef, String kindName) { + if (!(codeRef.value instanceof RuntimeCode code)) { + throw new IllegalArgumentException("regex callback is not a code reference"); + } + return new RuntimeScalar(new RuntimeRegexCallback(code, Kind.valueOf(kindName))); + } +} diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java new file mode 100644 index 000000000..0b9d56cb1 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java @@ -0,0 +1,92 @@ +package org.perlonjava.runtime.regex; + +import org.perlonjava.runtime.runtimetypes.RuntimeBase; +import org.perlonjava.runtime.runtimetypes.RuntimeList; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Runtime interpolation result that keeps executable regex callbacks out of strings. */ +public final class RuntimeRegexTemplate { + private static final Pattern CALLOUT_ID = Pattern.compile("\\(\\?\\{=CALL:(\\d+)\\}\\)"); + private final String pattern; + private final List callbacks; + + private RuntimeRegexTemplate(String pattern, List callbacks) { + this.pattern = pattern; + this.callbacks = List.copyOf(callbacks); + } + + public static RuntimeScalar build(RuntimeList parts) { + if (parts.elements.size() == 1) { + RuntimeScalar only = parts.elements.getFirst().scalar(); + // A lone interpolation must retain its runtime type so qr + // overloading and an already-compiled regex remain observable. + // Only a parser-created callback needs a new template skeleton. + if (!(only.value instanceof RuntimeRegexCallback)) return only; + } + StringBuilder pattern = new StringBuilder(); + List callbacks = new ArrayList<>(); + boolean tainted = false; + for (RuntimeBase part : parts.elements) { + RuntimeScalar scalar = part.scalar(); + tainted |= scalar.isTainted(); + if (scalar.value instanceof RuntimeRegexCallback callback) { + int id = callbacks.size(); + callbacks.add(callback); + if (callback.kind == RuntimeRegexCallback.Kind.CONDITION) { + pattern.append("?{=CALL:").append(id).append("})"); + } else { + pattern.append("(?{=CALL:").append(id).append("})"); + } + } else if (scalar.value instanceof RuntimeRegex regex + && !regex.executableCallbacks.isEmpty()) { + appendEmbeddedRegex(pattern, callbacks, regex.toString(), regex.executableCallbacks); + } else if (scalar.value instanceof RuntimeRegexTemplate template) { + appendEmbeddedRegex(pattern, callbacks, template.pattern, template.callbacks); + } else { + pattern.append(scalar); + } + } + RuntimeScalar result = callbacks.isEmpty() + ? new RuntimeScalar(pattern.toString()) + : new RuntimeScalar(new RuntimeRegexTemplate(pattern.toString(), callbacks)); + result.tainted = tainted; + return result; + } + + private static void appendEmbeddedRegex(StringBuilder pattern, + List callbacks, + String embeddedPattern, + List embeddedCallbacks) { + int offset = callbacks.size(); + Matcher matcher = CALLOUT_ID.matcher(embeddedPattern); + StringBuilder remapped = new StringBuilder(); + while (matcher.find()) { + int oldId = Integer.parseInt(matcher.group(1)); + if (oldId < 0 || oldId >= embeddedCallbacks.size()) { + throw new IllegalArgumentException("Invalid embedded regex callout ID " + oldId); + } + matcher.appendReplacement(remapped, "(?{=CALL:" + (offset + oldId) + "})"); + } + matcher.appendTail(remapped); + pattern.append(remapped); + callbacks.addAll(embeddedCallbacks); + } + + String pattern() { + return pattern; + } + + List callbacks() { + return callbacks; + } + + @Override + public String toString() { + return pattern; + } +} diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java index df5e6f739..1fef6b84d 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeGlob.java @@ -1444,8 +1444,17 @@ public RuntimeGlob undefine() { // Undefine FORMAT GlobalVariable.getGlobalFormatRef(this.globName).undefineFormat(); - // Undefine SCALAR - GlobalVariable.getGlobalVariable(this.globName).set(new RuntimeScalar()); + // Release a mutable scalar's current value first so tracked objects run + // DESTROY immediately. A typeglob scalar slot can also directly alias + // a read-only constant (for example after `*a = "a"`); that referent + // cannot and need not be cleared. + RuntimeScalar oldScalar = GlobalVariable.globalVariables.get(this.globName); + if (oldScalar != null && !(oldScalar instanceof RuntimeScalarReadOnly)) { + oldScalar.undefine(); + } + // Replace the slot in either case. Undefining the glob severs aliases; + // it must not leave a read-only constant installed as the glob's SV. + GlobalVariable.aliasGlobalVariable(this.globName, new RuntimeScalar()); // Undefine ARRAY - clear the existing array (fires DESTROY on blessed // elements via MortalList) before replacing with an empty one. Just diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java index 9cabdebdf..e96fedb0f 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java @@ -40,6 +40,8 @@ public final class RuntimeRegexState { /** Per-runtime callsite state for {@code /o} and {@code m?PAT?}. */ public final Map optimizedRegexCache = new LinkedHashMap<>(); + /** Stable scalar identities for literal regex targets, keyed by compiled call site. */ + public final Map literalRegexTargets = new LinkedHashMap<>(); public final Map userUnicodePropertyCache = new LinkedHashMap<>(); /** diff --git a/src/test/resources/unit/regex/executable_callbacks.t b/src/test/resources/unit/regex/executable_callbacks.t new file mode 100644 index 000000000..217a6f8cf --- /dev/null +++ b/src/test/resources/unit/regex/executable_callbacks.t @@ -0,0 +1,56 @@ +use strict; +use warnings; +use Test::More; + +our $localized = 'outer'; +my $executions = 0; +my @seen; +my $rx = qr/(a)(?{ + ++$executions; + push @seen, "$1:$localized"; + local $localized = 'inside'; + 17; +})(b|c)/; + +is($executions, 0, 'plain callback is not executed while qr is constructed'); +ok('ac' =~ $rx, 'plain callback pattern matches'); +is($executions, 1, 'plain callback executes at match time'); +is_deeply(\@seen, ['a:outer'], 'callback sees provisional captures and lexical state'); +is($^R, 17, 'last successful plain callback publishes $^R'); +is($localized, 'outer', 'callback local scope unwinds after a successful match'); +my $embedded = qr/(?:)$rx(?:)/; +is($executions, 1, 'embedding a callback regex does not execute its closure'); +ok('ab' =~ $embedded, 'embedded callback regex matches'); +is($executions, 2, 'embedded regex retains its callback table'); + +my $literal_call_id = qr/CALL:999(?{ ++$executions })/; +my $embedded_literal_call_id = qr/$literal_call_id/; +ok('CALL:999' =~ $embedded_literal_call_id, + 'single-segment embedding remaps only internal callout markers'); +is($executions, 3, 'literal callout-like text does not corrupt the callback table'); + +@seen = (); +my $backtracking = qr/ + a(?{ local $localized = 'branch'; push @seen, 'first'; 11 })b + | + a(?{ push @seen, $localized; 23 })c +/x; +ok('ac' =~ $backtracking, 'engine backtracks across callbacks'); +is_deeply(\@seen, ['first', 'outer'], 'backtracking restores callback local scope'); +is($^R, 23, 'backtracked callback result does not replace successful $^R'); + +my $condition_calls = 0; +ok('ab' =~ /a(?(?{ ++$condition_calls; 1 })b|c)/, + 'true callback condition selects yes branch'); +ok('ac' =~ /a(?(?{ ++$condition_calls; 0 })b|c)/, + 'false callback condition selects no branch'); +is($condition_calls, 2, 'each callback condition executes once'); + +my $literal_iterations = 0; +while ("\x{100}bc" =~ /(..?)(?{ $^N })/g) { + ++$literal_iterations; + last if $literal_iterations > 2; +} +is($literal_iterations, 2, 'literal target preserves /g position across callback matches'); + +done_testing(); diff --git a/third_party/joni/.github/workflows/maven.yml b/third_party/joni/.github/workflows/maven.yml new file mode 100644 index 000000000..8e2ae53d9 --- /dev/null +++ b/third_party/joni/.github/workflows/maven.yml @@ -0,0 +1,54 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: +# - https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-java-with-maven +# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions +name: Java CI with Maven + +on: + push: + branches-ignore: # build all branches except: + - 'dependabot/**' # prevent GHA triggered twice (once for commit to the branch and once for opening/syncing the PR) + tags-ignore: # don't build tags + - '**' + paths-ignore: + - 'Jenkinsfile' + - 'LICENSE' + - '**/*.md' + - '.git*' + - '.github/*.yml' + pull_request: + paths-ignore: + - 'Jenkinsfile' + - 'LICENSE' + - '**/*.md' + - '.git*' + - '.github/*.yml' + workflow_dispatch: + # https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/ + + +defaults: + run: + shell: bash + + +jobs: + build: + runs-on: ubuntu-latest + + env: + JAVA_VERSION: 11 + + steps: + - name: Git Checkout + uses: actions/checkout@v4 # https://github.com/actions/checkout + + - name: Set up JDK ${{ env.JAVA_VERSION }} ☕ + uses: actions/setup-java@v4 # https://github.com/actions/setup-java + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + cache: maven + + - name: Build with Maven 🔨 + run: mvn -ntp -B verify diff --git a/third_party/joni/.gitignore b/third_party/joni/.gitignore new file mode 100644 index 000000000..0bf6ba832 --- /dev/null +++ b/third_party/joni/.gitignore @@ -0,0 +1,2 @@ +target +.tool-versions diff --git a/third_party/joni/.mvn/wrapper/maven-wrapper.properties b/third_party/joni/.mvn/wrapper/maven-wrapper.properties new file mode 100755 index 000000000..d58dfb70b --- /dev/null +++ b/third_party/joni/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/third_party/joni/.travis.yml b/third_party/joni/.travis.yml new file mode 100644 index 000000000..ff3b50095 --- /dev/null +++ b/third_party/joni/.travis.yml @@ -0,0 +1,13 @@ +language: java + +jdk: +- openjdk9 + +arch: + - amd64 + - ppc64le + + +cache: + directories: + - $HOME/.m2 diff --git a/third_party/joni/Jenkinsfile b/third_party/joni/Jenkinsfile new file mode 100644 index 000000000..dc7d1d478 --- /dev/null +++ b/third_party/joni/Jenkinsfile @@ -0,0 +1,71 @@ +#!/usr/bin/env groovy + +pipeline { + agent none + options { + buildDiscarder(logRotator(numToKeepStr: '10')) + timeout(time: 1, unit: 'HOURS') + } + stages { + stage('OpenJDK 8') { + agent { docker 'openjdk:8-jdk' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + + stage('Alternative Platforms') { + parallel { + stage('OpenJDK 9') { + agent { docker 'openjdk:9-jdk' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + stage('Alpine Linux') { + agent { docker 'openjdk:8-jdk-alpine' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + stage('FreeBSD 11') { + agent { label 'freebsd' } + steps { + checkout scm + sh './mvnw test -B' + } + post { + always { + junit testResults: '**/surefire-reports/**/*.xml', allowEmptyResults: true + } + } + } + /* awaiting platform support in Code Valet */ + stage('Windows 2016') { + when { branch 'windows-support' } + steps { + echo 'Not yet available' + } + } + } + } + } +} diff --git a/third_party/joni/LICENSE b/third_party/joni/LICENSE new file mode 100644 index 000000000..9341f05ee --- /dev/null +++ b/third_party/joni/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 JRuby Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/joni/PERLONJAVA-NOTICE.md b/third_party/joni/PERLONJAVA-NOTICE.md new file mode 100644 index 000000000..a708cc8ef --- /dev/null +++ b/third_party/joni/PERLONJAVA-NOTICE.md @@ -0,0 +1,11 @@ +# Modified Joni distribution + +This directory contains Joni, the Java port of Oniguruma maintained by the +JRuby project, with generic match-time callout extensions maintained by the +PerlOnJava project. + +Joni remains licensed under its original MIT License. All upstream copyright, +license, and authorship notices are retained. PerlOnJava's modifications do not +replace or narrow those notices. + +Upstream project: https://github.com/jruby/joni diff --git a/third_party/joni/README.md b/third_party/joni/README.md new file mode 100644 index 000000000..b8e784bea --- /dev/null +++ b/third_party/joni/README.md @@ -0,0 +1,67 @@ +joni +==== + +[![Maven Central](https://img.shields.io/maven-central/v/org.jruby.joni/joni)](https://central.sonatype.com/artifact/org.jruby.joni/joni) +[![Build Status](https://github.com/jruby/joni/actions/workflows/maven.yml/badge.svg)](https://github.com/jruby/joni/actions/workflows/maven.yml) + +Java port of [Oniguruma](https://github.com/kkos/oniguruma) regexp library + +## Usage + +### Imports + +```java +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +``` + +### Matching + +```java +byte[] pattern = "a*".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +``` + +### Using captures + +```java +byte[] pattern = "(a*)".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +if (result != -1) { + Region region = matcher.getEagerRegion(); +} +``` + +### Using named captures + +```java +byte[] pattern = "(?a*)".getBytes(); +byte[] str = "aaa".getBytes(); + +Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); +Matcher matcher = regex.matcher(str); +int result = matcher.search(0, str.length, Option.DEFAULT); +if (result != -1) { + Region region = matcher.getEagerRegion(); + for (Iterator entry = regex.namedBackrefIterator(); entry.hasNext();) { + NameEntry e = entry.next(); + int number = e.getBackRefs()[0]; // can have many refs per name + // int begin = region.beg[number]; + // int end = region.end[number]; + } +} +``` + +## License + +Joni is released under the [MIT License](http://www.opensource.org/licenses/MIT). diff --git a/third_party/joni/mvnw b/third_party/joni/mvnw new file mode 100755 index 000000000..19529ddf8 --- /dev/null +++ b/third_party/joni/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/third_party/joni/mvnw.cmd b/third_party/joni/mvnw.cmd new file mode 100755 index 000000000..8dfb09e8d --- /dev/null +++ b/third_party/joni/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/third_party/joni/pom.xml b/third_party/joni/pom.xml new file mode 100644 index 000000000..e89cafa3d --- /dev/null +++ b/third_party/joni/pom.xml @@ -0,0 +1,287 @@ + + + 4.0.0 + org.jruby.joni + joni + jar + 2.2.7 + Joni + + Java port of Oniguruma: http://www.geocities.jp/kosako3/oniguruma + that uses byte arrays directly instead of java Strings and chars + + https://github.com/jruby/joni + + + UTF-8 + + + + GitHub + https://github.com/jruby/joni/issues + + + + scm:git:https://github.com/jruby/joni.git + scm:git:git@github.com:jruby/joni.git + https://github.com/jruby/joni + + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + + lopex + Marcin Mielzynski + lopx@gazeta.pl + + + + + + org.jruby.jcodings + jcodings + 1.0.64 + + + junit + junit + 4.13.2 + test + + + + + src + test + joni + + + org.apache.maven.wagon + wagon-webdav + + + + + maven-compiler-plugin + 3.13.0 + + 1.8 + 1.8 + + + + default-compile + + + module-info.java + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + maven-jar-plugin + 3.4.2 + + + + true + true + + + Joni (Java port of Oniguruma) + Joni (Java port of Oniguruma) + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.7.0 + true + + central + + + + + + + jacoco + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + + + + + default-prepare-agent + + prepare-agent + + + + pre-unit-test + + prepare-agent + + + ${project.build.directory}/coverage-reports/jacoco.exec + surefireArgLine + + + + post-unit-test + test + + report + + + ${project.build.directory}/coverage-reports/jacoco.exec + ${project.reporting.outputDirectory}/jacoco + + + + default-report + prepare-package + + report + + + + + + + + + + release-on-9 + + [9,) + + + + + maven-compiler-plugin + + + compile9 + + compile + + + 9 + + module-info.java + + + + + + + maven-source-plugin + 3.3.1 + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + 3.8.0 + + + attach-javadocs + + jar + + + + + -Xdoclint:none + -html5 + true + false + + + + + + + release + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + maven-gpg-plugin + 3.2.4 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + + true + + + + diff --git a/third_party/joni/src/module-info.java b/third_party/joni/src/module-info.java new file mode 100644 index 000000000..a8b9e8683 --- /dev/null +++ b/third_party/joni/src/module-info.java @@ -0,0 +1,7 @@ +open module org.jruby.joni { + exports org.joni; + exports org.joni.constants; + exports org.joni.exception; + + requires transitive org.jruby.jcodings; +} \ No newline at end of file diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java new file mode 100644 index 000000000..fe61398a4 --- /dev/null +++ b/third_party/joni/src/org/joni/Analyser.java @@ -0,0 +1,2374 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAll; +import static org.joni.BitStatus.bsAt; +import static org.joni.BitStatus.bsClear; +import static org.joni.BitStatus.bsOnAt; +import static org.joni.BitStatus.bsOnAtSimple; +import static org.joni.Option.isCaptureGroup; +import static org.joni.Option.isFindCondition; +import static org.joni.Option.isIgnoreCase; +import static org.joni.Option.isMultiline; +import static org.joni.ast.ListNode.newAlt; +import static org.joni.ast.ListNode.newList; +import static org.joni.ast.QuantifierNode.isRepeatInfinite; + +import java.util.IllegalFormatConversionException; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.ObjPtr; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.joni.ast.AnchorNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.ListNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.EncloseType; +import org.joni.constants.internal.NodeType; +import org.joni.constants.internal.StackPopLevel; +import org.joni.constants.internal.TargetInfo; + +final class Analyser extends Parser { + + protected Analyser(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex, syntax, bytes, p, end, warnings); + } + + protected final void compile() { + if (Config.DEBUG) Config.log.println(encStringToString(bytes, getBegin(), getEnd())); + reset(); + + regex.numMem = 0; + regex.numRepeat = 0; + regex.numNullCheck = 0; + //regex.repeatRangeAlloc = 0; + regex.repeatRangeLo = null; + regex.repeatRangeHi = null; + regex.numCombExpCheck = 0; + + if (Config.USE_CEC) regex.numCombExpCheck = 0; + + + Node root = parseRegexp(); // onig_parse_make_tree + regex.numMem = env.numMem; + + if (Config.USE_NAMED_GROUP) { + /* mixed use named group and no-named group */ + if (env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(regex.options)) { + if (env.numNamed != env.numMem) { + root = disableNoNameGroupCapture(root); + } else { + numberedRefCheck(root); + } + } + } // USE_NAMED_GROUP + + if (Config.USE_NAMED_GROUP) { + if (env.numCall > 0) { + env.unsetAddrList = new UnsetAddrList(env.numCall); + setupSubExpCall(root); + // r != 0 ??? + subexpRecursiveCheckTrav(root); + // r < 0 -< err, FOUND_CALLED_NODE = 1 + subexpInfRecursiveCheckTrav(root); + // r != 0 recursion infinite ??? + regex.numCall = env.numCall; + } else { + regex.numCall = 0; + } + } // USE_NAMED_GROUP + + if (Config.DEBUG_PARSE_TREE && Config.DEBUG_PARSE_TREE_RAW) Config.log.println("\n" + root + "\n"); + + Node.TopNode top = Node.newTop(root); + setupTree(root, 0); + root = top.getRoot(); + + if (Config.DEBUG_PARSE_TREE) Config.log.println("\n" + root + "\n"); + + regex.captureHistory = env.captureHistory; + regex.btMemStart = env.btMemStart; + + if (isFindCondition(regex.options)) { + regex.btMemEnd = bsAll(); + } else { + regex.btMemEnd = env.btMemEnd; + regex.btMemEnd |= regex.captureHistory; + } + + if (Config.USE_CEC) { + if (env.backrefedMem == 0 || (Config.USE_SUBEXP_CALL && env.numCall == 0)) { + setupCombExpCheck(root, 0); + + if (Config.USE_SUBEXP_CALL && env.hasRecursion) { + env.numCombExpCheck = 0; + } else { // USE_SUBEXP_CALL + if (env.combExpMaxRegNum > 0) { + for (int i=1; i 1) { + int p_ = p; + while (p_ < end) { + int code = enc.mbcToCode(bytes, p_, end); + if (code >= 0x80) { + try { + sb.append(String.format(" 0x%04x ", code)); + } catch (IllegalFormatConversionException ifce) { + sb.append(code); + } + } else { + sb.append((char)code); + } + p_ += enc.length(bytes, p_, end); + } + } else { + while (p < end) { + sb.append(new String(bytes, p, 1)); + p++; + } + } + return sb.append("/").toString(); + } + + private void noNameDisableMapFor_listAlt(Node node, int[]map, Ptr counter) { + ListNode can = (ListNode)node; + do { + can.setValue(noNameDisableMap(can.value, map, counter)); + } while ((can = can.tail) != null); + } + + private void noNameDisableMapFor_quantifier(Node node, int[]map, Ptr counter) { + QuantifierNode qn = (QuantifierNode)node; + Node target = qn.target; + Node old = target; + target = noNameDisableMap(target, map, counter); + + if (target != old) { + qn.setTarget(target); + if (target.getType() == NodeType.QTFR) qn.reduceNestedQuantifier((QuantifierNode)target); + } + } + + private Node noNameDisableMapFor_enclose(Node node, int[]map, Ptr counter) { + EncloseNode en = (EncloseNode)node; + if (en.type == EncloseType.MEMORY) { + if (en.isNamedGroup()) { + counter.p++; + map[en.regNum] = counter.p; + en.regNum = counter.p; + en.setTarget(noNameDisableMap(en.target, map, counter)); + } else { + node = en.target; + en.target = null; // remove first enclose: /(a)(?c)/ + node = noNameDisableMap(node, map, counter); + } + } else { + en.setTarget(noNameDisableMap(en.target, map, counter)); + } + return node; + } + + private void noNameDisableMapFor_anchor(Node node, int[]map, Ptr counter) { + AnchorNode an = (AnchorNode)node; + if (an.target != null) an.setTarget(noNameDisableMap(an.target, map, counter)); + } + + private Node noNameDisableMap(Node node, int[]map, Ptr counter) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + noNameDisableMapFor_listAlt(node, map, counter); + break; + case NodeType.QTFR: + noNameDisableMapFor_quantifier(node, map, counter); + break; + case NodeType.ENCLOSE: + node = noNameDisableMapFor_enclose(node, map, counter); + break; + case NodeType.ANCHOR: + noNameDisableMapFor_anchor(node, map, counter); + break; + } // switch + return node; + } + + private void renumberByMap(Node node, int[]map) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + renumberByMap(can.value, map); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + renumberByMap(((QuantifierNode)node).target, map); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.type == EncloseType.CONDITION && en.calloutConditionId < 0) { + en.regNum = map[en.regNum]; + } + renumberByMap(en.target, map); + break; + + case NodeType.BREF: + ((BackRefNode)node).renumber(map); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.target != null) renumberByMap(an.target, map); + break; + } // switch + } + + protected final void numberedRefCheck(Node node) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + numberedRefCheck(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + numberedRefCheck(((QuantifierNode)node).target); + break; + + case NodeType.ENCLOSE: + numberedRefCheck(((EncloseNode)node).target); + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (!br.isNameRef()) newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.target != null) numberedRefCheck(an.target); + break; + } // switch + } + + protected final Node disableNoNameGroupCapture(Node root) { + int[]map = new int[env.numMem + 1]; + + root = noNameDisableMap(root, map, new Ptr(0)); + renumberByMap(root, map); + + for (int i=1, pos=1; i<=env.numMem; i++) { + if (map[i] > 0) { + env.memNodes[pos] = env.memNodes[i]; + pos++; + } + } + + int loc = env.captureHistory; + env.captureHistory = bsClear(); + + for (int i=1; i<=Config.MAX_CAPTURE_HISTORY_GROUP; i++) { + if (bsAt(loc, i)) { + env.captureHistory = bsOnAtSimple(env.captureHistory, map[i]); + } + } + + env.numMem = env.numNamed; + regex.numMem = env.numNamed; + + regex.renumberNameTable(map); + + return root; + } + + // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + private int quantifiersMemoryInfo(Node node) { + int info = 0; + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + int v = quantifiersMemoryInfo(can.value); + if (v > info) info = v; + } while ((can = can.tail) != null); + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + return TargetInfo.IS_EMPTY_REC; /* tiny version */ + } else { + info = quantifiersMemoryInfo(cn.target); + } + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.upper != 0) { + info = quantifiersMemoryInfo(qn.target); + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + return TargetInfo.IS_EMPTY_MEM; + + case EncloseType.OPTION: + case EncloseNode.STOP_BACKTRACK: + case EncloseNode.CONDITION: + case EncloseNode.ABSENT: + info = quantifiersMemoryInfo(en.target); + break; + + default: + break; + } // inner switch + break; + + case NodeType.BREF: + case NodeType.STR: + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + case NodeType.ANCHOR: + default: + break; + } // switch + + return info; + } + + private int getMinMatchLength(Node node) { + int min = 0; + + switch (node.getType()) { + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (br.isRecursion()) break; + + if (br.back[0] > env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + min = getMinMatchLength(env.memNodes[br.back[0]]); + } + + for (int i=1; i env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + int tmin = getMinMatchLength(env.memNodes[br.back[i]]); + if (min > tmin) min = tmin; + } + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + EncloseNode en = cn.target; + if (en.isMinFixed()) min = en.minLength; + } else { + min = getMinMatchLength(cn.target); + } + } // USE_SUBEXP_CALL + break; + + case NodeType.LIST: + ListNode can = (ListNode)node; + do { + min += getMinMatchLength(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.ALT: + ListNode y = (ListNode)node; + do { + Node x = y.value; + int tmin = getMinMatchLength(x); + if (y == node) { + min = tmin; + } else if (min > tmin) { + min = tmin; + } + } while ((y = y.tail) != null); + break; + + case NodeType.STR: + min = ((StringNode)node).length(); + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + min = 1; + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.lower > 0) { + min = getMinMatchLength(qn.target); + min = MinMaxLen.distanceMultiply(min, qn.lower); + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isMinFixed()) { + min = en.minLength; + } else { + if (en.isMark1()) { + min = 0; /* recursive */ + } else { + en.setMark1(); + min = getMinMatchLength(en.target); + en.clearMark1(); + en.minLength = min; + en.setMinFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + min = getMinMatchLength(en.target); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + default: + break; + } // switch + + return min; + } + + private int getMaxMatchLength(Node node) { + int max = 0; + + switch (node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + do { + int tmax = getMaxMatchLength(ln.value); + max = MinMaxLen.distanceAdd(max, tmax); + } while ((ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + do { + int tmax = getMaxMatchLength(an.value); + if (max < tmax) max = tmax; + } while ((an = an.tail) != null); + break; + + case NodeType.STR: + max = ((StringNode)node).length(); + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + max = enc.maxLength(); + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + if (br.isRecursion()) { + max = MinMaxLen.INFINITE_DISTANCE; + break; + } + + for (int i=0; i env.numMem) { + if(!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + int tmax = getMaxMatchLength(env.memNodes[br.back[i]]); + if (max < tmax) max = tmax; + } + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (!cn.isRecursion()) { + max = getMaxMatchLength(cn.target); + } else { + max = MinMaxLen.INFINITE_DISTANCE; + } + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.upper != 0) { + max = getMaxMatchLength(qn.target); + if (max != 0) { + if (!isRepeatInfinite(qn.upper)) { + max = MinMaxLen.distanceMultiply(max, qn.upper); + } else { + max = MinMaxLen.INFINITE_DISTANCE; + } + } + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isMaxFixed()) { + max = en.maxLength; + } else { + if (en.isMark1()) { + max = MinMaxLen.INFINITE_DISTANCE; + } else { + en.setMark1(); + max = getMaxMatchLength(en.target); + en.clearMark1(); + en.maxLength = max; + en.setMaxFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + max = getMaxMatchLength(en.target); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + default: + break; + } // switch + + return max; + } + + private static final int GET_CHAR_LEN_VARLEN = -1; + private static final int GET_CHAR_LEN_TOP_ALT_VARLEN = -2; + protected final int getCharLengthTree(Node node) { + return getCharLengthTree(node, 0); + } + + private int getCharLengthTree(Node node, int level) { + level++; + + int len = 0; + returnCode = 0; + + switch(node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + do { + int tlen = getCharLengthTree(ln.value, level); + if (returnCode == 0) len = MinMaxLen.distanceAdd(len, tlen); + } while (returnCode == 0 && (ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + boolean varLen = false; + + int tlen = getCharLengthTree(an.value, level); + while (returnCode == 0 && (an = an.tail) != null) { + int tlen2 = getCharLengthTree(an.value, level); + if (returnCode == 0) { + if (tlen != tlen2) varLen = true; + } + } + + if (returnCode == 0) { + if (varLen) { + if (level == 1) { + returnCode = GET_CHAR_LEN_TOP_ALT_VARLEN; + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + } else { + len = tlen; + } + } + break; + + case NodeType.STR: + StringNode sn = (StringNode)node; + len = sn.length(enc); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + if (qn.lower == qn.upper) { + tlen = getCharLengthTree(qn.target, level); + if (returnCode == 0) len = MinMaxLen.distanceMultiply(tlen, qn.lower); + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (!cn.isRecursion()) { + len = getCharLengthTree(cn.target, level); + } else { + returnCode = GET_CHAR_LEN_VARLEN; + } + } // USE_SUBEXP_CALL + break; + + case NodeType.CTYPE: + case NodeType.CCLASS: + case NodeType.CANY: + len = 1; + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch(en.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL) { + if (en.isCLenFixed()) { + len = en.charLength; + } else { + len = getCharLengthTree(en.target, level); + if (returnCode == 0) { + en.charLength = len; + en.setCLenFixed(); + } + } + } // USE_SUBEXP_CALL + break; + + case EncloseType.OPTION: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + len = getCharLengthTree(en.target, level); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + break; + + default: + returnCode = GET_CHAR_LEN_VARLEN; + } // switch + return len; + } + + /* x is not included y ==> 1 : 0 */ + private boolean isNotIncluded(Node x, Node y) { + Node tmp; + + retry: while(true) { + int yType = y.getType(); + + switch(x.getType()) { + case NodeType.CTYPE: + switch(yType) { + case NodeType.CTYPE: + { + CTypeNode cny = (CTypeNode)y; + CTypeNode cnx = (CTypeNode)x; + return cny.ctype == cnx.ctype && cny.not != cnx.not && cny.asciiRange == cnx.asciiRange; + } + case NodeType.CCLASS: + // !swap:! + tmp = x; + x = y; + y = tmp; + // !goto retry;! + continue retry; + + case NodeType.STR: + // !goto swap;! + tmp = x; + x = y; + y = tmp; + continue retry; + + default: + break; + } // inner switch + break; + + case NodeType.CCLASS: + CClassNode xc = (CClassNode)x; + + switch(yType) { + case NodeType.CTYPE: + { + CTypeNode yc = (CTypeNode)y; + switch(yc.ctype) { + case CharacterType.WORD: + if (!yc.not) { + if (xc.mbuf == null && !xc.isNot()) { + for (int i=0; i ys.length()) len = ys.length(); + if (xs.isAmbig() || ys.isAmbig()) { + /* tiny version */ + return false; + } else { + for (int i=0, p=ys.p, q=xs.p; i 0) { + if (qn.headExact != null) { + n = qn.headExact; + } else { + n = getHeadValueNode(qn.target, exact); + } + } + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + + switch (en.type) { + case EncloseType.OPTION: + int options = regex.options; + regex.options = en.option; + n = getHeadValueNode(en.target, exact); + regex.options = options; + break; + + case EncloseType.MEMORY: + case EncloseType.STOP_BACKTRACK: + case EncloseNode.CONDITION: + n = getHeadValueNode(en.target, exact); + break; + + case EncloseType.ABSENT: + break; + } // inner switch + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if (an.type == AnchorType.PREC_READ) n = getHeadValueNode(an.target, exact); + break; + + default: + break; + } // switch + + return n; + } + + // true: invalid + private boolean checkTypeTree(Node node, int typeMask, int encloseMask, int anchorMask) { + if ((node.getType2Bit() & typeMask) == 0) return true; + + boolean invalid = false; + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + invalid = checkTypeTree(can.value, typeMask, encloseMask, anchorMask); + } while (!invalid && (can = can.tail) != null); + break; + + case NodeType.QTFR: + invalid = checkTypeTree(((QuantifierNode)node).target, typeMask, encloseMask, anchorMask); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if ((en.type & encloseMask) == 0) return true; + invalid = checkTypeTree(en.target, typeMask, encloseMask, anchorMask); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + if ((an.type & anchorMask) == 0) return true; + + if (an.target != null) invalid = checkTypeTree(an.target, typeMask, encloseMask, anchorMask); + break; + + default: + break; + + } // switch + + return invalid; + } + + private static final int RECURSION_EXIST = 1; + private static final int RECURSION_INFINITE = 2; + private int subexpInfRecursiveCheck(Node node, boolean head) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + int min; + ListNode x = (ListNode)node; + do { + int ret = subexpInfRecursiveCheck(x.value, head); + if (ret == RECURSION_INFINITE) return ret; + r |= ret; + if (head) { + min = getMinMatchLength(x.value); + if (min != 0) head = false; + } + } while ((x = x.tail) != null); + break; + + case NodeType.ALT: + ListNode can = (ListNode)node; + r = RECURSION_EXIST; + do { + int ret = subexpInfRecursiveCheck(can.value, head); + if (ret == RECURSION_INFINITE) return ret; + r &= ret; + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + r = subexpInfRecursiveCheck(qn.target, head); + if (r == RECURSION_EXIST) { + if (qn.lower == 0) r = 0; + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpInfRecursiveCheck(an.target, head); + break; + } // inner switch + break; + + case NodeType.CALL: + r = subexpInfRecursiveCheck(((CallNode)node).target, head); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isMark2()) { + return 0; + } else if (en.isMark1()) { + return !head ? RECURSION_EXIST : RECURSION_INFINITE; + // throw exception here ??? + } else { + en.setMark2(); + r = subexpInfRecursiveCheck(en.target, head); + en.clearMark2(); + } + break; + + default: + break; + } // switch + return r; + } + + protected final int subexpInfRecursiveCheckTrav(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + r = subexpInfRecursiveCheckTrav(can.value); + } while (r == 0 && (can = can.tail) != null); + break; + + case NodeType.QTFR: + r = subexpInfRecursiveCheckTrav(((QuantifierNode)node).target); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpInfRecursiveCheckTrav(an.target); + break; + } // inner switch + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isRecursion()) { + en.setMark1(); + r = subexpInfRecursiveCheck(en.target, true); + if (r > 0) newValueException(NEVER_ENDING_RECURSION); + en.clearMark1(); + } + r = subexpInfRecursiveCheckTrav(en.target); + break; + + default: + break; + } // switch + + return r; + } + + private int subexpRecursiveCheck(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + r |= subexpRecursiveCheck(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + r = subexpRecursiveCheck(((QuantifierNode)node).target); + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpRecursiveCheck(an.target); + break; + } // inner switch + break; + + case NodeType.CALL: + CallNode cn = (CallNode)node; + r = subexpRecursiveCheck(cn.target); + if (r != 0) cn.setRecursion(); + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (en.isMark2()) { + return 0; + } else if (en.isMark1()) { + return 1; /* recursion */ + } else { + en.setMark2(); + r = subexpRecursiveCheck(en.target); + en.clearMark2(); + } + break; + + default: + break; + } // switch + + return r; + } + + private static final int FOUND_CALLED_NODE = 1; + protected final int subexpRecursiveCheckTrav(Node node) { + int r = 0; + + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + int ret = subexpRecursiveCheckTrav(can.value); + if (ret == FOUND_CALLED_NODE) { + r = FOUND_CALLED_NODE; + } + // else if (ret < 0) return ret; ??? + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + r = subexpRecursiveCheckTrav(qn.target); + if (qn.upper == 0) { + if (r == FOUND_CALLED_NODE) qn.isRefered = true; + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + r = subexpRecursiveCheckTrav(an.target); + break; + } // inner switch + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + if (!en.isRecursion()) { + if (en.isCalled()) { + en.setMark1(); + r = subexpRecursiveCheck(en.target); + if (r != 0) en.setRecursion(); + en.clearMark1(); + } + } + r = subexpRecursiveCheckTrav(en.target); + if (en.isCalled()) r |= FOUND_CALLED_NODE; + break; + + default: + break; + } // switch + + return r; + } + + private void setCallAttr(CallNode cn) { + EncloseNode en = env.memNodes[cn.groupNum]; + if (en == null) newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + en.setCalled(); + cn.setTarget(en); + env.btMemStart = BitStatus.bsOnAt(env.btMemStart, cn.groupNum); + cn.unsetAddrList = env.unsetAddrList; + } + + protected final void setupSubExpCall(Node node) { + + switch(node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + ListNode can = (ListNode)node; + do { + setupSubExpCall(can.value); + } while ((can = can.tail) != null); + break; + + case NodeType.QTFR: + setupSubExpCall(((QuantifierNode)node).target); + break; + + case NodeType.ENCLOSE: + setupSubExpCall(((EncloseNode)node).target); + break; + + case NodeType.CALL: + CallNode cn = (CallNode)node; + if (cn.groupNum != 0) { + int gNum = cn.groupNum; + + if (Config.USE_NAMED_GROUP) { + if (env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(env.option)) { + newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + } + } // USE_NAMED_GROUP + if (gNum > env.numMem) newValueException(UNDEFINED_GROUP_REFERENCE, cn.nameP, cn.nameEnd); + setCallAttr(cn); + } else { + if (Config.USE_NAMED_GROUP) { + if (Config.USE_PERL_SUBEXP_CALL && cn.nameP == cn.nameEnd) { + setCallAttr(cn); + } else { + NameEntry ne = regex.nameToGroupNumbers(cn.name, cn.nameP, cn.nameEnd); + + if (ne == null) { + newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + } else if (ne.backNum > 1) { + newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); + } else { + cn.groupNum = ne.backRef1; // ne.backNum == 1 ? ne.backRef1 : ne.backRefs[0]; // ??? need to check ? + setCallAttr(cn); + } + } + } + } + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + case AnchorType.PREC_READ_NOT: + case AnchorType.LOOK_BEHIND: + case AnchorType.LOOK_BEHIND_NOT: + setupSubExpCall(an.target); + break; + } + break; + + } // switch + } + + /* divide different length alternatives in look-behind. + (?<=A|B) ==> (?<=A)|(?<=B) + (? (? list */ + } while ((np = ((ListNode)np).tail) != null); + } + + return node; + } + + private Node setupLookBehind(AnchorNode node) { + int len = getCharLengthTree(node.target); + switch(returnCode) { + case 0: + node.charLength = len; + break; + case GET_CHAR_LEN_VARLEN: + newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + break; + case GET_CHAR_LEN_TOP_ALT_VARLEN: + if (syntax.differentLengthAltLookBehind()) { + return divideLookBehindAlternatives(node); + } else { + newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + } + } + return node; + } + + private void nextSetup(Node node, Node nextNode) { + retry: while(true) { + + int type = node.getType(); + if (type == NodeType.QTFR) { + QuantifierNode qn = (QuantifierNode)node; + if (qn.greedy && isRepeatInfinite(qn.upper)) { + if (Config.USE_QTFR_PEEK_NEXT) { + StringNode n = (StringNode)getHeadValueNode(nextNode, true); + /* '\0': for UTF-16BE etc... */ + if (n != null && n.bytes[n.p] != 0) { + qn.nextHeadExact = n; + } + } // USE_QTFR_PEEK_NEXT + + /* automatic possessification a*b ==> (?>a*)b */ + if (qn.lower <= 1) { + if (qn.target.isSimple()) { + Node x = getHeadValueNode(qn.target, false); + if (x != null) { + Node y = getHeadValueNode(nextNode, false); + if (y != null && isNotIncluded(x, y)) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); + en.setStopBtSimpleRepeat(); + node.replaceWith(en); + en.setTarget(node); + } + } + } + } + } + } else if (type == NodeType.ENCLOSE) { + EncloseNode en = (EncloseNode)node; + if (en.isMemory()) { + node = en.target; + continue retry; + } + } + + break; + } // while + } + + private void updateStringNodeCaseFoldSingleByte(StringNode sn, byte[]toLower) { + int end = sn.end; + byte[]bytes = sn.bytes; + int sp = 0; + int p = sn.p; + + while (p < end) { + byte lower = toLower[bytes[p] & 0xff]; + if (lower != bytes[p]) { + byte[]sbuf = new byte[end - sn.p]; + System.arraycopy(bytes, sn.p, sbuf, 0, sp); + + while (p < end) sbuf[sp++] = toLower[bytes[p++] & 0xff]; + + sn.set(sbuf, 0, sp); + break; + } else { + sp++; + p++; + } + } + } + + private void updateStringNodeCaseFoldMultiByte(StringNode sn) { + byte[]bytes = sn.bytes; + int end = sn.end; + value = sn.p; + int sp = 0; + byte[]buf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN]; + + while (value < end) { + int ovalue = value; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, buf); + + for (int i = 0; i < len; i++) { + if (bytes[ovalue + i] != buf[i]) { + + byte[]sbuf = new byte[sn.length() << 1]; + System.arraycopy(bytes, sn.p, sbuf, 0, ovalue - sn.p); + value = ovalue; + while (value < end) { + len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, buf); + for (i = 0; i < len; i++) { + if (sp >= sbuf.length) { + byte[]tmp = new byte[sbuf.length << 1]; + System.arraycopy(sbuf, 0, tmp, 0, sbuf.length); + sbuf = tmp; + } + sbuf[sp++] = buf[i]; + } + } + sn.set(sbuf, 0, sp); + return; + } + } + sp += len; + } + } + + private void updateStringNodeCaseFold(Node node) { + StringNode sn = (StringNode)node; + byte[] toLower = enc.toLowerCaseTable(); + if (toLower != null) { + updateStringNodeCaseFoldSingleByte(sn, toLower); + } else { + updateStringNodeCaseFoldMultiByte(sn); + } + } + + private Node expandCaseFoldMakeRemString(byte[]bytes, int p, int end) { + StringNode node = new StringNode(bytes, p, end); + + updateStringNodeCaseFold(node); + node.setAmbig(); + node.setDontGetOptInfo(); + return node; + } + + private boolean isCaseFoldVariableLength(int itemNum, CaseFoldCodeItem[] items, int slen) { + for(int i = 0; i < itemNum; i++) { + if (items[i].byteLen != slen || items[i].code.length != 1) return true; + } + return false; + } + + private boolean expandCaseFoldStringAlt(int itemNum, CaseFoldCodeItem[]items, + byte[]bytes, int p, int slen, int end, ObjPtr node) { + boolean varlen = false; + for (int i=0; i prevNode = new ObjPtr<>(); + StringNode stringNode = null; + + while (p < end) { + CaseFoldCodeItem[]items = enc.caseFoldCodesByString(regex.caseFoldFlag, bytes, p, end); + int len = enc.length(bytes, p, end); + + if (items.length == 0 || !isCaseFoldVariableLength(items.length, items, len)) { + if (stringNode == null) { + if (root == null && prevNode.p != null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + prevNode.p = stringNode = new StringNode(); // onig_node_new_str(NULL, NULL); + + if (root != null) ListNode.listAdd(root, stringNode); + + } + + stringNode.catBytes(bytes, p, p + len); + } else { + altNum *= (items.length + 1); + if (altNum > THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) break; + if (stringNode != null) { + updateStringNodeCaseFold(stringNode); + stringNode.setAmbig(); + } + + if (root == null && prevNode.p != null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + if (expandCaseFoldStringAlt(items.length, items, bytes, p, len, end, prevNode)) { // if (r == 1) + if (root == null) { + topRoot = (ListNode)prevNode.p; + } else { + ListNode.listAdd(root, prevNode.p); + } + + root = (ListNode)((ListNode)prevNode.p).value; + } else { /* r == 0 */ + if (root != null) ListNode.listAdd(root, prevNode.p); + } + stringNode = null; + } + p += len; + } + + if (stringNode != null) { + updateStringNodeCaseFold(stringNode); + stringNode.setAmbig(); + } + + if (p < end) { + Node srem = expandCaseFoldMakeRemString(bytes, p, end); + + if (prevNode.p != null && root == null) { + topRoot = root = ListNode.listAdd(null, prevNode.p); + } + + if (root == null) { + prevNode.p = srem; + } else { + ListNode.listAdd(root, srem); + } + } + /* ending */ + Node xnode = topRoot != null ? topRoot : prevNode.p; + + node.replaceWith(xnode); + return xnode; + } + + private static final int CEC_THRES_NUM_BIG_REPEAT = 512; + private static final int CEC_INFINITE_NUM = 0x7fffffff; + + private static final int CEC_IN_INFINITE_REPEAT = (1<<0); + private static final int CEC_IN_FINITE_REPEAT = (1<<1); + private static final int CEC_CONT_BIG_REPEAT = (1<<2); + + protected final int setupCombExpCheck(Node node, int state) { + int r = state; + int ret; + + switch (node.getType()) { + case NodeType.LIST: + ListNode ln = (ListNode)node; + + do { + r = setupCombExpCheck(ln.value, r); + //prev = ((ConsAltNode)node).value; + } while (r >= 0 && (ln = ln.tail) != null); + break; + + case NodeType.ALT: + ListNode an = (ListNode)node; + do { + ret = setupCombExpCheck(an.value, state); + r |= ret; + } while (ret >= 0 && (an = an.tail) != null); + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + int childState = state; + int addState = 0; + int varNum; + + if (!isRepeatInfinite(qn.upper)) { + if (qn.upper > 1) { + /* {0,1}, {1,1} are allowed */ + childState |= CEC_IN_FINITE_REPEAT; + + /* check (a*){n,m}, (a+){n,m} => (a*){n,n}, (a+){n,n} */ + if (env.backrefedMem == 0) { + if (qn.target.getType() == NodeType.ENCLOSE) { + EncloseNode en = (EncloseNode)qn.target; + if (en.type == EncloseType.MEMORY) { + if (en.target.getType() == NodeType.QTFR) { + QuantifierNode q = (QuantifierNode)en.target; + if (isRepeatInfinite(q.upper) && q.greedy == qn.greedy) { + qn.upper = qn.lower == 0 ? 1 : qn.lower; + if (qn.upper == 1) childState = state; + } + } + } + } + } + } + } + + if ((state & CEC_IN_FINITE_REPEAT) != 0) { + qn.combExpCheckNum = -1; + } else { + if (isRepeatInfinite(qn.upper)) { + varNum = CEC_INFINITE_NUM; + childState |= CEC_IN_INFINITE_REPEAT; + } else { + varNum = qn.upper - qn.lower; + } + + if (varNum >= CEC_THRES_NUM_BIG_REPEAT) addState |= CEC_CONT_BIG_REPEAT; + + if (((state & CEC_IN_INFINITE_REPEAT) != 0 && varNum != 0) || + ((state & CEC_CONT_BIG_REPEAT) != 0 && varNum >= CEC_THRES_NUM_BIG_REPEAT)) { + if (qn.combExpCheckNum == 0) { + env.numCombExpCheck++; + qn.combExpCheckNum = env.numCombExpCheck; + if (env.currMaxRegNum > env.combExpMaxRegNum) { + env.combExpMaxRegNum = env.currMaxRegNum; + } + } + } + } + r = setupCombExpCheck(qn.target, childState); + r |= addState; + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch( en.type) { + case EncloseNode.MEMORY: + if (env.currMaxRegNum < en.regNum) { + env.currMaxRegNum = en.regNum; + } + r = setupCombExpCheck(en.target, state); + break; + + default: + r = setupCombExpCheck(en.target, state); + } // inner switch + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + env.hasRecursion = true; + } else { + r = setupCombExpCheck(cn.target, state); + } + } // USE_SUBEXP_CALL + break; + + default: + break; + + } // switch + + return r; + } + + private static final int IN_ALT = (1<<0); + private static final int IN_NOT = (1<<1); + private static final int IN_REPEAT = (1<<2); + private static final int IN_VAR_REPEAT = (1<<3); + private static final int IN_CALL = (1<<4); + private static final int IN_RECCALL = (1<<5); + private static final int EXPAND_STRING_MAX_LENGTH = 100; + + /* setup_tree does the following work. + 1. check empty loop. (set qn->target_empty_info) + 2. expand ignore-case in char class. + 3. set memory status bit flags. (reg->mem_stats) + 4. set qn->head_exact for [push, exact] -> [push_or_jump_exact1, exact]. + 5. find invalid patterns in look-behind. + 6. expand repeated string. + */ + protected final Node setupTree(Node node, int state) { + restart: while (true) { + switch (node.getType()) { + case NodeType.LIST: + ListNode lin = (ListNode)node; + Node prev = null; + do { + setupTree(lin.value, state); + if (prev != null) { + nextSetup(prev, lin.value); + } + prev = lin.value; + } while ((lin = lin.tail) != null); + break; + + case NodeType.ALT: + ListNode aln = (ListNode)node; + do { + setupTree(aln.value, (state | IN_ALT)); + } while ((aln = aln.tail) != null); + break; + + case NodeType.CCLASS: + break; + + case NodeType.STR: + if (isIgnoreCase(regex.options) && !((StringNode)node).isRaw()) { + node = expandCaseFoldString(node); + } + break; + + case NodeType.CTYPE: + case NodeType.CANY: + break; + + case NodeType.CALL: // if (Config.USE_SUBEXP_CALL) ? + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + for (int i=0; i env.numMem) { + if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); + } else { + env.backrefedMem = bsOnAt(env.backrefedMem, br.back[i]); + env.btMemStart = bsOnAt(env.btMemStart, br.back[i]); + if (Config.USE_BACKREF_WITH_LEVEL) { + if (br.isNestLevel()) { + env.btMemEnd = bsOnAt(env.btMemEnd, br.back[i]); + } + } // USE_BACKREF_AT_LEVEL + env.memNodes[br.back[i]].setMemBackrefed(); + } + } + break; + + case NodeType.QTFR: + QuantifierNode qn = (QuantifierNode)node; + Node target = qn.target; + + if ((state & IN_REPEAT) != 0) qn.setInRepeat(); + + if (isRepeatInfinite(qn.upper) || qn.lower >= 1) { + int d = getMinMatchLength(target); + if (d == 0) { + qn.targetEmptyInfo = TargetInfo.IS_EMPTY; + if (Config.USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT) { + int info = quantifiersMemoryInfo(target); + if (info > 0) qn.targetEmptyInfo = info; + } // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + // strange stuff here (turned off) + } + } + + state |= IN_REPEAT; + if (qn.lower != qn.upper) state |= IN_VAR_REPEAT; + + target = setupTree(target, state); + + /* expand string */ + if (target.getType() == NodeType.STR && !(target instanceof CalloutNode)) { + StringNode sn = (StringNode)target; + if (qn.lower > 1) { + StringNode str = new StringNode(sn.bytes, sn.p, sn.end); + str.flag = sn.flag; + + int i; + int n = qn.lower; + int len = sn.length(); + for (i = 1; i < n && (i + 1) * len <= EXPAND_STRING_MAX_LENGTH; i++) { + str.catBytes(sn.bytes, sn.p, sn.end); + } + + if (i < qn.upper || isRepeatInfinite(qn.upper)) { + qn.lower -= i; + if (!isRepeatInfinite(qn.upper)) qn.upper -= i; + ListNode list = ListNode.newList(str, null); + qn.replaceWith(list); + ListNode.listAdd(list, qn); + } else { + qn.replaceWith(str); + } + break; + } + } + + if (Config.USE_OP_PUSH_OR_JUMP_EXACT) { + if (qn.greedy && qn.targetEmptyInfo != 0) { + if (target.getType() == NodeType.QTFR) { + QuantifierNode tqn = (QuantifierNode)target; + if (tqn.headExact != null) { + qn.headExact = tqn.headExact; + tqn.headExact = null; + } + } else { + qn.headExact = getHeadValueNode(qn.target, true); + } + } + } // USE_OP_PUSH_OR_JUMP_EXACT + break; + + case NodeType.ENCLOSE: + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.OPTION: + int options = regex.options; + regex.options = en.option; + setupTree(en.target, state); + regex.options = options; + break; + + case EncloseType.MEMORY: + if ((state & (IN_ALT | IN_NOT | IN_VAR_REPEAT | IN_CALL)) != 0) { + env.btMemStart = bsOnAt(env.btMemStart, en.regNum); + /* SET_ENCLOSE_STATUS(node, NST_MEM_IN_ALT_NOT); */ + } + if (en.isCalled()) state |= IN_CALL; + if (en.isRecursion()) { + state |= IN_RECCALL; + } else if ((state & IN_RECCALL) != 0){ + en.setRecursion(); + } + + setupTree(en.target, state); + break; + + case EncloseType.STOP_BACKTRACK: + setupTree(en.target, state); + if (en.target.getType() == NodeType.QTFR) { + QuantifierNode tqn = (QuantifierNode)en.target; + if (isRepeatInfinite(tqn.upper) && tqn.lower <= 1 && tqn.greedy) { + /* (?>a*), a*+ etc... */ + if (tqn.target.isSimple()) en.setStopBtSimpleRepeat(); + } + } + break; + + case EncloseNode.CONDITION: + if (Config.USE_NAMED_GROUP) { + if (!en.isNameRef() && env.numNamed > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(env.option)) { + newValueException(NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + } + } + if (en.regNum > env.numMem) newValueException(INVALID_BACKREF); + setupTree(en.target, state); + break; + + case EncloseType.ABSENT: + setupTree(en.target, state); + break; + } // inner switch + break; + + case NodeType.ANCHOR: + AnchorNode an = (AnchorNode)node; + switch (an.type) { + case AnchorType.PREC_READ: + setupTree(an.target, state); + break; + + case AnchorType.PREC_READ_NOT: + setupTree(an.target, (state | IN_NOT)); + break; + + case AnchorType.LOOK_BEHIND: + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB, AnchorType.ALLOWED_IN_LB)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + node = setupLookBehind(an); + if (node.getType() != NodeType.ANCHOR) continue restart; + setupTree(((AnchorNode)node).target, state); + node = setupLookBehind(an); + break; + + case AnchorType.LOOK_BEHIND_NOT: + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB_NOT, AnchorType.ALLOWED_IN_LB_NOT)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + node = setupLookBehind(an); + if (node.getType() != NodeType.ANCHOR) continue restart; + setupTree(((AnchorNode)node).target, (state | IN_NOT)); + node = setupLookBehind(an); + break; + + } // inner switch + break; + } // switch + return node; + } // restart: while + } + + private static final int MAX_NODE_OPT_INFO_REF_COUNT = 5; + private void optimizeNodeLeft(Node node, NodeOptInfo opt, OptEnvironment oenv) { // oenv remove, pass mmd + opt.clear(); + opt.setBoundNode(oenv.mmd); + + switch (node.getType()) { + case NodeType.LIST: { + OptEnvironment nenv = new OptEnvironment(); + NodeOptInfo nopt = new NodeOptInfo(); + nenv.copy(oenv); + ListNode lin = (ListNode)node; + do { + optimizeNodeLeft(lin.value, nopt, nenv); + nenv.mmd.add(nopt.length); + opt.concatLeftNode(nopt, enc); + } while ((lin = lin.tail) != null); + break; + } + + case NodeType.ALT: { + NodeOptInfo nopt = new NodeOptInfo(); + ListNode aln = (ListNode)node; + do { + optimizeNodeLeft(aln.value, nopt, oenv); + if (aln == node) { + opt.copy(nopt); + } else { + opt.altMerge(nopt, oenv); + } + } while ((aln = aln.tail) != null); + break; + } + + case NodeType.STR: { + StringNode sn = (StringNode)node; + + int slen = sn.length(); + + if (!sn.isAmbig()) { + opt.exb.concatStr(sn.bytes, sn.p, sn.end, sn.isRaw(), enc); + opt.exb.ignoreCase = 0; + + if (slen > 0) { + opt.map.addChar(sn.bytes[sn.p], enc); + } + + opt.length.set(slen, slen); + } else { + int max; + if (sn.isDontGetOptInfo()) { + int n = sn.length(enc); + max = enc.maxLength() * n; + } else { + opt.exb.concatStr(sn.bytes, sn.p, sn.end, sn.isRaw(), enc); + opt.exb.ignoreCase = 1; + + if (slen > 0) { + opt.map.addCharAmb(sn.bytes, sn.p, sn.end, enc, oenv.caseFoldFlag); + } + + max = slen; + } + opt.length.set(slen, max); + } + + if (opt.exb.length == slen) { + opt.exb.reachEnd = true; + } + break; + } + + case NodeType.CCLASS: { + CClassNode cc = (CClassNode)node; + /* no need to check ignore case. (setted in setup_tree()) */ + if (cc.mbuf != null || cc.isNot()) { + int min = enc.minLength(); + int max = enc.maxLength(); + opt.length.set(min, max); + } else { + for (int i=0; i= maxCode) { + opt.map.addChar((byte)i, enc); + } + } + } else { + for (int i=0; i 0) { + opt.expr.copy(nopt.exb); + } else if (nopt.exm.length > 0) { + opt.expr.copy(nopt.exm); + } + opt.expr.reachEnd = false; + if (nopt.map.value > 0) opt.map.copy(nopt.map); + break; + + case AnchorType.LOOK_BEHIND_NOT: + break; + + } // inner switch + break; + } + + case NodeType.BREF: { + BackRefNode br = (BackRefNode)node; + + if (br.isRecursion()) { + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + break; + } + + Node[]nodes = oenv.scanEnv.memNodes; + + int min = 0; + int max = 0; + + if (nodes != null && nodes[br.back[0]] != null) { + min = getMinMatchLength(nodes[br.back[0]]); + max = getMaxMatchLength(nodes[br.back[0]]); + } + + for (int i=1; i tmin) min = tmin; + if (max < tmax) max = tmax; + } + } + opt.length.set(min, max); + break; + } + + case NodeType.CALL: { + if (Config.USE_SUBEXP_CALL) { + CallNode cn = (CallNode)node; + if (cn.isRecursion()) { + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + } else { + int safe = oenv.options; + oenv.options = cn.target.option; + optimizeNodeLeft(cn.target, opt, oenv); + oenv.options = safe; + } + } // USE_SUBEXP_CALL + break; + } + + case NodeType.QTFR: { + NodeOptInfo nopt = new NodeOptInfo(); + QuantifierNode qn = (QuantifierNode)node; + optimizeNodeLeft(qn.target, nopt, oenv); + if (/*qn.lower == 0 &&*/ isRepeatInfinite(qn.upper)) { + if (oenv.mmd.max == 0 && qn.target.getType() == NodeType.CANY && qn.greedy) { + if (isMultiline(oenv.options)) { + opt.anchor.add(AnchorType.ANYCHAR_STAR_ML); + } else { + opt.anchor.add(AnchorType.ANYCHAR_STAR); + } + } + } /*else*/ { + if (qn.lower > 0) { + opt.copy(nopt); + if (nopt.exb.length > 0) { + if (nopt.exb.reachEnd) { + int i; + for (i = 2; i <= qn.lower && !opt.exb.isFull(); i++) { + opt.exb.concat(nopt.exb, enc); + } + if (i < qn.lower) { + opt.exb.reachEnd = false; + } + } + } + if (qn.lower != qn.upper) { + opt.exb.reachEnd = false; + opt.exm.reachEnd = false; + } + if (qn.lower > 1) { + opt.exm.reachEnd = false; + } + + } + } + int min = MinMaxLen.distanceMultiply(nopt.length.min, qn.lower); + int max; + if (isRepeatInfinite(qn.upper)) { + max = nopt.length.max > 0 ? MinMaxLen.INFINITE_DISTANCE : 0; + } else { + max = MinMaxLen.distanceMultiply(nopt.length.max, qn.upper); + } + opt.length.set(min, max); + break; + } + + case NodeType.ENCLOSE: { + EncloseNode en = (EncloseNode)node; + switch (en.type) { + case EncloseType.OPTION: + int save = oenv.options; + oenv.options = en.option; + optimizeNodeLeft(en.target, opt, oenv); + oenv.options = save; + break; + + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL && ++en.optCount > MAX_NODE_OPT_INFO_REF_COUNT) { + int min = 0; + int max = MinMaxLen.INFINITE_DISTANCE; + if (en.isMinFixed()) min = en.minLength; + if (en.isMaxFixed()) max = en.maxLength; + opt.length.set(min, max); + } else { // USE_SUBEXP_CALL + optimizeNodeLeft(en.target, opt, oenv); + if (opt.anchor.isSet(AnchorType.ANYCHAR_STAR_MASK)) { + if (bsAt(oenv.scanEnv.backrefedMem, en.regNum)) { + opt.anchor.remove(AnchorType.ANYCHAR_STAR_MASK); + } + } + } + break; + + case EncloseType.STOP_BACKTRACK: + case EncloseType.CONDITION: + optimizeNodeLeft(en.target, opt, oenv); + break; + + case EncloseType.ABSENT: + opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); + break; + } // inner switch + break; + } + + default: + newInternalException(PARSER_BUG); + } // switch + } + + protected final void setOptimizedInfoFromTree(Node node) { + NodeOptInfo opt = new NodeOptInfo(); + OptEnvironment oenv = new OptEnvironment(); + + oenv.enc = regex.enc; + oenv.options = regex.options; + oenv.caseFoldFlag = regex.caseFoldFlag; + oenv.scanEnv = env; + oenv.mmd.clear(); // ?? + + optimizeNodeLeft(node, opt, oenv); + + regex.anchor = opt.anchor.leftAnchor & (AnchorType.BEGIN_BUF | + AnchorType.BEGIN_POSITION | + AnchorType.ANYCHAR_STAR | + AnchorType.ANYCHAR_STAR_ML | + AnchorType.LOOK_BEHIND); + + if ((opt.anchor.leftAnchor & (AnchorType.LOOK_BEHIND | AnchorType.PREC_READ_NOT)) != 0) regex.anchor &= ~AnchorType.ANYCHAR_STAR_ML; + + regex.anchor |= opt.anchor.rightAnchor & (AnchorType.END_BUF | + AnchorType.SEMI_END_BUF | + AnchorType.PREC_READ_NOT); + + if ((regex.anchor & (AnchorType.END_BUF | AnchorType.SEMI_END_BUF)) != 0) { + regex.anchorDmin = opt.length.min; + regex.anchorDmax = opt.length.max; + } + + if (opt.exb.length > 0 || opt.exm.length > 0) { + opt.exb.select(opt.exm, enc); + if (opt.map.value > 0 && opt.exb.compare(opt.map) > 0) { + // !goto set_map;! + regex.setOptimizeMapInfo(opt.map); + regex.setSubAnchor(opt.map.anchor); + } else { + regex.setOptimizeExactInfo(opt.exb); + regex.setSubAnchor(opt.exb.anchor); + } + } else if (opt.map.value > 0) { + // !set_map:! + regex.setOptimizeMapInfo(opt.map); + regex.setSubAnchor(opt.map.anchor); + } else { + regex.subAnchor |= opt.anchor.leftAnchor & AnchorType.BEGIN_LINE; + if (opt.length.max == 0) regex.subAnchor |= opt.anchor.rightAnchor & AnchorType.END_LINE; + } + + if (Config.DEBUG_COMPILE || Config.DEBUG_MATCH) { + Config.log.println(regex.optimizeInfoToString()); + } + } +} diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java new file mode 100644 index 000000000..77b27fdda --- /dev/null +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -0,0 +1,109 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.ApplyAllCaseFoldFunction; +import org.jcodings.Encoding; +import org.joni.ast.CClassNode; +import org.joni.ast.ListNode; +import org.joni.ast.StringNode; + +final class ApplyCaseFold implements ApplyAllCaseFoldFunction { + + // i_apply_case_fold + @Override + public void apply(int from, int[]to, int length, Object o) { + ApplyCaseFoldArg arg = (ApplyCaseFoldArg)o; + + ScanEnvironment env = arg.env; + Encoding enc = env.enc; + CClassNode cc = arg.cc; + CClassNode ascCc = arg.ascCc; + BitSet bs = cc.bs; + boolean addFlag; + + if (ascCc == null) { + addFlag = false; + } else if (Encoding.isAscii(from) == Encoding.isAscii(to[0])) { + addFlag = true; + } else { + addFlag = ascCc.isCodeInCC(enc, from); + if (ascCc.isNot()) addFlag = !addFlag; + } + + if (length == 1) { + boolean inCC = cc.isCodeInCC(enc, from); + if (Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS) { + if ((inCC && !cc.isNot()) || (!inCC && cc.isNot())) { + if (addFlag) { + if (enc.minLength() > 1 || to[0] >= BitSet.SINGLE_BYTE_SIZE || enc.codeToMbcLength(to[0]) > 1) { + cc.addCodeRange(env, to[0], to[0], false); + } else { + /* /(?i:[^A-C])/.match("a") ==> fail. */ + bs.set(to[0]); + } + } + } + } else { + if (inCC) { + if (addFlag) { + if (enc.minLength() > 1 || to[0] >= BitSet.SINGLE_BYTE_SIZE) { + if (cc.isNot()) cc.clearNotFlag(env); + cc.addCodeRange(env, to[0], to[0], false); + } else { + if (cc.isNot()) { + bs.clear(to[0]); + } else { + bs.set(to[0]); + } + } + } + } + } // CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS + + } else { + if (cc.isCodeInCC(enc, from) && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { + StringNode node = null; + for (int i=0; i 1 || cc.bs.isEmpty()) { + len = OPSize.OPCODE; + } else { + len = OPSize.OPCODE + BitSet.BITSET_SIZE; + } + + len += OPSize.LENGTH + cc.mbuf.getUsed(); + } + return len; + } + + @Override + protected void compileCClassNode(CClassNode cc) { + if (cc.mbuf == null) { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_NOT); + } else { + addOpcode(OPCode.CCLASS); + } + addInts(cc.bs.bits, BitSet.BITSET_SIZE); // add_bitset + } else { + if (enc.minLength() > 1 || cc.bs.isEmpty()) { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_MB_NOT); + } else { + addOpcode(OPCode.CCLASS_MB); + } + addMultiByteCClass(cc.mbuf); + } else { + if (cc.isNot()) { + addOpcode(OPCode.CCLASS_MIX_NOT); + } else { + addOpcode(OPCode.CCLASS_MIX); + } + // store the bit set and mbuf themself! + addInts(cc.bs.bits, BitSet.BITSET_SIZE); // add_bitset + addMultiByteCClass(cc.mbuf); + } + } + } + + @Override + protected void compileCTypeNode(CTypeNode node) { + CTypeNode cn = node; + int op; + switch (cn.ctype) { + case CharacterType.WORD: + if (cn.not) { + if (cn.asciiRange) { + op = OPCode.ASCII_NOT_WORD; + } else { + op = OPCode.NOT_WORD; + } + } else { + if (cn.asciiRange) { + op = OPCode.ASCII_WORD; + } else { + op = OPCode.WORD; + } + } + break; + + default: + newInternalException(PARSER_BUG); + return; // not reached + } // inner switch + addOpcode(op); + } + + @Override + protected void compileAnyCharNode() { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML); + } else { + addOpcode(OPCode.ANYCHAR); + } + } + + @Override + protected void compileCallNode(CallNode node) { + addOpcode(OPCode.CALL); + node.unsetAddrList.add(codeLength, node.target); + addAbsAddr(0); /*dummy addr.*/ + } + + @Override + protected void compileBackrefNode(BackRefNode node) { + BackRefNode br = node; + if (Config.USE_BACKREF_WITH_LEVEL && br.isNestLevel()) { + addOpcode(OPCode.BACKREF_WITH_LEVEL); + addOption(regex.options & Option.IGNORECASE); + addLength(br.nestLevel); + // !goto add_bacref_mems;! + addLength(br.backNum); + for (int i=br.backNum-1; i>=0; i--) addMemNum(br.back[i]); + return; + } else { // USE_BACKREF_AT_LEVEL + if (br.backNum == 1) { + if (isIgnoreCase(regex.options)) { + addOpcode(OPCode.BACKREFN_IC); + addMemNum(br.back[0]); + } else { + switch (br.back[0]) { + case 1: + addOpcode(OPCode.BACKREF1); + break; + case 2: + addOpcode(OPCode.BACKREF2); + break; + default: + addOpcode(OPCode.BACKREFN); + addOpcode(br.back[0]); + break; + } // switch + } + } else { + if (isIgnoreCase(regex.options)) { + addOpcode(OPCode.BACKREF_MULTI_IC); + } else { + addOpcode(OPCode.BACKREF_MULTI); + } + // !add_bacref_mems:! + addLength(br.backNum); + for (int i=br.backNum-1; i>=0; i--) addMemNum(br.back[i]); + } + } + } + + private static final int REPEAT_RANGE_ALLOC = 8; + private void entryRepeatRange(int id, int lower, int upper) { + if (regex.repeatRangeLo == null) { + regex.repeatRangeLo = new int[REPEAT_RANGE_ALLOC]; + regex.repeatRangeHi = new int[REPEAT_RANGE_ALLOC]; + } else if (id >= regex.repeatRangeLo.length){ + int[]tmp = new int[regex.repeatRangeLo.length + REPEAT_RANGE_ALLOC]; + System.arraycopy(regex.repeatRangeLo, 0, tmp, 0, regex.repeatRangeLo.length); + regex.repeatRangeLo = tmp; + tmp = new int[regex.repeatRangeHi.length + REPEAT_RANGE_ALLOC]; + System.arraycopy(regex.repeatRangeHi, 0, tmp, 0, regex.repeatRangeHi.length); + regex.repeatRangeHi = tmp; + } + + regex.repeatRangeLo[id] = lower; + regex.repeatRangeHi[id] = isRepeatInfinite(upper) ? 0x7fffffff : upper; + } + + private void compileRangeRepeatNode(QuantifierNode qn, int targetLen, int emptyInfo) { + regex.requireStack = true; + int numRepeat = regex.numRepeat; + addOpcode(qn.greedy ? OPCode.REPEAT : OPCode.REPEAT_NG); + addMemNum(numRepeat); /* OP_REPEAT ID */ + regex.numRepeat++; + addRelAddr(targetLen + OPSize.REPEAT_INC); + + entryRepeatRange(numRepeat, qn.lower, qn.upper); + + compileTreeEmptyCheck(qn.target, emptyInfo); + + if ((Config.USE_SUBEXP_CALL && regex.numCall > 0) || qn.isInRepeat()) { + addOpcode(qn.greedy ? OPCode.REPEAT_INC_SG : OPCode.REPEAT_INC_NG_SG); + } else { + addOpcode(qn.greedy ? OPCode.REPEAT_INC : OPCode.REPEAT_INC_NG); + } + + addMemNum(numRepeat); /* OP_REPEAT ID */ + } + + private static final int QUANTIFIER_EXPAND_LIMIT_SIZE = 50; // was 50 + + private static boolean cknOn(int ckn) { + return ckn > 0; + } + + private int compileCECLengthQuantifierNode(QuantifierNode qn) { + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + int ckn = regex.numCombExpCheck > 0 ? qn.combExpCheckNum : 0; + int cklen = cknOn(ckn) ? OPSize.STATE_CHECK_NUM : 0; + + /* anychar repeat */ + if (qn.target.getType() == NodeType.CANY) { + if (qn.greedy && infinite) { + if (qn.nextHeadExact != null && !cknOn(ckn)) { + return OPSize.ANYCHAR_STAR_PEEK_NEXT + tlen * qn.lower + cklen; + } else { + return OPSize.ANYCHAR_STAR + tlen * qn.lower + cklen; + } + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + + int len; + if (infinite && qn.lower <= 1) { + if (qn.greedy) { + if (qn.lower == 1) { + len = OPSize.JUMP; + } else { + len = 0; + } + len += OPSize.PUSH + cklen + modTLen + OPSize.JUMP; + } else { + if (qn.lower == 0) { + len = OPSize.JUMP; + } else { + len = 0; + } + len += modTLen + OPSize.PUSH + cklen; + } + } else if (qn.upper == 0) { + if (qn.isRefered) { /* /(?..){0}/ */ + len = OPSize.JUMP + tlen; + } else { + len = 0; + } + } else if (qn.upper == 1 && qn.greedy) { + if (qn.lower == 0) { + if (cknOn(ckn)) { + len = OPSize.STATE_CHECK_PUSH + tlen; + } else { + len = OPSize.PUSH + tlen; + } + } else { + len = tlen; + } + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0) { /* '??' */ + len = OPSize.PUSH + cklen + OPSize.JUMP + tlen; + } else { + len = OPSize.REPEAT_INC + modTLen + OPSize.OPCODE + OPSize.RELADDR + OPSize.MEMNUM; + + if (cknOn(ckn)) { + len += OPSize.STATE_CHECK; + } + } + return len; + } + + @Override + protected void compileCECQuantifierNode(QuantifierNode qn) { + regex.requireStack = true; + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + int ckn = regex.numCombExpCheck > 0 ? qn.combExpCheckNum : 0; + + if (qn.isAnyCharStar()) { + compileTreeNTimes(qn.target, qn.lower); + if (qn.nextHeadExact != null && !cknOn(ckn)) { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR_PEEK_NEXT); + } else { + addOpcode(OPCode.ANYCHAR_STAR_PEEK_NEXT); + } + if (cknOn(ckn)) { + addStateCheckNum(ckn); + } + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + return; + } else { + if (isMultiline(regex.options)) { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_ANYCHAR_ML_STAR); + } else { + addOpcode(OPCode.ANYCHAR_ML_STAR); + } + } else { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_ANYCHAR_STAR); + } else { + addOpcode(OPCode.ANYCHAR_STAR); + } + } + if (cknOn(ckn)) { + addStateCheckNum(ckn); + } + return; + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + if (infinite && qn.lower <= 1) { + if (qn.greedy) { + if (qn.lower == 1) { + addOpcodeRelAddr(OPCode.JUMP, cknOn(ckn) ? OPSize.STATE_CHECK_PUSH : + OPSize.PUSH); + } + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(modTLen + OPSize.JUMP); + } else { + addOpcodeRelAddr(OPCode.PUSH, modTLen + OPSize.JUMP); + } + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + (cknOn(ckn) ? + OPSize.STATE_CHECK_PUSH : + OPSize.PUSH))); + } else { + if (qn.lower == 0) { + addOpcodeRelAddr(OPCode.JUMP, modTLen); + } + compileTreeEmptyCheck(qn.target, emptyInfo); + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH_OR_JUMP); + addStateCheckNum(ckn); + addRelAddr(-(modTLen + OPSize.STATE_CHECK_PUSH_OR_JUMP)); + } else { + addOpcodeRelAddr(OPCode.PUSH, -(modTLen + OPSize.PUSH)); + } + } + } else if (qn.upper == 0) { + if (qn.isRefered) { /* /(?..){0}/ */ + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } // else r=0 ??? + } else if (qn.upper == 1 && qn.greedy) { + if (qn.lower == 0) { + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(tlen); + } else { + addOpcodeRelAddr(OPCode.PUSH, tlen); + } + } + compileTree(qn.target); + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0){ /* '??' */ + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK_PUSH); + addStateCheckNum(ckn); + addRelAddr(OPSize.JUMP); + } else { + addOpcodeRelAddr(OPCode.PUSH, OPSize.JUMP); + } + + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } else { + compileRangeRepeatNode(qn, modTLen, emptyInfo); + if (cknOn(ckn)) { + addOpcode(OPCode.STATE_CHECK); + addStateCheckNum(ckn); + } + } + } + + private int compileNonCECLengthQuantifierNode(QuantifierNode qn) { + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + /* anychar repeat */ + if (qn.target.getType() == NodeType.CANY) { + if (qn.greedy && infinite) { + if (qn.nextHeadExact != null) { + return OPSize.ANYCHAR_STAR_PEEK_NEXT + tlen * qn.lower; + } else { + return OPSize.ANYCHAR_STAR + tlen * qn.lower; + } + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + + int len; + if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) { + len = OPSize.JUMP; + } else { + len = tlen * qn.lower; + } + + if (qn.greedy) { + if (qn.headExact != null) { + len += OPSize.PUSH_OR_JUMP_EXACT1 + modTLen + OPSize.JUMP; + } else if (qn.nextHeadExact != null) { + len += OPSize.PUSH_IF_PEEK_NEXT + modTLen + OPSize.JUMP; + } else { + len += OPSize.PUSH + modTLen + OPSize.JUMP; + } + } else { + len += OPSize.JUMP + modTLen + OPSize.PUSH; + } + + } else if (qn.upper == 0 && qn.isRefered) { /* /(?..){0}/ */ + len = OPSize.JUMP + tlen; + } else if (!infinite && qn.greedy && + (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE )) { + len = tlen * qn.lower; + len += (OPSize.PUSH + tlen) * (qn.upper - qn.lower); + } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0) { /* '??' */ + len = OPSize.PUSH + OPSize.JUMP + tlen; + } else { + len = OPSize.REPEAT_INC + modTLen + OPSize.OPCODE + OPSize.RELADDR + OPSize.MEMNUM; + } + return len; + } + + @Override + protected void compileNonCECQuantifierNode(QuantifierNode qn) { + regex.requireStack = true; + boolean infinite = isRepeatInfinite(qn.upper); + int emptyInfo = qn.targetEmptyInfo; + + int tlen = compileLengthTree(qn.target); + + if (qn.isAnyCharStar()) { + compileTreeNTimes(qn.target, qn.lower); + if (qn.nextHeadExact != null) { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR_PEEK_NEXT); + } else { + addOpcode(OPCode.ANYCHAR_STAR_PEEK_NEXT); + } + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + return; + } else { + if (isMultiline(regex.options)) { + addOpcode(OPCode.ANYCHAR_ML_STAR); + } else { + addOpcode(OPCode.ANYCHAR_STAR); + } + return; + } + } + + int modTLen; + if (emptyInfo != 0) { + modTLen = tlen + (OPSize.NULL_CHECK_START + OPSize.NULL_CHECK_END); + } else { + modTLen = tlen; + } + if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) { + if (qn.greedy) { + if (qn.headExact != null) { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH_OR_JUMP_EXACT1); + } else if (qn.nextHeadExact != null) { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH_IF_PEEK_NEXT); + } else { + addOpcodeRelAddr(OPCode.JUMP, OPSize.PUSH); + } + } else { + addOpcodeRelAddr(OPCode.JUMP, OPSize.JUMP); + } + } else { + compileTreeNTimes(qn.target, qn.lower); + } + + if (qn.greedy) { + if (qn.headExact != null) { + addOpcodeRelAddr(OPCode.PUSH_OR_JUMP_EXACT1, modTLen + OPSize.JUMP); + StringNode sn = (StringNode)qn.headExact; + addBytes(sn.bytes, sn.p, 1); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH_OR_JUMP_EXACT1)); + } else if (qn.nextHeadExact != null) { + addOpcodeRelAddr(OPCode.PUSH_IF_PEEK_NEXT, modTLen + OPSize.JUMP); + StringNode sn = (StringNode)qn.nextHeadExact; + addBytes(sn.bytes, sn.p, 1); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH_IF_PEEK_NEXT)); + } else { + addOpcodeRelAddr(OPCode.PUSH, modTLen + OPSize.JUMP); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.JUMP, -(modTLen + OPSize.JUMP + OPSize.PUSH)); + } + } else { + addOpcodeRelAddr(OPCode.JUMP, modTLen); + compileTreeEmptyCheck(qn.target, emptyInfo); + addOpcodeRelAddr(OPCode.PUSH, -(modTLen + OPSize.PUSH)); + } + } else if (qn.upper == 0 && qn.isRefered) { /* /(?..){0}/ */ + addOpcodeRelAddr(OPCode.JUMP, tlen); + compileTree(qn.target); + } else if (!infinite && qn.greedy && + (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE)) { + int n = qn.upper - qn.lower; + compileTreeNTimes(qn.target, qn.lower); + + for (int i=0; i= 0 ? OPSize.CALLOUT_CONDITION : OPSize.CONDITION; + if (node.target.getType() == NodeType.ALT) { + ListNode x = (ListNode)node.target; + tlen = compileLengthTree(x.value); /* yes-node */ + len += tlen + OPSize.JUMP; + if (x.tail == null) newInternalException(PARSER_BUG); + x = x.tail; + tlen = compileLengthTree(x.value); /* no-node */ + len += tlen; + if (x.tail != null) newSyntaxException(INVALID_CONDITION_PATTERN); + } else { + newInternalException(PARSER_BUG); + } + break; + case EncloseType.ABSENT: + len = OPSize.PUSH_ABSENT_POS + OPSize.ABSENT + tlen + OPSize.ABSENT_END; + break; + default: + newInternalException(PARSER_BUG); + return 0; // not reached + } // switch + return len; + } + + @Override + protected void compileEncloseNode(EncloseNode node) { + int len; + switch (node.type) { + case EncloseType.MEMORY: + if (Config.USE_SUBEXP_CALL && node.isCalled()) { + regex.requireStack = true; + addOpcode(OPCode.CALL); + node.callAddr = codeLength + OPSize.ABSADDR + OPSize.JUMP; + node.setAddrFixed(); + addAbsAddr(node.callAddr); + len = compileLengthTree(node.target); + len += OPSize.MEMORY_START_PUSH + OPSize.RETURN; + if (bsAt(regex.btMemEnd, node.regNum)) { + len += node.isRecursion() ? OPSize.MEMORY_END_PUSH_REC : OPSize.MEMORY_END_PUSH; + } else { + len += node.isRecursion() ? OPSize.MEMORY_END_REC : OPSize.MEMORY_END; + } + addOpcodeRelAddr(OPCode.JUMP, len); + } // USE_SUBEXP_CALL + + if (bsAt(regex.btMemStart, node.regNum)) { + regex.requireStack = true; + addOpcode(OPCode.MEMORY_START_PUSH); + } else { + addOpcode(OPCode.MEMORY_START); + } + + addMemNum(node.regNum); + compileTree(node.target); + + if (Config.USE_SUBEXP_CALL && node.isCalled()) { + if (bsAt(regex.btMemEnd, node.regNum)) { + addOpcode(node.isRecursion() ? OPCode.MEMORY_END_PUSH_REC : OPCode.MEMORY_END_PUSH); + } else { + addOpcode(node.isRecursion() ? OPCode.MEMORY_END_REC : OPCode.MEMORY_END); + } + addMemNum(node.regNum); + addOpcode(OPCode.RETURN); + } else if (Config.USE_SUBEXP_CALL && node.isRecursion()) { // USE_SUBEXP_CALL + if (bsAt(regex.btMemEnd, node.regNum)) { + addOpcode(OPCode.MEMORY_END_PUSH_REC); + } else { + addOpcode(OPCode.MEMORY_END_REC); + } + addMemNum(node.regNum); + } else { + if (bsAt(regex.btMemEnd, node.regNum)) { + addOpcode(OPCode.MEMORY_END_PUSH); + } else { + addOpcode(OPCode.MEMORY_END); + } + addMemNum(node.regNum); + } + break; + + case EncloseType.STOP_BACKTRACK: + regex.requireStack = true; + if (node.isStopBtSimpleRepeat()) { + QuantifierNode qn = (QuantifierNode)node.target; + + compileTreeNTimes(qn.target, qn.lower); + + len = compileLengthTree(qn.target); + addOpcodeRelAddr(OPCode.PUSH, len + OPSize.POP + OPSize.JUMP); + compileTree(qn.target); + addOpcode(OPCode.POP); + addOpcodeRelAddr(OPCode.JUMP, -(OPSize.PUSH + len + OPSize.POP + OPSize.JUMP)); + } else { + addOpcode(OPCode.PUSH_STOP_BT); + compileTree(node.target); + addOpcode(OPCode.POP_STOP_BT); + } + break; + + case EncloseType.CONDITION: + if (node.calloutConditionId >= 0) regex.requireStack = true; + addOpcode(node.calloutConditionId >= 0 + ? OPCode.CALLOUT_CONDITION : OPCode.CONDITION); + addMemNum(node.calloutConditionId >= 0 + ? node.calloutConditionId : node.regNum); + if (node.target.getType() == NodeType.ALT) { + ListNode x = (ListNode)node.target; + len = compileLengthTree(x.value); /* yes-node */ + if (x.tail == null) newInternalException(PARSER_BUG); + x = x.tail; + int len2 = compileLengthTree(x.value); /* no-node */ + if (x.tail != null) newSyntaxException(INVALID_CONDITION_PATTERN); + x = (ListNode)node.target; + addRelAddr(len + OPSize.JUMP); + compileTree(x.value); /* yes-node */ + addOpcodeRelAddr(OPCode.JUMP, len2); + x = x.tail; + compileTree(x.value); /* no-node */ + } else { + newInternalException(PARSER_BUG); + } + break; + + case EncloseType.ABSENT: + regex.requireStack = true; + len = compileLengthTree(node.target); + addOpcode(OPCode.PUSH_ABSENT_POS); + addOpcodeRelAddr(OPCode.ABSENT, len + OPSize.ABSENT_END); + compileTree(node.target); + addOpcode(OPCode.ABSENT_END); + break; + + default: + newInternalException(PARSER_BUG); + break; + } // switch + } + + private int compileLengthAnchorNode(AnchorNode node) { + int tlen; + if (node.target != null) { + tlen = compileLengthTree(node.target); + } else { + tlen = 0; + } + + int len; + switch (node.type) { + case AnchorType.PREC_READ: + len = OPSize.PUSH_POS + tlen + OPSize.POP_POS; + break; + + case AnchorType.PREC_READ_NOT: + len = OPSize.PUSH_POS_NOT + tlen + OPSize.FAIL_POS; + break; + + case AnchorType.LOOK_BEHIND: + len = OPSize.LOOK_BEHIND + tlen; + break; + + case AnchorType.LOOK_BEHIND_NOT: + len = OPSize.PUSH_LOOK_BEHIND_NOT + tlen + OPSize.FAIL_LOOK_BEHIND_NOT; + break; + + default: + len = OPSize.OPCODE; + break; + } // switch + return len; + } + + @Override + protected void compileAnchorNode(AnchorNode node) { + int len; + int n; + + switch (node.type) { + case AnchorType.BEGIN_BUF: addOpcode(OPCode.BEGIN_BUF); break; + case AnchorType.END_BUF: addOpcode(OPCode.END_BUF); break; + case AnchorType.BEGIN_LINE: addOpcode(OPCode.BEGIN_LINE); break; + case AnchorType.END_LINE: addOpcode(OPCode.END_LINE); break; + case AnchorType.SEMI_END_BUF: addOpcode(OPCode.SEMI_END_BUF); break; + case AnchorType.BEGIN_POSITION: addOpcode(OPCode.BEGIN_POSITION); break; + + case AnchorType.WORD_BOUND: + if (node.asciiRange) { + addOpcode(OPCode.ASCII_WORD_BOUND); + } else { + addOpcode(OPCode.WORD_BOUND); + } + break; + + case AnchorType.NOT_WORD_BOUND: + if (node.asciiRange) { + addOpcode(OPCode.ASCII_NOT_WORD_BOUND); + } else { + addOpcode(OPCode.NOT_WORD_BOUND); + } + break; + + case AnchorType.WORD_BEGIN: + if (Config.USE_WORD_BEGIN_END) { + if (node.asciiRange) { + addOpcode(OPCode.ASCII_WORD_BEGIN); + } else { + addOpcode(OPCode.WORD_BEGIN); + } + } + break; + + case AnchorType.WORD_END: + if (Config.USE_WORD_BEGIN_END) { + if (node.asciiRange) { + addOpcode(OPCode.ASCII_WORD_END); + } else { + addOpcode(OPCode.WORD_END); + } + } + break; + + case AnchorType.KEEP: + addOpcode(OPCode.KEEP); + break; + + case AnchorType.PREC_READ: + regex.requireStack = true; + addOpcode(OPCode.PUSH_POS); + compileTree(node.target); + addOpcode(OPCode.POP_POS); + break; + + case AnchorType.PREC_READ_NOT: + regex.requireStack = true; + len = compileLengthTree(node.target); + addOpcodeRelAddr(OPCode.PUSH_POS_NOT, len + OPSize.FAIL_POS); + compileTree(node.target); + addOpcode(OPCode.FAIL_POS); + break; + + case AnchorType.LOOK_BEHIND: + addOpcode(OPCode.LOOK_BEHIND); + if (node.charLength < 0) { + n = analyser.getCharLengthTree(node.target); + if (analyser.returnCode != 0) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + } else { + n = node.charLength; + } + addLength(n); + compileTree(node.target); + break; + + case AnchorType.LOOK_BEHIND_NOT: + regex.requireStack = true; + len = compileLengthTree(node.target); + addOpcodeRelAddr(OPCode.PUSH_LOOK_BEHIND_NOT, len + OPSize.FAIL_LOOK_BEHIND_NOT); + if (node.charLength < 0) { + n = analyser.getCharLengthTree(node.target); + if (analyser.returnCode != 0) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + } else { + n = node.charLength; + } + addLength(n); + compileTree(node.target); + addOpcode(OPCode.FAIL_LOOK_BEHIND_NOT); + break; + + default: + newInternalException(PARSER_BUG); + } // switch + } + + private int compileLengthTree(Node node) { + if (node instanceof CalloutNode) return OPSize.CALLOUT; + int len = 0; + + switch (node.getType()) { + case NodeType.LIST: + ListNode lin = (ListNode)node; + do { + len += compileLengthTree(lin.value); + } while ((lin = lin.tail) != null); + break; + + case NodeType.ALT: + ListNode aln = (ListNode)node; + int n = 0; + do { + len += compileLengthTree(aln.value); + n++; + } while ((aln = aln.tail) != null); + len += (OPSize.PUSH + OPSize.JUMP) * (n - 1); + break; + + case NodeType.STR: + StringNode sn = (StringNode)node; + if (sn.isRaw()) { + len = compileLengthStringRawNode(sn); + } else { + len = compileLengthStringNode(sn); + } + break; + + case NodeType.CCLASS: + len = compileLengthCClassNode((CClassNode)node); + break; + + case NodeType.CTYPE: + case NodeType.CANY: + len = OPSize.OPCODE; + break; + + case NodeType.BREF: + BackRefNode br = (BackRefNode)node; + + if (Config.USE_BACKREF_WITH_LEVEL && br.isNestLevel()) { + len = OPSize.OPCODE + OPSize.OPTION + OPSize.LENGTH + + OPSize.LENGTH + (OPSize.MEMNUM * br.backNum); + } else { // USE_BACKREF_AT_LEVEL + if (br.backNum == 1) { + len = ((!isIgnoreCase(regex.options) && br.back[0] <= 2) + ? OPSize.OPCODE : (OPSize.OPCODE + OPSize.MEMNUM)); + } else { + len = OPSize.OPCODE + OPSize.LENGTH + (OPSize.MEMNUM * br.backNum); + } + } + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + len = OPSize.CALL; + break; + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + if (Config.USE_CEC) { + len = compileCECLengthQuantifierNode((QuantifierNode)node); + } else { + len = compileNonCECLengthQuantifierNode((QuantifierNode)node); + } + break; + + case NodeType.ENCLOSE: + len = compileLengthEncloseNode((EncloseNode)node); + break; + + case NodeType.ANCHOR: + len = compileLengthAnchorNode((AnchorNode)node); + break; + + default: + newInternalException(PARSER_BUG); + + } //switch + return len; + } + + private void ensure(int size) { + if (size >= code.length) { + int length = code.length << 1; + while (length <= size) length <<= 1; + int[]tmp = new int[length]; + System.arraycopy(code, 0, tmp, 0, code.length); + code = tmp; + } + } + + private void addInt(int i) { + if (codeLength >= code.length) { + int[]tmp = new int[code.length << 1]; + System.arraycopy(code, 0, tmp, 0, code.length); + code = tmp; + } + code[codeLength++] = i; + } + + void setInt(int i, int offset) { + ensure(offset); + regex.code[offset] = i; + } + + private void addBytes(byte[]bytes, int p ,int length) { + ensure(codeLength + length); + int end = p + length; + + while (p < end) code[codeLength++] = bytes[p++]; + } + + private void addInts(int[]ints, int length) { + ensure(codeLength + length); + System.arraycopy(ints, 0, code, codeLength, length); + codeLength += length; + } + + private void addOpcode(int opcode) { + addInt(opcode); + } + + private void addStateCheckNum(int num) { + addInt(num); + } + + private void addRelAddr(int addr) { + addInt(addr); + } + + private void addAbsAddr(int addr) { + addInt(addr); + } + + private void addLength(int length) { + addInt(length); + } + + private void addMemNum(int num) { + addInt(num); + } + + private void addOption(int option) { + addInt(option); + } + + private void addOpcodeRelAddr(int opcode, int addr) { + addOpcode(opcode); + addRelAddr(addr); + } + + private void addOpcodeOption(int opcode, int option) { + addOpcode(opcode); + addOption(option); + } + + private void addTemplate(byte[]bytes) { + if (templateNum == 0) { + templates = new byte[2][]; + } else if (templateNum == templates.length) { + byte[][]tmp = new byte[templateNum * 2][]; + System.arraycopy(templates, 0, tmp, 0, templateNum); + templates = tmp; + } + templates[templateNum++] = bytes; + } +} diff --git a/third_party/joni/src/org/joni/BitSet.java b/third_party/joni/src/org/joni/BitSet.java new file mode 100644 index 000000000..fe8d3c1aa --- /dev/null +++ b/third_party/joni/src/org/joni/BitSet.java @@ -0,0 +1,123 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class BitSet { + static final int BITS_PER_BYTE = 8; + public static final int SINGLE_BYTE_SIZE = (1 << BITS_PER_BYTE); + public static final int BITS_IN_ROOM = 4 * BITS_PER_BYTE; + public static final int BITSET_SIZE = (SINGLE_BYTE_SIZE / BITS_IN_ROOM); + static final int ROOM_SHIFT = log2(BITS_IN_ROOM); + + public final int[] bits = new int[BITSET_SIZE]; + + public boolean at(int pos) { + return (bits[pos >>> ROOM_SHIFT] & bit(pos)) != 0; + } + + public void set(ScanEnvironment env, int pos) { + if (at(pos)) env.ccDuplicateWarn(); + set(pos); + } + + public void set(int pos) { + bits[pos >>> ROOM_SHIFT] |= bit(pos); + } + + public void clear(int pos) { + bits[pos >>> ROOM_SHIFT] &= ~bit(pos); + } + + public void invert(int pos) { + bits[pos >>> ROOM_SHIFT] ^= bit(pos); + } + + public void clear() { + for (int i=0; i>>= 1) != 0) log++; + return log; + } + + private static final int BITS_TO_STRING_WRAP = 4; + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append("BitSet"); + for (int i=0; i s) ? s : pkeep) - str; + node.end = s - str; + + stkp = 0; + makeCaptureHistoryTree(region.getCaptureTree()); + } + + private byte[]cfbuf; + private byte[]cfbuf2; + + protected final byte[]cfbuf() { + return cfbuf == null ? cfbuf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : cfbuf; + } + + protected final byte[]cfbuf2() { + return cfbuf2 == null ? cfbuf2 = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : cfbuf2; + } + + private boolean stringCmpIC(int caseFlodFlag, int s1, IntHolder ps2, int mbLen, int textEnd) { + byte[]buf1 = cfbuf(); + byte[]buf2 = cfbuf2(); + + int s2 = ps2.value; + int end1 = s1 + mbLen; + + while (s1 < end1) { + value = s1; + int len1 = enc.mbcCaseFold(caseFlodFlag, bytes, this, textEnd, buf1); + s1 = value; + value = s2; + int len2 = enc.mbcCaseFold(caseFlodFlag, bytes, this, textEnd, buf2); + s2 = value; + + if (len1 != len2) return false; + int p1 = 0; + int p2 = 0; + + while (len1-- > 0) { + if (buf1[p1] != buf2[p2]) return false; + p1++; p2++; + } + } + ps2.value = s2; + return true; + } + + @Override + protected final int matchAt(int _range, int _sstart, int _sprev, boolean interrupt) throws InterruptedException { + range = _range; + sstart = _sstart; + sprev = _sprev; + stk = 0; + ip = 0; + + if (Config.DEBUG_MATCH) debugMatchBegin(); + stackInit(); + + bestLen = -1; + s = _sstart; + pkeep = _sstart; + int result = -1; + try { + result = enc.isSingleByte() || (msaOptions & Option.CR_7_BIT) != 0 + ? executeSb(interrupt) : execute(interrupt); + return result; + } finally { + if (result >= 0) completeActiveCallouts(); + else unwindActiveCallouts(); + } + } + + private final int execute(final boolean checkThreadInterrupt) throws InterruptedException { + final int[] code = this.code; + int interruptCheckCounter = 0; + while (true) { + if (interruptCheckCounter++ >= interruptCheckEvery) { + if (timeout != -1) handleTimeout(); + handleInterrupted(checkThreadInterrupt); + interruptCheckCounter = 0; + } + + if (Config.DEBUG_MATCH) debugMatchLoop(); + + sbegin = s; + switch (code[ip++]) { + case OPCode.END: if (opEnd()) return finish(); break; + case OPCode.EXACT1: opExact1(); break; + case OPCode.EXACT2: opExact2(); continue; + case OPCode.EXACT3: opExact3(); continue; + case OPCode.EXACT4: opExact4(); continue; + case OPCode.EXACT5: opExact5(); continue; + case OPCode.EXACTN: opExactN(); continue; + + case OPCode.EXACTMB2N1: opExactMB2N1(); break; + case OPCode.EXACTMB2N2: opExactMB2N2(); continue; + case OPCode.EXACTMB2N3: opExactMB2N3(); continue; + case OPCode.EXACTMB2N: opExactMB2N(); continue; + case OPCode.EXACTMB3N: opExactMB3N(); continue; + case OPCode.EXACTMBN: opExactMBN(); continue; + + case OPCode.EXACT1_IC: opExact1IC(); break; + case OPCode.EXACTN_IC: opExactNIC(); continue; + + case OPCode.CCLASS: opCClass(); break; + case OPCode.CCLASS_MB: opCClassMB(); break; + case OPCode.CCLASS_MIX: opCClassMIX(); break; + case OPCode.CCLASS_NOT: opCClassNot(); break; + case OPCode.CCLASS_MB_NOT: opCClassMBNot(); break; + case OPCode.CCLASS_MIX_NOT: opCClassMIXNot(); break; + + case OPCode.ANYCHAR: opAnyChar(); break; + case OPCode.ANYCHAR_ML: opAnyCharML(); break; + case OPCode.ANYCHAR_STAR: opAnyCharStar(); break; + case OPCode.ANYCHAR_ML_STAR: opAnyCharMLStar(); break; + case OPCode.ANYCHAR_STAR_PEEK_NEXT: opAnyCharStarPeekNext(); break; + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: opAnyCharMLStarPeekNext(); break; + + case OPCode.WORD: opWord(); break; + case OPCode.NOT_WORD: opNotWord(); break; + case OPCode.WORD_BOUND: opWordBound(); continue; + case OPCode.NOT_WORD_BOUND: opNotWordBound(); continue; + case OPCode.WORD_BEGIN: opWordBegin(); continue; + case OPCode.WORD_END: opWordEnd(); continue; + + case OPCode.ASCII_WORD: opAsciiWord(); break; + case OPCode.ASCII_NOT_WORD: opNotAsciiWord(); break; + case OPCode.ASCII_WORD_BOUND: opAsciiWordBound(); break; + case OPCode.ASCII_NOT_WORD_BOUND: opNotAsciiWordBound(); continue; + case OPCode.ASCII_WORD_BEGIN: opAsciiWordBegin(); continue; + case OPCode.ASCII_WORD_END: opAsciiWordEnd(); continue; + + case OPCode.BEGIN_BUF: opBeginBuf(); continue; + case OPCode.END_BUF: opEndBuf(); continue; + case OPCode.BEGIN_LINE: opBeginLine(); continue; + case OPCode.END_LINE: opEndLine(); continue; + case OPCode.SEMI_END_BUF: opSemiEndBuf(); continue; + case OPCode.BEGIN_POSITION: opBeginPosition(); continue; + + case OPCode.MEMORY_START_PUSH: opMemoryStartPush(); continue; + case OPCode.MEMORY_START: opMemoryStart(); continue; + case OPCode.MEMORY_END_PUSH: opMemoryEndPush(); continue; + case OPCode.MEMORY_END: opMemoryEnd(); continue; // + case OPCode.KEEP: opKeep(); continue; + case OPCode.MEMORY_END_PUSH_REC: opMemoryEndPushRec(); continue; + case OPCode.MEMORY_END_REC: opMemoryEndRec(); continue; + + case OPCode.BACKREF1: opBackRef1(); continue; + case OPCode.BACKREF2: opBackRef2(); continue; + case OPCode.BACKREFN: opBackRefN(); continue; + case OPCode.BACKREFN_IC: opBackRefNIC(); continue; + case OPCode.BACKREF_MULTI: opBackRefMulti(); continue; + case OPCode.BACKREF_MULTI_IC: opBackRefMultiIC(); continue; + case OPCode.BACKREF_WITH_LEVEL: opBackRefAtLevel(); continue; + + case OPCode.NULL_CHECK_START: opNullCheckStart(); continue; + case OPCode.NULL_CHECK_END: opNullCheckEnd(); continue; + case OPCode.NULL_CHECK_END_MEMST: opNullCheckEndMemST(); continue; + case OPCode.NULL_CHECK_END_MEMST_PUSH: opNullCheckEndMemSTPush(); continue; + + case OPCode.JUMP: opJump(); continue; + case OPCode.PUSH: opPush(); continue; + + case OPCode.POP: opPop(); continue; + case OPCode.PUSH_OR_JUMP_EXACT1: opPushOrJumpExact1(); continue; + case OPCode.PUSH_IF_PEEK_NEXT: opPushIfPeekNext(); continue; + + case OPCode.REPEAT: opRepeat(); continue; + case OPCode.REPEAT_NG: opRepeatNG(); continue; + case OPCode.REPEAT_INC: opRepeatInc(); continue; + case OPCode.REPEAT_INC_SG: opRepeatIncSG(); continue; + case OPCode.REPEAT_INC_NG: opRepeatIncNG(); continue; + case OPCode.REPEAT_INC_NG_SG: opRepeatIncNGSG(); continue; + + case OPCode.PUSH_POS: opPushPos(); continue; + case OPCode.POP_POS: opPopPos(); continue; + case OPCode.PUSH_POS_NOT: opPushPosNot(); continue; + case OPCode.FAIL_POS: opFailPos(); continue; + case OPCode.PUSH_STOP_BT: opPushStopBT(); continue; + case OPCode.POP_STOP_BT: opPopStopBT(); continue; + + case OPCode.LOOK_BEHIND: opLookBehind(); continue; + case OPCode.PUSH_LOOK_BEHIND_NOT: opPushLookBehindNot(); continue; + case OPCode.FAIL_LOOK_BEHIND_NOT: opFailLookBehindNot(); continue; + + case OPCode.PUSH_ABSENT_POS: opPushAbsentPos(); continue; + case OPCode.ABSENT: opAbsent(); continue; + case OPCode.ABSENT_END: opAbsentEnd(); continue; + + case OPCode.CALL: opCall(); continue; + case OPCode.RETURN: opReturn(); continue; + case OPCode.CONDITION: opCondition(); continue; + case OPCode.FINISH: return finish(); + case OPCode.FAIL: opFail(); continue; + case OPCode.CALLOUT: opCallout(); continue; + case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; + + case OPCode.STATE_CHECK_ANYCHAR_STAR: if (USE_CEC) {opStateCheckAnyCharStar(); break;} + case OPCode.STATE_CHECK_ANYCHAR_ML_STAR:if (USE_CEC) {opStateCheckAnyCharMLStar();break;} + case OPCode.STATE_CHECK_PUSH: if (USE_CEC) {opStateCheckPush(); continue;} + case OPCode.STATE_CHECK_PUSH_OR_JUMP: if (USE_CEC) {opStateCheckPushOrJump(); continue;} + case OPCode.STATE_CHECK: if (USE_CEC) {opStateCheck(); continue;} + + default: + throw new InternalException(ErrorMessages.UNDEFINED_BYTECODE); + + } // main switch + } // main while + } + + private void handleTimeout() throws InterruptedException { + if (System.nanoTime() - startTime > timeout) throw TIMEOUT_EXCEPTION; + } + + private final int executeSb(final boolean checkThreadInterrupt) throws InterruptedException { + final int[] code = this.code; + int interruptCheckCounter = 0; + while (true) { + if (interruptCheckCounter++ >= interruptCheckEvery) { + if (timeout != -1) handleTimeout(); + handleInterrupted(checkThreadInterrupt); + interruptCheckCounter = 0; + } + + if (Config.DEBUG_MATCH) debugMatchLoop(); + + sbegin = s; + switch (code[ip++]) { + case OPCode.END: if (opEnd()) return finish(); break; + case OPCode.EXACT1: opExact1(); break; + case OPCode.EXACT2: opExact2(); continue; + case OPCode.EXACT3: opExact3(); continue; + case OPCode.EXACT4: opExact4(); continue; + case OPCode.EXACT5: opExact5(); continue; + case OPCode.EXACTN: opExactN(); continue; + + case OPCode.EXACTMB2N1: opExactMB2N1(); break; + case OPCode.EXACTMB2N2: opExactMB2N2(); continue; + case OPCode.EXACTMB2N3: opExactMB2N3(); continue; + case OPCode.EXACTMB2N: opExactMB2N(); continue; + case OPCode.EXACTMB3N: opExactMB3N(); continue; + case OPCode.EXACTMBN: opExactMBN(); continue; + + case OPCode.EXACT1_IC: opExact1IC(); break; + case OPCode.EXACTN_IC: opExactNIC(); continue; + + case OPCode.CCLASS: opCClassSb(); break; + case OPCode.CCLASS_MB: opCClassMBSb(); break; + case OPCode.CCLASS_MIX: opCClassMIXSb(); break; + case OPCode.CCLASS_NOT: opCClassNotSb(); break; + case OPCode.CCLASS_MB_NOT: opCClassMBNotSb(); break; + case OPCode.CCLASS_MIX_NOT: opCClassMIXNotSb(); break; + + case OPCode.ANYCHAR: opAnyCharSb(); break; + case OPCode.ANYCHAR_ML: opAnyCharMLSb(); break; + case OPCode.ANYCHAR_STAR: opAnyCharStarSb(); break; + case OPCode.ANYCHAR_ML_STAR: opAnyCharMLStarSb(); break; + case OPCode.ANYCHAR_STAR_PEEK_NEXT: opAnyCharStarPeekNextSb(); break; + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: opAnyCharMLStarPeekNextSb(); break; + + case OPCode.WORD: opWordSb(); break; + case OPCode.NOT_WORD: opNotWordSb(); break; + case OPCode.WORD_BOUND: opWordBoundSb(); continue; + case OPCode.NOT_WORD_BOUND: opNotWordBoundSb(); continue; + case OPCode.WORD_BEGIN: opWordBeginSb(); continue; + case OPCode.WORD_END: opWordEndSb(); continue; + + case OPCode.ASCII_WORD: opAsciiWord(); break; + case OPCode.ASCII_NOT_WORD: opNotAsciiWord(); break; + case OPCode.ASCII_WORD_BOUND: opAsciiWordBound(); break; + case OPCode.ASCII_NOT_WORD_BOUND: opNotAsciiWordBound(); continue; + case OPCode.ASCII_WORD_BEGIN: opAsciiWordBegin(); continue; + case OPCode.ASCII_WORD_END: opAsciiWordEnd(); continue; + + case OPCode.BEGIN_BUF: opBeginBuf(); continue; + case OPCode.END_BUF: opEndBuf(); continue; + case OPCode.BEGIN_LINE: opBeginLineSb(); continue; + case OPCode.END_LINE: opEndLineSb(); continue; + case OPCode.SEMI_END_BUF: opSemiEndBuf(); continue; + case OPCode.BEGIN_POSITION: opBeginPosition(); continue; + + case OPCode.MEMORY_START_PUSH: opMemoryStartPush(); continue; + case OPCode.MEMORY_START: opMemoryStart(); continue; + case OPCode.MEMORY_END_PUSH: opMemoryEndPush(); continue; + case OPCode.MEMORY_END: opMemoryEnd(); continue; + case OPCode.KEEP: opKeep(); continue; + case OPCode.MEMORY_END_PUSH_REC: opMemoryEndPushRec(); continue; + case OPCode.MEMORY_END_REC: opMemoryEndRec(); continue; + + case OPCode.BACKREF1: opBackRef1(); continue; + case OPCode.BACKREF2: opBackRef2(); continue; + case OPCode.BACKREFN: opBackRefN(); continue; + case OPCode.BACKREFN_IC: opBackRefNIC(); continue; + case OPCode.BACKREF_MULTI: opBackRefMulti(); continue; + case OPCode.BACKREF_MULTI_IC: opBackRefMultiIC(); continue; + case OPCode.BACKREF_WITH_LEVEL: opBackRefAtLevel(); continue; + + case OPCode.NULL_CHECK_START: opNullCheckStart(); continue; + case OPCode.NULL_CHECK_END: opNullCheckEnd(); continue; + case OPCode.NULL_CHECK_END_MEMST: opNullCheckEndMemST(); continue; + case OPCode.NULL_CHECK_END_MEMST_PUSH: opNullCheckEndMemSTPush(); continue; + + case OPCode.JUMP: opJump(); continue; + case OPCode.PUSH: opPush(); continue; + + case OPCode.POP: opPop(); continue; + case OPCode.PUSH_OR_JUMP_EXACT1: opPushOrJumpExact1(); continue; + case OPCode.PUSH_IF_PEEK_NEXT: opPushIfPeekNext(); continue; + + case OPCode.REPEAT: opRepeat(); continue; + case OPCode.REPEAT_NG: opRepeatNG(); continue; + case OPCode.REPEAT_INC: opRepeatInc(); continue; + case OPCode.REPEAT_INC_SG: opRepeatIncSG(); continue; + case OPCode.REPEAT_INC_NG: opRepeatIncNG(); continue; + case OPCode.REPEAT_INC_NG_SG: opRepeatIncNGSG(); continue; + + case OPCode.PUSH_POS: opPushPos(); continue; + case OPCode.POP_POS: opPopPos(); continue; + case OPCode.PUSH_POS_NOT: opPushPosNot(); continue; + case OPCode.FAIL_POS: opFailPos(); continue; + case OPCode.PUSH_STOP_BT: opPushStopBT(); continue; + case OPCode.POP_STOP_BT: opPopStopBT(); continue; + + case OPCode.LOOK_BEHIND: opLookBehindSb(); continue; + case OPCode.PUSH_LOOK_BEHIND_NOT: opPushLookBehindNot(); continue; + case OPCode.FAIL_LOOK_BEHIND_NOT: opFailLookBehindNot(); continue; + + case OPCode.PUSH_ABSENT_POS: opPushAbsentPos(); continue; + case OPCode.ABSENT: opAbsent(); continue; + case OPCode.ABSENT_END: opAbsentEnd(); continue; + + case OPCode.CALL: opCall(); continue; + case OPCode.RETURN: opReturn(); continue; + case OPCode.CONDITION: opCondition(); continue; + case OPCode.FINISH: return finish(); + case OPCode.FAIL: opFail(); continue; + case OPCode.CALLOUT: opCallout(); continue; + case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; + + case OPCode.EXACT1_IC_SB: opExact1ICSb(); break; + case OPCode.EXACTN_IC_SB: opExactNICSb(); continue; + + case OPCode.STATE_CHECK_ANYCHAR_STAR: if (USE_CEC) {opStateCheckAnyCharStarSb(); break;} + case OPCode.STATE_CHECK_ANYCHAR_ML_STAR:if (USE_CEC) {opStateCheckAnyCharMLStarSb();break;} + case OPCode.STATE_CHECK_PUSH: if (USE_CEC) {opStateCheckPush(); continue;} + case OPCode.STATE_CHECK_PUSH_OR_JUMP: if (USE_CEC) {opStateCheckPushOrJump(); continue;} + case OPCode.STATE_CHECK: if (USE_CEC) {opStateCheck(); continue;} + + default: + throw new InternalException(ErrorMessages.UNDEFINED_BYTECODE); + + } // main switch + } // main while + } + + private void handleInterrupted(final boolean checkThreadInterrupt) throws InterruptedException { + if (interrupted || (checkThreadInterrupt && Thread.currentThread().isInterrupted())) { + Thread.interrupted(); + throw INTERRUPTED_EXCEPTION; + } + interruptCheckEvery = Math.min(interruptCheckEvery << 1, MAX_INTERRUPT_CHECK_EVERY); + } + + private boolean opEnd() { + int n = s - sstart; + + if (n > bestLen) { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (isFindLongest(regex.options)) { + if (n > msaBestLen) { + msaBestLen = n; + msaBestS = sstart; + } else { + // goto end_best_len; + return endBestLength(); + } + } + } // USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE + + bestLen = n; + final Region region = msaRegion; + if (region != null) { + // USE_POSIX_REGION_OPTION ... else ... + region.setBeg(0, msaBegin = ((pkeep > s) ? s : pkeep) - str); + region.setEnd(0, msaEnd = s - str); + for (int i = 1; i <= regex.numMem; i++) { + int me = repeatStk[memEndStk + i]; + if (me != INVALID_INDEX) { + int ms = repeatStk[memStartStk + i]; + region.setBeg(i, (bsAt(regex.btMemStart, i) ? stack[ms].getMemPStr() : ms) - str); + region.setEnd(i, (bsAt(regex.btMemEnd, i) ? stack[me].getMemPStr() : me) - str); + } else { + region.setBeg(i, Region.REGION_NOTPOS); + region.setEnd(i, Region.REGION_NOTPOS); + } + } + + if (Config.USE_CAPTURE_HISTORY && regex.captureHistory != 0) checkCaptureHistory(region); + } else { + msaBegin = ((pkeep > s) ? s : pkeep) - str; + msaEnd = s - str; + } + } else { + Region region = msaRegion; + if (region != null) { + region.clear(); + } else { + msaBegin = msaEnd = 0; + } + } + // end_best_len: + /* default behavior: return first-matching result. */ + return endBestLength(); + } + + private boolean endBestLength() { + if (isFindCondition(regex.options)) { + if (isFindNotEmpty(regex.options) && s == sstart) { + bestLen = -1; + {opFail(); return false;} /* for retry */ + } + if (isFindLongest(regex.options) && s < range) { + {opFail(); return false;} /* for retry */ + } + } + // goto finish; + return true; + } + + private void opExact1() { + if (s >= range || code[ip] != bytes[s]) { + opFail(); + } else { + ip++; s++; + sprev = sbegin; // break; + } + } + + private void opExact2() { + if (s + 2 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] ) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact3() { + if (s + 3 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact4() { + if (s + 4 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExact5() { + if (s + 5 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + sprev = s; + ip++; s++; + } + } + + private void opExactN() { + int tlen = code[ip++]; + if (s + tlen > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen-- > 0) if (bs[ps++] != bytes[s++]) {opFail(); return;} + + } else { + while (tlen-- > 0) if (code[ip++] != bytes[s++]) {opFail(); return;} + } + sprev = s - 1; + } + + private void opExactMB2N1() { + if (s + 2 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s]) { + opFail(); + } else { + ip++; s++; + sprev = sbegin; // break; + } + } + + private void opExactMB2N2() { + if (s + 4 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + sprev = s; + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + + private void opExactMB2N3() { + if (s + 6 > range || code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + sprev = s; + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + + private void opExactMB2N() { + int tlen = code[ip++]; + if (s + tlen * 2 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while(tlen-- > 0) { + if (bs[ps] != bytes[s] || bs[++ps] != bytes[++s]) {opFail(); return;} + ps++; s++; + } + } else { + while(tlen-- > 0) { + if (code[ip] != bytes[s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + } + sprev = s - 2; + } + + private void opExactMB3N() { + int tlen = code[ip++]; + if (s + tlen * 3 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen-- > 0) { + if (bs[ps] != bytes[s] || bs[++ps] != bytes[++s] || bs[++ps] != bytes[++s]) {opFail(); return;} + ps++; s++; + } + } else { + while (tlen-- > 0) { + if (code[ip] != bytes[s] || code[++ip] != bytes[++s] || code[++ip] != bytes[++s]) {opFail(); return;} + ip++; s++; + } + } + + sprev = s - 3; + } + + private void opExactMBN() { + int tlen = code[ip++]; /* mb-len */ + int tlen2= code[ip++]; /* string len */ + + tlen2 *= tlen; + if (s + tlen2 > range) {opFail(); return;} + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + + while (tlen2-- > 0) { + if (bs[ps] != bytes[s]) {opFail(); return;} + ps++; s++; + } + } else { + while (tlen2-- > 0) { + if (code[ip] != bytes[s]) {opFail(); return;} + ip++; s++; + } + } + + sprev = s - tlen; + } + + private void opExact1IC() { + if (s >= range) {opFail(); return;} + + byte[]lowbuf = cfbuf(); + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + + int q = 0; + while (len-- > 0) { + if (code[ip] != lowbuf[q]) {opFail(); return;} + ip++; q++; + } + sprev = sbegin; // break; + } + + private void opExact1ICSb() { + if (s >= range || code[ip] != enc.toLowerCaseTable()[bytes[s++] & 0xff]) {opFail(); return;} + ip++; + sprev = sbegin; // break; + } + + private void opExactNIC() { + int tlen = code[ip++]; + byte[]lowbuf = cfbuf(); + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + int endp = ps + tlen; + + while (ps < endp) { + sprev = s; + if (s >= range) {opFail(); return;} + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + int q = 0; + while (len-- > 0) { + if (bs[ps] != lowbuf[q]) {opFail(); return;} + ps++; q++; + } + } + } else { + int endp = ip + tlen; + + while (ip < endp) { + sprev = s; + if (s >= range) {opFail(); return;} + + value = s; + int len = enc.mbcCaseFold(regex.caseFoldFlag, bytes, this, end, lowbuf); + s = value; + + if (s > range) {opFail(); return;} + int q = 0; + while (len-- > 0) { + if (code[ip] != lowbuf[q]) {opFail(); return;} + ip++; q++; + } + } + } + + } + + private void opExactNICSb() { + int tlen = code[ip++]; + if (s + tlen > range) {opFail(); return;} + byte[]toLowerTable = enc.toLowerCaseTable(); + + if (Config.USE_STRING_TEMPLATES) { + byte[]bs = regex.templates[code[ip++]]; + int ps = code[ip++]; + while (tlen-- > 0) if (bs[ps++] != toLowerTable[bytes[s++] & 0xff]) {opFail(); return;} + } else { + while (tlen-- > 0) if (code[ip++] != toLowerTable[bytes[s++] & 0xff]) {opFail(); return;} + } + sprev = s - 1; + } + + private void opCondition() { + int mem = code[ip++]; + int addr = code[ip++]; + if (mem > regex.numMem || repeatStk[memEndStk + mem] == INVALID_INDEX || repeatStk[memStartStk + mem] == INVALID_INDEX) { + ip += addr; + } + } + + private boolean isInBitSet() { + int c = bytes[s] & 0xff; + return ((code[ip + (c >>> BitSet.ROOM_SHIFT)] & (1 << c)) != 0); + } + + private void opCClass() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s += enc.length(bytes, s, end); /* OP_CCLASS can match mb-code. \D, \S */ + if (s > end) s = end; + sprev = sbegin; // break; + } + + private void opCClassSb() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + sprev = sbegin; // break; + } + + private boolean isInClassMB() { + int tlen = code[ip++]; + if (s >= range) return false; + int mbLen = enc.length(bytes, s, end); + if (s + mbLen > range) return false; + int ss = s; + s += mbLen; + int c = enc.mbcToCode(bytes, ss, s); + if (!CodeRange.isInCodeRange(code, ip, c)) return false; + ip += tlen; + return true; + } + + private void opCClassMB() { + // beyond string check + if (s >= range || !enc.isMbcHead(bytes, s, end)) {opFail(); return;} + if (!isInClassMB()) {opFail(); return;} // not!!! + sprev = sbegin; // break; + } + + private void opCClassMBSb() { + opFail(); + } + + private void opCClassMIX() { + if (s >= range) {opFail(); return;} + if (enc.isMbcHead(bytes, s, end)) { + ip += BitSet.BITSET_SIZE; + if (!isInClassMB()) {opFail(); return;} + } else { + if (!isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; // by code range length + ip += tlen; + s++; + } + sprev = sbegin; // break; + } + + private void opCClassMIXSb() { + if (s >= range || !isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; + ip += tlen; + s++; + sprev = sbegin; // break; + } + + private void opCClassNot() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s += enc.length(bytes, s, end); + if (s > end) s = end; + sprev = sbegin; // break; + } + + private void opCClassNotSb() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + sprev = sbegin; // break; + } + + private boolean isNotInClassMB() { + int tlen = code[ip++]; + int mbLen = enc.length(bytes, s, end); + + if (!(s + mbLen <= range)) { + if (s >= range) return false; + s = end; + ip += tlen; + return true; + } + + int ss = s; + s += mbLen; + int c = enc.mbcToCode(bytes, ss, s); + + if (CodeRange.isInCodeRange(code, ip, c)) return false; + ip += tlen; + return true; + } + + private void opCClassMBNot() { + if (s >= range) {opFail(); return;} + if (!enc.isMbcHead(bytes, s, end)) { + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + return; + } + if (!isNotInClassMB()) {opFail(); return;} + sprev = sbegin; // break; + } + + private void opCClassMBNotSb() { + if (s >= range) {opFail(); return;} + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + } + + private void opCClassMIXNot() { + if (s >= range) {opFail(); return;} + if (enc.isMbcHead(bytes, s, end)) { + ip += BitSet.BITSET_SIZE; + if (!isNotInClassMB()) {opFail(); return;} + } else { + if (isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + int tlen = code[ip++]; + ip += tlen; + s++; + } + sprev = sbegin; // break; + } + + private void opCClassMIXNotSb() { + if (s >= range || isInBitSet()) {opFail(); return;} + ip += BitSet.BITSET_SIZE; + s++; + int tlen = code[ip++]; + ip += tlen; + sprev = sbegin; // break; + } + + private void opAnyChar() { + final int n; + if (s >= range || s + (n = enc.length(bytes, s, end)) > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + s += n; + sprev = sbegin; // break; + } + + private void opAnyCharSb() { + if (s >= range || bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAnyCharML() { + if (s >= range) {opFail(); return;} + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + s += n; + sprev = sbegin; // break; + } + + private void opAnyCharMLSb() { + if (s >= range) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAnyCharStar() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + if (enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + } + + private void opAnyCharStarSb() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + if (bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + } + + private void opAnyCharMLStar() { + final byte[]bytes = this.bytes; + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + } + + private void opAnyCharMLStarSb() { + while (s < range) { + pushAlt(ip, s, sprev, pkeep); + sprev = s; + s++; + } + } + + private void opAnyCharStarPeekNext() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharStarPeekNextSb() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + byte b = bytes[s]; + if (c == b) pushAlt(ip + 1, s, sprev, pkeep); + if (b == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharMLStarPeekNext() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + ip++; + sprev = sbegin; // break; + } + + private void opAnyCharMLStarPeekNextSb() { + final byte c = (byte)code[ip]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (c == bytes[s]) pushAlt(ip + 1, s, sprev, pkeep); + sprev = s; + s++; + } + ip++; + sprev = sbegin; // break; + } + + // CEC + private void opStateCheckAnyCharStar() { + int mem = code[ip++]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range || enc.isNewLine(bytes, s, end)) {opFail(); return;} + sprev = s; + s += n; + } + sprev = sbegin; // break; + } + + private void opStateCheckAnyCharStarSb() { + int mem = code[ip++]; + final byte[]bytes = this.bytes; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + if (bytes[s] == Encoding.NEW_LINE) {opFail(); return;} + sprev = s; + s++; + } + sprev = sbegin; // break; + } + + // CEC + private void opStateCheckAnyCharMLStar() { + int mem = code[ip++]; + + final byte[]bytes = this.bytes; + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + int n = enc.length(bytes, s, end); + if (s + n > range) {opFail(); return;} + sprev = s; + s += n; + } + sprev = sbegin; // break; + } + + private void opStateCheckAnyCharMLStarSb() { + int mem = code[ip++]; + + while (s < range) { + if (stateCheckVal(s, mem)) {opFail(); return;} + pushAltWithStateCheck(ip, s, sprev, mem, pkeep); + sprev = s; + s++; + } + sprev = sbegin; // break; + } + + private void opWord() { + if (s >= range || !enc.isMbcWord(bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opWordSb() { + if (s >= range || !enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opAsciiWord() { + if (s >= range || !isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opNotWord() { + if (s >= range || enc.isMbcWord(bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opNotWordSb() { + if (s >= range || enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + s++; + sprev = sbegin; // break; + } + + private void opNotAsciiWord() { + if (s >= range || isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + s += enc.length(bytes, s, end); + sprev = sbegin; // break; + } + + private void opWordBound() { + if (s == str) { + if (s >= range || !enc.isMbcWord(bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } else { + if (enc.isMbcWord(bytes, s, end) == enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } + } + + private void opWordBoundSb() { + if (s == str) { + if (s >= range || !enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } else { + if (enc.isWord(bytes[s] & 0xff) == enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } + } + + private void opAsciiWordBound() { + if (s == str) { + if (s >= range || !isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev >= end || !isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } else { + if (isMbcAsciiWord(enc, bytes, s, end) == isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } + } + + private void opNotWordBound() { + if (s == str) { + if (s < range && enc.isMbcWord(bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } else { + if (enc.isMbcWord(bytes, s, end) != enc.isMbcWord(bytes, sprev, end)) {opFail(); return;} + } + } + + private void opNotWordBoundSb() { + if (s == str) { + if (s < range && enc.isWord(bytes[s] & 0xff)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } else { + if (enc.isWord(bytes[s] & 0xff) != enc.isWord(bytes[sprev] & 0xff)) {opFail(); return;} + } + } + + private void opNotAsciiWordBound() { + if (s == str) { + if (s < range && isMbcAsciiWord(enc, bytes, s, end)) {opFail(); return;} + } else if (s == end) { + if (sprev < end && isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } else { + if (isMbcAsciiWord(enc, bytes, s, end) != isMbcAsciiWord(enc, bytes, sprev, end)) {opFail(); return;} + } + } + + private void opWordBegin() { + if (s < range && enc.isMbcWord(bytes, s, end)) { + if (s == str || !enc.isMbcWord(bytes, sprev, end)) return; + } + opFail(); + } + + private void opWordBeginSb() { + if (s < range && enc.isWord(bytes[s] & 0xff)) { + if (s == str || !enc.isWord(bytes[sprev] & 0xff)) return; + } + opFail(); + } + + private void opAsciiWordBegin() { + if (s < range && isMbcAsciiWord(enc, bytes, s, end)) { + if (s == str || !isMbcAsciiWord(enc, bytes, sprev, end)) return; + } + opFail(); + } + + private void opWordEnd() { + if (s != str && enc.isMbcWord(bytes, sprev, end)) { + if (s == end || !enc.isMbcWord(bytes, s, end)) return; + } + opFail(); + } + + private void opWordEndSb() { + if (s != str && enc.isWord(bytes[sprev] & 0xff)) { + if (s == end || !enc.isWord(bytes[s] & 0xff)) return; + } + opFail(); + } + + private void opAsciiWordEnd() { + if (s != str && isMbcAsciiWord(enc, bytes, sprev, end)) { + if (s == end || !isMbcAsciiWord(enc, bytes, s, end)) return; + } + opFail(); + } + + private void opBeginBuf() { + if (s != str) opFail(); + } + + private void opEndBuf() { + if (s != end) opFail(); + } + + private void opBeginLine() { + if (s == str) { + if (isNotBol(msaOptions)) opFail(); + return; + } else if (enc.isNewLine(bytes, sprev, end) && s != end) { + return; + } + opFail(); + } + + private void opBeginLineSb() { + if (s == str) { + if (isNotBol(msaOptions)) opFail(); + return; + } else if (bytes[sprev] == Encoding.NEW_LINE && s != end) { + return; + } + opFail(); + } + + private void opEndLine() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !enc.isNewLine(bytes, sprev, end)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (enc.isNewLine(bytes, s, end) || (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end))) { + return; + } + opFail(); + } + + private void opEndLineSb() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !(sprev < end && bytes[sprev] == Encoding.NEW_LINE)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (bytes[s] == Encoding.NEW_LINE || (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end))) { + return; + } + opFail(); + } + + private void opSemiEndBuf() { + if (s == end) { + if (Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + if (str == end || !enc.isNewLine(bytes, sprev, end)) { + if (isNotEol(msaOptions)) opFail(); + } + return; + } else { + if (isNotEol(msaOptions)) opFail(); + return; + } + } else if (enc.isNewLine(bytes, s, end) && (s + enc.length(bytes, s, end)) == end) { + return; + } else if (Config.USE_CRNL_AS_LINE_TERMINATOR && enc.isMbcCrnl(bytes, s, end)) { + int ss = s + enc.length(bytes, s, end); + ss += enc.length(bytes, ss, end); + if (ss == end) return; + } + opFail(); + } + + private void opBeginPosition() { + if (s != msaGpos) opFail(); + } + + private void opMemoryStartPush() { + int mem = code[ip++]; + pushMemStart(mem, s); + } + + private void opMemoryStart() { + int mem = code[ip++]; + repeatStk[memStartStk + mem] = s; + repeatStk[memEndStk + mem] = -1; + } + + private void opMemoryEndPush() { + int mem = code[ip++]; + pushMemEnd(mem, s); + } + + private void opMemoryEnd() { + int mem = code[ip++]; + repeatStk[memEndStk + mem] = s; + } + + private void opKeep() { + pkeep = s; + } + + private void opMemoryEndPushRec() { + int mem = code[ip++]; + int stkp = getMemStart(mem); /* should be before push mem-end. */ + pushMemEnd(mem, s); + repeatStk[memStartStk + mem] = stkp; + } + + private void opMemoryEndRec() { + int mem = code[ip++]; + repeatStk[memEndStk + mem] = s; + int stkp = getMemStart(mem); + repeatStk[memStartStk + mem] = bsAt(regex.btMemStart, mem) ? stkp : stack[stkp].getMemPStr(); + pushMemEndMark(mem); + } + + private boolean backrefInvalid(int mem) { + return repeatStk[memEndStk + mem] == INVALID_INDEX || repeatStk[memStartStk + mem] == INVALID_INDEX; + } + + private int backrefStart(int mem) { + int ms = repeatStk[memStartStk + mem]; + return bsAt(regex.btMemStart, mem) ? stack[ms].getMemPStr() : ms; + } + + private int backrefEnd(int mem) { + int me = repeatStk[memEndStk + mem]; + return bsAt(regex.btMemEnd, mem) ? stack[me].getMemPStr() : me; + } + + private void backref(int mem) { + if (mem > regex.numMem || backrefInvalid(mem)) {opFail(); return;} + int pstart = backrefStart(mem); + int pend = backrefEnd(mem); + int n = pend - pstart; + if (s + n > range) {opFail(); return;} + sprev = s; + + while (n-- > 0) if (bytes[pstart++] != bytes[s++]) {opFail(); return;} + + if (sprev < range) { // beyond string check + int len; + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + } + + private void opBackRef1() { + backref(1); + } + + private void opBackRef2() { + backref(2); + } + + private void opBackRefN() { + backref(code[ip++]); + } + + private void opBackRefNIC() { + int mem = code[ip++]; + if (mem > regex.numMem || backrefInvalid(mem)) {opFail(); return;} + int pstart = backrefStart(mem); + int pend = backrefEnd(mem); + int n = pend - pstart; + if (s + n > range) {opFail(); return;} + sprev = s; + + value = s; + if (!stringCmpIC(regex.caseFoldFlag, pstart, this, n, end)) {opFail(); return;} + s = value; + + if (sprev < range) { + int len; + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + } + + private void opBackRefMulti() { + int tlen = code[ip++]; + + int i; + loop:for (i=0; i range) continue; + + sprev = s; + int swork = s; + + while (n-- > 0) { + if (bytes[pstart++] != bytes[swork++]) continue loop; + } + + s = swork; + + int len; + + // beyond string check + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + + ip += tlen - i - 1; // * SIZE_MEMNUM (1) + break; /* success */ + } + if (i == tlen) {opFail(); return;} + } + + private void opBackRefMultiIC() { + int tlen = code[ip++]; + + int i; + loop:for (i=0; i range) continue; + + sprev = s; + + value = s; + if (!stringCmpIC(regex.caseFoldFlag, pstart, this, n, end)) continue loop; // STRING_CMP_VALUE_IC + s = value; + + int len; + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + + ip += tlen - i - 1; // * SIZE_MEMNUM (1) + break; /* success */ + } + if (i == tlen) {opFail(); return;} + } + + private boolean memIsInMemp(int mem, int num, int memp) { + for (int i=0; i= 0) { + StackEntry e = stack[k]; + + if (e.type == CALL_FRAME) { + level--; + } else if (e.type == RETURN) { + level++; + } else if (level == nest) { + if (e.type == MEM_START) { + if (memIsInMemp(e.getMemNum(), memNum, memp)) { + int pstart = e.getMemPStr(); + if (pend != -1) { + if (pend - pstart > end - s) return false; /* or goto next_mem; */ + int p = pstart; + + value = s; + if (ignoreCase) { + if (!stringCmpIC(caseFoldFlag, pstart, this, pend - pstart, end)) { + return false; /* or goto next_mem; */ + } + } else { + while (p < pend) { + if (bytes[p++] != bytes[value++]) return false; /* or goto next_mem; */ + } + } + s = value; + + return true; + } + } + } else if (e.type == MEM_END) { + if (memIsInMemp(e.getMemNum(), memNum, memp)) { + pend = e.getMemPStr(); + } + } + } + k--; + } + return false; + } + + private void opBackRefAtLevel() { + int ic = code[ip++]; + int level = code[ip++]; + int tlen = code[ip++]; + + sprev = s; + if (backrefMatchAtNestedLevel(ic != 0, regex.caseFoldFlag, level, tlen, ip)) { // (s) and (end) implicit + int len; + if (sprev < range) { + while (sprev + (len = enc.length(bytes, sprev, end)) < s) sprev += len; + } + ip += tlen; // * SIZE_MEMNUM + } else { + {opFail(); return;} + } + } + + /* no need: IS_DYNAMIC_OPTION() == 0 */ + @SuppressWarnings("unused") + private void opSetOptionPush() { + // option = code[ip++]; // final for now + pushAlt(ip, s, sprev, pkeep); + ip += OPSize.SET_OPTION + OPSize.FAIL; + } + + @SuppressWarnings("unused") + private void opSetOption() { + // option = code[ip++]; // final for now + } + + private void opNullCheckStart() { + int mem = code[ip++]; + pushNullCheckStart(mem, s); + } + + private void nullCheckFound() { + // null_check_found: + /* empty loop founded, skip next instruction */ + switch(code[ip++]) { + case OPCode.JUMP: + case OPCode.PUSH: + ip++; // p += SIZE_RELADDR; + break; + case OPCode.REPEAT_INC: + case OPCode.REPEAT_INC_NG: + case OPCode.REPEAT_INC_SG: + case OPCode.REPEAT_INC_NG_SG: + ip++; // p += SIZE_MEMNUM; + break; + default: + throw new InternalException(ErrorMessages.UNEXPECTED_BYTECODE); + } // switch + } + + private void opNullCheckEnd() { + int mem = code[ip++]; + int isNull = nullCheck(mem, s); /* mem: null check id */ + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END: skip id:" + mem + ", s:" + s); + } + + nullCheckFound(); + } + } + + // USE_INFINITE_REPEAT_MONOMANIAC_MEM_STATUS_CHECK + private void opNullCheckEndMemST() { + int mem = code[ip++]; /* mem: null check id */ + int isNull = nullCheckMemSt(mem, s); + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END_MEMST: skip id:" + mem + ", s:" + s); + } + + if (isNull == -1) {opFail(); return;} + nullCheckFound(); + } + } + + // USE_SUBEXP_CALL + private void opNullCheckEndMemSTPush() { + int mem = code[ip++]; /* mem: null check id */ + + int isNull; + if (Config.USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT) { + isNull = nullCheckMemStRec(mem, s); + } else { + isNull = nullCheckRec(mem, s); + } + + if (isNull != 0) { + if (Config.DEBUG_MATCH) { + Config.log.println("NULL_CHECK_END_MEMST_PUSH: skip id:" + mem + ", s:" + s); + } + + if (isNull == -1) {opFail(); return;} + nullCheckFound(); + } else { + pushNullCheckEnd(mem); + } + } + + private void opJump() { + ip += code[ip] + 1; + } + + private void opPush() { + int addr = code[ip++]; + pushAlt(ip + addr, s, sprev, pkeep); + } + + // CEC + private void opStateCheckPush() { + int mem = code[ip++]; + if (stateCheckVal(s, mem)) {opFail(); return;} + int addr = code[ip++]; + pushAltWithStateCheck(ip + addr, s, sprev, mem, pkeep); + } + + // CEC + private void opStateCheckPushOrJump() { + int mem = code[ip++]; + int addr= code[ip++]; + + if (stateCheckVal(s, mem)) { + ip += addr; + } else { + pushAltWithStateCheck(ip + addr, s, sprev, mem, pkeep); + } + } + + // CEC + private void opStateCheck() { + int mem = code[ip++]; + if (stateCheckVal(s, mem)) {opFail(); return;} + pushStateCheck(s, mem); + } + + private void opPop() { + popOne(); + } + + private void opPushOrJumpExact1() { + int addr = code[ip++]; + // beyond string check + if (s < range && code[ip] == bytes[s]) { + ip++; + pushAlt(ip + addr, s, sprev, pkeep); + return; + } + ip += addr + 1; + } + + private void opPushIfPeekNext() { + int addr = code[ip++]; + // beyond string check + if (s < range && code[ip] == bytes[s]) { + ip++; + pushAlt(ip + addr, s, sprev, pkeep); + return; + } + ip++; + } + + private void opRepeat() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int addr= code[ip++]; + + // ensure1(); + repeatStk[mem] = stk; + pushRepeat(mem, ip); + + if (regex.repeatRangeLo[mem] == 0) { // lower + pushAlt(ip + addr, s, sprev, pkeep); + } + } + + private void opRepeatNG() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int addr= code[ip++]; + + // ensure1(); + repeatStk[mem] = stk; + pushRepeat(mem, ip); + + if (regex.repeatRangeLo[mem] == 0) { + pushAlt(ip, s, sprev, pkeep); + ip += addr; + } + } + + private void repeatInc(int mem, int si) { + StackEntry e = stack[si]; + + e.increaseRepeatCount(); + + if (e.getRepeatCount() >= regex.repeatRangeHi[mem]) { + /* end of repeat. Nothing to do. */ + } else if (e.getRepeatCount() >= regex.repeatRangeLo[mem]) { + pushAlt(ip, s, sprev, pkeep); + ip = e.getRepeatPCode(); /* Don't use stkp after PUSH. */ + } else { + ip = e.getRepeatPCode(); + } + pushRepeatInc(si); + } + + private void opRepeatInc() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int si = repeatStk[mem]; + repeatInc(mem, si); + } + + private void opRepeatIncSG() { + int mem = code[ip++]; /* mem: OP_REPEAT ID */ + int si = getRepeat(mem); + repeatInc(mem, si); + } + + private void repeatIncNG(int mem, int si) { + StackEntry e = stack[si]; + + e.increaseRepeatCount(); + + if (e.getRepeatCount() < regex.repeatRangeHi[mem]) { + if (e.getRepeatCount() >= regex.repeatRangeLo[mem]) { + int pcode = e.getRepeatPCode(); + pushRepeatInc(si); + pushAlt(pcode, s, sprev, pkeep); + } else { + ip = e.getRepeatPCode(); + pushRepeatInc(si); + } + } else if (e.getRepeatCount() == regex.repeatRangeHi[mem]) { + pushRepeatInc(si); + } + } + + private void opRepeatIncNG() { + int mem = code[ip++]; + int si = repeatStk[mem]; + repeatIncNG(mem, si); + } + + private void opRepeatIncNGSG() { + int mem = code[ip++]; + int si = getRepeat(mem); + repeatIncNG(mem, si); + } + + private void opPushPos() { + pushPos(s, sprev, pkeep); + } + + private void opPopPos() { + StackEntry e = stack[posEnd()]; + s = e.getStatePStr(); + sprev= e.getStatePStrPrev(); + } + + private void opPushPosNot() { + int addr = code[ip++]; + pushPosNot(ip + addr, s, sprev, pkeep); + } + + private void opFailPos() { + popTilPosNot(); + opFail(); + } + + private void opPushStopBT() { + pushStopBT(); + } + + private void opPopStopBT() { + stopBtEnd(); + } + + private void opLookBehind() { + int tlen = code[ip++]; + s = enc.stepBack(bytes, str, s, end, tlen); + if (s == -1) {opFail(); return;} + sprev = enc.prevCharHead(bytes, str, s, end); + } + + private void opLookBehindSb() { + int tlen = code[ip++]; + s -= tlen; + if (s < str) {opFail(); return;} + sprev = s == str ? -1 : s - 1; + } + + private void opPushLookBehindNot() { + int addr = code[ip++]; + int tlen = code[ip++]; + int q = enc.stepBack(bytes, str, s, end, tlen); + if (q == -1) { + /* too short case -> success. ex. /(? aend && s > absent) { + pop(); + opFail(); + return; + } else if (s >= aend && s > absent) { + if (s > aend || s > end) { + opFail(); + return; + } + ip += addr; + } else { + + pushAlt(ip + addr, s, sprev, pkeep); + int n = (s >= end) ? 1 : enc.length(bytes, s, end); + pushAbsentPos(absent, range); + pushAlt(selfip, s + n, s, pkeep); + pushAbsent(); + range = aend; + } + } + + private void opAbsentEnd() { + if (sprev < range) range = sprev; + if (Config.DEBUG_MATCH) System.out.println("ABSENT_END: end:" + range); + popTilAbsent(); + opFail(); + return; + // sprev = sbegin; // break; + } + + private void opCall() { + int addr = code[ip++]; + pushCallFrame(ip); + ip = addr; // absolute address + } + + private void opCallout() { + int calloutId = code[ip++]; + CalloutHandler handler = getCalloutHandler(); + if (handler == null) throw new IllegalStateException("callout pattern has no handler"); + CalloutResult result = handler.execute(calloutId, this); + if (result == null) throw new NullPointerException("callout result"); + if (result.getBacktrackToken() != null) pushCallout(result.getBacktrackToken()); + if (result.getAction() == CalloutAction.FAIL) opFail(); + } + + private void opCalloutCondition() { + int calloutId = code[ip++]; + int addr = code[ip++]; + CalloutHandler handler = getCalloutHandler(); + if (handler == null) throw new IllegalStateException("callout pattern has no handler"); + CalloutResult result = handler.execute(calloutId, this); + if (result == null) throw new NullPointerException("callout result"); + if (result.getBacktrackToken() != null) pushCallout(result.getBacktrackToken()); + if (result.getAction() == CalloutAction.FAIL) ip += addr; + } + + @Override + public int currentBytePosition() { + return s - str; + } + + @Override + public int captureCount() { + return regex.numMem; + } + + @Override + public int captureBegin(int capture) { + checkCapture(capture); + if (capture == 0) return sstart - str; + int value = repeatStk[memStartStk + capture]; + if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + return (bsAt(regex.btMemStart, capture) ? stack[value].getMemPStr() : value) - str; + } + + @Override + public int captureEnd(int capture) { + checkCapture(capture); + if (capture == 0) return s - str; + int value = repeatStk[memEndStk + capture]; + if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + return (bsAt(regex.btMemEnd, capture) ? stack[value].getMemPStr() : value) - str; + } + + private void checkCapture(int capture) { + if (capture < 0 || capture > regex.numMem) { + throw new IndexOutOfBoundsException("capture " + capture); + } + } + + private void opReturn() { + ip = sreturn(); + pushReturn(); + } + + private void opFail() { + if (stack == null) { + ip = regex.codeLength - 1; + return; + } + + + StackEntry e = pop(); + ip = e.getStatePCode(); + s = e.getStatePStr(); + sprev = e.getStatePStrPrev(); + pkeep = e.getPKeep(); + + if (USE_CEC) { + if (((SCStackEntry)e).getStateCheck() != 0) { + e.type = STATE_CHECK_MARK; + stk++; + } + } + } + + private int finish() { + return bestLen; + } + + private void debugMatchBegin() { + Config.log.println("match_at: " + "str: " + str + ", end: " + end + ", start: " + sstart + ", sprev: " + sprev); + Config.log.println("size: " + (end - str) + ", start offset: " + (sstart - str)); + } + + private void debugMatchLoop() { + Config.log.printf("%4d", (s - str)).print("> \""); + int q, i; + for (i = 0, q = s; i < 7 && q < end && s >= 0; i++) { + int len = enc.length(bytes, q, end); + while (len-- > 0) { + if (q < end) { + Config.log.print(new String(bytes, q++, 1)); + } + } + } + String str = q < end ? "...\"" : "\""; + q += str.length(); + Config.log.print(str); + for (i = 0; i < 20 - (q - s); i++) + Config.log.print(" "); + StringBuilder sb = new StringBuilder(); + new ByteCodePrinter(regex).compiledByteCodeToString(sb, ip); + Config.log.println(sb.toString()); + } +} diff --git a/third_party/joni/src/org/joni/ByteCodePrinter.java b/third_party/joni/src/org/joni/ByteCodePrinter.java new file mode 100644 index 000000000..80c4b7615 --- /dev/null +++ b/third_party/joni/src/org/joni/ByteCodePrinter.java @@ -0,0 +1,392 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.constants.internal.Arguments; +import org.joni.constants.internal.OPCode; +import org.joni.constants.internal.OPSize; +import org.joni.exception.InternalException; + +class ByteCodePrinter { + final int[] code; + final int codeLength; + final byte[][] templates; + final Encoding enc; + + public ByteCodePrinter(Regex regex) { + code = regex.code; + codeLength = regex.codeLength; + templates = regex.templates; + enc = regex.enc; + } + + public String byteCodeListToString() { + return compiledByteCodeListToString(); + } + + private void pString(StringBuilder sb, int len, int s) { + sb.append(':'); + while (len-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + } + + private void pLenString(StringBuilder sb, int len, int mbLen, int s) { + int x = len * mbLen; + sb.append(':').append(len).append(':'); + while (x-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + } + + private void pLenStringFromTemplate(StringBuilder sb, int len, int mbLen, byte[]tm, int idx) { + int x = len * mbLen; + sb.append(":T:").append(len).append(':'); + while (x-- > 0) sb.append(new String(tm, idx++, 1)); + } + + public int compiledByteCodeToString(StringBuilder sb, int bp) { + int len, n, mem, addr, scn, cod; + BitSet bs; + int tm, idx; + + sb.append('[').append(OPCode.OpCodeNames[code[bp]]); + int argType = OPCode.OpCodeArgTypes[code[bp]]; + int ip = bp; + if (argType != Arguments.SPECIAL) { + bp++; + switch (argType) { + case Arguments.NON: + break; + + case Arguments.RELADDR: + sb.append(":(").append(code[bp]).append(')'); + bp += OPSize.RELADDR; + break; + + case Arguments.ABSADDR: + sb.append(":(").append(code[bp]).append(')'); + bp += OPSize.ABSADDR; + break; + + case Arguments.LENGTH: + sb.append(':').append(code[bp]); + bp += OPSize.LENGTH; + break; + + case Arguments.MEMNUM: + sb.append(':').append(code[bp]); + bp += OPSize.MEMNUM; + break; + + case Arguments.OPTION: + sb.append(':').append(code[bp]); + bp += OPSize.OPTION; + break; + + case Arguments.STATE_CHECK: + sb.append(':').append(code[bp]); + bp += OPSize.STATE_CHECK; + break; + } + } else { + switch (code[bp++]) { + case OPCode.EXACT1: + case OPCode.ANYCHAR_STAR_PEEK_NEXT: + case OPCode.ANYCHAR_ML_STAR_PEEK_NEXT: + pString(sb, 1, bp++); + break; + + case OPCode.EXACT2: + pString(sb, 2, bp); + bp += 2; + break; + + case OPCode.EXACT3: + pString(sb, 3, bp); + bp += 3; + break; + + case OPCode.EXACT4: + pString(sb, 4, bp); + bp += 4; + break; + + case OPCode.EXACT5: + pString(sb, 5, bp); + bp += 5; + break; + + case OPCode.EXACTN: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 1, templates[tm], idx); + } else { + pLenString(sb, len, 1, bp); + bp += len; + } + break; + + case OPCode.EXACTMB2N1: + pString(sb, 2, bp); + bp += 2; + break; + + case OPCode.EXACTMB2N2: + pString(sb, 4, bp); + bp += 4; + break; + + case OPCode.EXACTMB2N3: + pString(sb, 6, bp); + bp += 6; + break; + + case OPCode.EXACTMB2N: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 2, templates[tm], idx); + } else { + pLenString(sb, len, 2, bp); + bp += len * 2; + } + break; + + case OPCode.EXACTMB3N: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 3, templates[tm], idx); + } else { + pLenString(sb, len, 3, bp); + bp += len * 3; + } + break; + + case OPCode.EXACTMBN: + int mbLen = code[bp]; + bp += OPSize.LENGTH; + len = code[bp]; + bp += OPSize.LENGTH; + n = len * mbLen; + + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + sb.append(":T:").append(mbLen).append(":").append(len).append(":"); + while (n-- > 0) sb.append(new String(templates[tm], idx++, 1)); + } else { + sb.append(":").append(mbLen).append(":").append(len).append(":"); + while (n-- > 0) sb.append(new String(new byte[]{(byte)code[bp++]})); + } + + break; + + case OPCode.EXACT1_IC: + case OPCode.EXACT1_IC_SB: + final int MAX_CHAR_LENGTH = 6; + byte[]bytes = new byte[MAX_CHAR_LENGTH]; + for (int i = 0; bp + i < code.length && i < MAX_CHAR_LENGTH; i++) bytes[i] = (byte)code[bp + i]; + len = enc.length(bytes, 0, MAX_CHAR_LENGTH); + pString(sb, len, bp); + bp += len; + break; + + case OPCode.EXACTN_IC: + case OPCode.EXACTN_IC_SB: + len = code[bp]; + bp += OPSize.LENGTH; + if (Config.USE_STRING_TEMPLATES) { + tm = code[bp]; + bp += OPSize.INDEX; + idx = code[bp]; + bp += OPSize.INDEX; + pLenStringFromTemplate(sb, len, 1, templates[tm], idx); + } else { + pLenString(sb, len, 1, bp); + bp += len; + } + break; + + case OPCode.CCLASS: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + sb.append(':').append(n); + break; + + case OPCode.CCLASS_NOT: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + sb.append(':').append(n); + break; + + case OPCode.CCLASS_MB: + case OPCode.CCLASS_MB_NOT: + len = code[bp]; + bp += OPSize.LENGTH; + cod = code[bp]; + //bp += OPSize.CODE_POINT; + bp += len; + sb.append(':').append(cod).append(':').append(len); + break; + + case OPCode.CCLASS_MIX: + case OPCode.CCLASS_MIX_NOT: + bs = new BitSet(); + System.arraycopy(code, bp, bs.bits, 0, BitSet.BITSET_SIZE); + n = bs.numOn(); + bp += BitSet.BITSET_SIZE; + len = code[bp]; + bp += OPSize.LENGTH; + cod = code[bp]; + //bp += OPSize.CODE_POINT; + bp += len; + sb.append(':').append(n).append(':').append(cod).append(':').append(len); + break; + + case OPCode.BACKREFN_IC: + mem = code[bp]; + bp += OPSize.MEMNUM; + sb.append(':').append(mem); + break; + + case OPCode.BACKREF_MULTI_IC: + case OPCode.BACKREF_MULTI: + sb.append(' '); + len = code[bp]; + bp += OPSize.LENGTH; + for (int i=0; i 0) sb.append(", "); + sb.append(mem); + } + break; + + case OPCode.BACKREF_WITH_LEVEL: { + int option = code[bp]; + bp += OPSize.OPTION; + sb.append(':').append(option); + int level = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(level); + sb.append(' '); + len = code[bp]; + bp += OPSize.LENGTH; + for (int i=0; i 0) sb.append(", "); + sb.append(mem); + } + break; + } + + case OPCode.REPEAT: + case OPCode.REPEAT_NG: + mem = code[bp]; + bp += OPSize.MEMNUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(mem).append(':').append(addr); + break; + + case OPCode.PUSH_OR_JUMP_EXACT1: + case OPCode.PUSH_IF_PEEK_NEXT: + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(":(").append(addr).append(')'); + pString(sb, 1, bp); + bp++; + break; + + case OPCode.LOOK_BEHIND: + len = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(len); + break; + + case OPCode.PUSH_LOOK_BEHIND_NOT: + addr = code[bp]; + bp += OPSize.RELADDR; + len = code[bp]; + bp += OPSize.LENGTH; + sb.append(':').append(len).append(":(").append(addr).append(')'); + break; + + case OPCode.STATE_CHECK_PUSH: + case OPCode.STATE_CHECK_PUSH_OR_JUMP: + scn = code[bp]; + bp += OPSize.STATE_CHECK_NUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(scn).append(":(").append(addr).append(')'); + break; + + case OPCode.CONDITION: + mem = code[bp]; + bp += OPSize.MEMNUM; + addr = code[bp]; + bp += OPSize.RELADDR; + sb.append(':').append(mem).append(":").append(addr); + break; + + default: + throw new InternalException("undefined code: " + code[--bp]); + } + } + + sb.append(']'); + if (Config.DEBUG_COMPILE_BYTE_CODE_INFO) sb.append('@').append(ip).append('(').append(bp - ip).append(')'); + return bp; + } + + private String compiledByteCodeListToString() { + StringBuilder sb = new StringBuilder("code length: ").append(codeLength).append('\n'); + int ncode = -1; + int bp = 0; + int end = codeLength; + + while (bp < end) { + ncode++; + sb.append(ncode % 5 == 0 ? '\n' : ' '); + bp = compiledByteCodeToString(sb, bp); + } + sb.append("\n"); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/CalloutAction.java b/third_party/joni/src/org/joni/CalloutAction.java new file mode 100644 index 000000000..a92d779f2 --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutAction.java @@ -0,0 +1,26 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Match control requested by a generic callout handler. */ +public enum CalloutAction { + CONTINUE, + FAIL +} diff --git a/third_party/joni/src/org/joni/CalloutHandler.java b/third_party/joni/src/org/joni/CalloutHandler.java new file mode 100644 index 000000000..8ab64a7a1 --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutHandler.java @@ -0,0 +1,32 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Runtime-neutral bridge invoked from the matcher bytecode loop. */ +public interface CalloutHandler { + CalloutResult execute(int calloutId, MatchView match); + + void unwind(Object backtrackToken); + + /** Complete a token on a successful path. The default preserves the original cleanup contract. */ + default void complete(Object backtrackToken) { + unwind(backtrackToken); + } +} diff --git a/third_party/joni/src/org/joni/CalloutResult.java b/third_party/joni/src/org/joni/CalloutResult.java new file mode 100644 index 000000000..194ee57bc --- /dev/null +++ b/third_party/joni/src/org/joni/CalloutResult.java @@ -0,0 +1,53 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Immutable result of a match-time callout. */ +public final class CalloutResult { + public static final CalloutResult CONTINUE = new CalloutResult(CalloutAction.CONTINUE, null); + public static final CalloutResult FAIL = new CalloutResult(CalloutAction.FAIL, null); + + private final CalloutAction action; + private final Object backtrackToken; + + private CalloutResult(CalloutAction action, Object backtrackToken) { + if (action == null) throw new NullPointerException("action"); + this.action = action; + this.backtrackToken = backtrackToken; + } + + public static CalloutResult continueWith(Object backtrackToken) { + return backtrackToken == null ? CONTINUE + : new CalloutResult(CalloutAction.CONTINUE, backtrackToken); + } + + public static CalloutResult failWith(Object backtrackToken) { + return backtrackToken == null ? FAIL + : new CalloutResult(CalloutAction.FAIL, backtrackToken); + } + + public CalloutAction getAction() { + return action; + } + + public Object getBacktrackToken() { + return backtrackToken; + } +} diff --git a/third_party/joni/src/org/joni/CaptureTreeNode.java b/third_party/joni/src/org/joni/CaptureTreeNode.java new file mode 100644 index 000000000..ff0c52fde --- /dev/null +++ b/third_party/joni/src/org/joni/CaptureTreeNode.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +final class CaptureTreeNode { + int group; + int beg; + int end; + // int allocated; + int numChildren; + CaptureTreeNode[] children; + + CaptureTreeNode() { + beg = Region.REGION_NOTPOS; + end = Region.REGION_NOTPOS; + group = -1; + } + + static final int HISTORY_TREE_INIT_ALLOC_SIZE = 8; + + void addChild(CaptureTreeNode child) { + if (children == null) { + children = new CaptureTreeNode[HISTORY_TREE_INIT_ALLOC_SIZE]; + } else if (numChildren >= children.length) { + CaptureTreeNode[] tmp = new CaptureTreeNode[children.length << 1]; + System.arraycopy(children, 0, tmp, 0, children.length); + children = tmp; + } + + children[numChildren] = child; + numChildren++; + } + + void clear() { + for (int i = 0; i < numChildren; i++) { + children[i] = null; // ??? + } + numChildren = 0; + beg = end = Region.REGION_NOTPOS; + group = -1; + } + + CaptureTreeNode cloneTree() { + CaptureTreeNode clone = new CaptureTreeNode(); + clone.beg = beg; + clone.end = end; + + for (int i = 0; i < numChildren; i++) { + CaptureTreeNode child = children[i].cloneTree(); + clone.addChild(child); + } + return clone; + } +} diff --git a/third_party/joni/src/org/joni/CodeRangeBuffer.java b/third_party/joni/src/org/joni/CodeRangeBuffer.java new file mode 100644 index 000000000..c5d0e6ca4 --- /dev/null +++ b/third_party/joni/src/org/joni/CodeRangeBuffer.java @@ -0,0 +1,403 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.exception.ErrorMessages; +import org.joni.exception.ValueException; + +public final class CodeRangeBuffer { + private static final int INIT_MULTI_BYTE_RANGE_SIZE = 5; + public static final int LAST_CODE_POINT = 0x7fffffff; + + private int[]p; + private int used; + + public CodeRangeBuffer() { + p = new int[INIT_MULTI_BYTE_RANGE_SIZE]; + writeCodePoint(0, 0); + } + + public int[]getCodeRange() { + return p; + } + + public int getUsed() { + return used; + } + + private CodeRangeBuffer(CodeRangeBuffer orig) { + p = new int[orig.p.length]; + System.arraycopy(orig.p, 0, p, 0, p.length); + used = orig.used; + } + + public void expand(int low) { + int length = p.length; + do { length <<= 1; } while (length < low); + int[]tmp = new int[length]; + System.arraycopy(p, 0, tmp, 0, used); + p = tmp; + } + + public void ensureSize(int size) { + int length = p.length; + while (length < size ) { length <<= 1; } + if (p.length != length) { + int[]tmp = new int[length]; + System.arraycopy(p, 0, tmp, 0, used); + p = tmp; + } + } + + private void moveRight(int from, int to, int n) { + if (to + n > p.length) expand(to + n); + System.arraycopy(p, from, p, to, n); + if (to + n > used) used = to + n; + } + + protected void moveLeft(int from, int to, int n) { + System.arraycopy(p, from, p, to, n); + } + + private void moveLeftAndReduce(int from, int to) { + System.arraycopy(p, from, p, to, used - from); + used -= from - to; + } + + public void writeCodePoint(int pos, int b) { + int u = pos + 1; + if (p.length < u) expand(u); + p[pos] = b; + if (used < u) used = u; + } + + @Override + public CodeRangeBuffer clone() { + return new CodeRangeBuffer(this); + } + + // add_code_range_to_buf + public static CodeRangeBuffer addCodeRangeToBuff(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to) { + return addCodeRangeToBuff(pbuf, env, from, to, true); + } + + public static CodeRangeBuffer addCodeRangeToBuff(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to, boolean checkDup) { + if (from > to) { + int n = from; + from = to; + to = n; + } + + int n; + + if (pbuf == null) { + pbuf = new CodeRangeBuffer(); // move to CClassNode + n = 0; + } else { + n = pbuf.p[0]; + } + + int[] data = pbuf.p; // data = bbuf.p + final int dataOff = 1; // data++ + + int bound = from == 0 ? 0 : n; + int low; + for (low = 0; low < bound; ) { + int x = (low + bound) >>> 1; + if (from - 1 > data[dataOff + x * 2 + 1]) { + low = x + 1; + } else { + bound = x; + } + } + + int high = to == LAST_CODE_POINT ? n : low; + for (bound = n; high < bound; ) { + int x = (high + bound) >>> 1; + if (to + 1 >= data[dataOff + x * 2]) { + high = x + 1; + } else { + bound = x; + } + } + + int incN = low + 1 - high; + + if (n + incN > Config.MAX_MULTI_BYTE_RANGES_NUM) throw new ValueException(ErrorMessages.TOO_MANY_MULTI_BYTE_RANGES); + + if (incN != 1) { + if (checkDup) { + if (from <= data[dataOff + low * 2 + 1] && (data[dataOff + low * 2] <= from || data[dataOff + low * 2 + 1] <= to)) env.ccDuplicateWarn(); + } + + if (from > data[dataOff + low * 2]) from = data[dataOff + low * 2]; + if (to < data[dataOff + (high - 1) * 2 + 1]) to = data[dataOff + (high - 1) * 2 + 1]; + } + + if (incN != 0) { + int fromPos = 1 + high * 2; + int toPos = 1 + (low + 1) * 2; + + if (incN > 0) { + if (high < n) { + int size = (n - high) * 2; + pbuf.moveRight(fromPos, toPos, size); + } + } else { + pbuf.moveLeftAndReduce(fromPos, toPos); + } + } + + int pos = 1 + low * 2; + // pbuf.ensureSize(pos + 2); + pbuf.writeCodePoint(pos, from); + pbuf.writeCodePoint(pos + 1, to); + n += incN; + pbuf.writeCodePoint(0, n); + + return pbuf; + } + + // add_code_range, be aware of it returning null! + public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to) { + return addCodeRange(pbuf, env, from, to, true); + } + + public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment env, int from, int to, boolean checkDup) { + if (from >to) { + if (env.syntax.allowEmptyRangeInCC()) { + return pbuf; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + return addCodeRangeToBuff(pbuf, env, from, to, checkDup); + } + + private static int mbcodeStartPosition(Encoding enc) { + return enc.minLength() > 1 ? 0 : 0x80; + } + + + // SET_ALL_MULTI_BYTE_RANGE + protected static CodeRangeBuffer setAllMultiByteRange(ScanEnvironment env, CodeRangeBuffer pbuf) { + return addCodeRangeToBuff(pbuf, env, mbcodeStartPosition(env.enc), LAST_CODE_POINT); + } + + // ADD_ALL_MULTI_BYTE_RANGE + public static CodeRangeBuffer addAllMultiByteRange(ScanEnvironment env, CodeRangeBuffer pbuf) { + if (!env.enc.isSingleByte()) return setAllMultiByteRange(env, pbuf); + return pbuf; + } + + // not_code_range_buf + public static CodeRangeBuffer notCodeRangeBuff(ScanEnvironment env, CodeRangeBuffer bbuf) { + CodeRangeBuffer pbuf = null; + + if (bbuf == null) return setAllMultiByteRange(env, pbuf); + + int[]p = bbuf.p; + int n = p[0]; + + if (n <= 0) return setAllMultiByteRange(env, pbuf); + + int pre = mbcodeStartPosition(env.enc); + + int from; + int to = 0; + for (int i=0; i to1) break; + } + + if (from1 <= to1) { + pbuf = addCodeRangeToBuff(pbuf, env, from1, to1); + } + + return pbuf; + } + + // and_code_range_buf + public static CodeRangeBuffer andCodeRangeBuff(CodeRangeBuffer bbuf1, boolean not1, + CodeRangeBuffer bbuf2, boolean not2, ScanEnvironment env) { + CodeRangeBuffer pbuf = null; + + if (bbuf1 == null) { + if (not1 && bbuf2 != null) return bbuf2.clone(); /* not1 != 0 -> not2 == 0 */ + return null; + } else if (bbuf2 == null) { + if (not2) return bbuf1.clone(); + return null; + } + + if (not1) { + CodeRangeBuffer tbuf; + boolean tnot; + // swap + tnot = not1; not1 = not2; not2 = tnot; + tbuf = bbuf1; bbuf1 = bbuf2; bbuf2 = tbuf; + } + + int[]p1 = bbuf1.p; + int n1 = p1[0]; + int[]p2 = bbuf2.p; + int n2 = p2[0]; + + if (!not2 && !not1) { /* 1 AND 2 */ + for (int i=0; i to1) break; + if (to2 < from1) continue; + int from = from1 > from2 ? from1 : from2; + int to = to1 < to2 ? to1 : to2; + pbuf = addCodeRangeToBuff(pbuf, env, from, to); + } + } + } else if (!not1) { /* 1 AND (not 2) */ + for (int i=0; i 0 && i % 6 == 0) buf.append("\n "); + } + + return buf.toString(); + } + + private static String rangeNumToString(int num){ + return "0x" + Integer.toString(num, 16); + } +} diff --git a/third_party/joni/src/org/joni/Compiler.java b/third_party/joni/src/org/joni/Compiler.java new file mode 100644 index 000000000..2cffd94ba --- /dev/null +++ b/third_party/joni/src/org/joni/Compiler.java @@ -0,0 +1,196 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.joni.ast.AnchorNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.ListNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.NodeType; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.SyntaxException; + +abstract class Compiler implements ErrorMessages { + protected final Analyser analyser; + protected final Encoding enc; + protected final Regex regex; + + protected Compiler(Analyser analyser) { + this.analyser = analyser; + this.regex = analyser.regex; + this.enc = regex.enc; + } + + final void compile(Node root) { + prepare(); + compileTree(root); + finish(); + } + + protected abstract void prepare(); + protected abstract void finish(); + + protected abstract void compileAltNode(ListNode node); + + private void compileStringRawNode(StringNode sn) { + if (sn.length() <= 0) return; + addCompileString(sn.bytes, sn.p, 1 /*sb*/, sn.length(), false); + } + + private void compileStringNode(StringNode node) { + StringNode sn = node; + if (sn.length() <= 0) return; + + boolean ambig = sn.isAmbig(); + + int p, prev; + p = prev = sn.p; + int end = sn.end; + byte[]bytes = sn.bytes; + int prevLen = enc.length(bytes, p, end); + p += prevLen; + int blen = prevLen; + + while (p < end) { + int len = enc.length(bytes, p, end); + if (len == prevLen || ambig) { + blen += len; + } else { + addCompileString(bytes, prev, prevLen, blen, ambig); + prev = p; + blen = len; + prevLen = len; + } + p += len; + } + addCompileString(bytes, prev, prevLen, blen, ambig); + } + + protected abstract void addCompileString(byte[]bytes, int p, int mbLength, int strLength, boolean ignoreCase); + + protected abstract void compileCClassNode(CClassNode node); + protected abstract void compileCTypeNode(CTypeNode node); + protected abstract void compileAnyCharNode(); + protected abstract void compileCallNode(CallNode node); + protected abstract void compileCalloutNode(CalloutNode node); + protected abstract void compileBackrefNode(BackRefNode node); + protected abstract void compileCECQuantifierNode(QuantifierNode node); + protected abstract void compileNonCECQuantifierNode(QuantifierNode node); + protected abstract void compileOptionNode(EncloseNode node); + protected abstract void compileEncloseNode(EncloseNode node); + protected abstract void compileAnchorNode(AnchorNode node); + + protected final void compileTree(Node node) { + if (node instanceof CalloutNode) { + compileCalloutNode((CalloutNode)node); + return; + } + switch (node.getType()) { + case NodeType.LIST: + ListNode lin = (ListNode)node; + do { + compileTree(lin.value); + } while ((lin = lin.tail) != null); + break; + + case NodeType.ALT: + compileAltNode((ListNode)node); + break; + + case NodeType.STR: + StringNode sn = (StringNode)node; + if (sn.isRaw()) { + compileStringRawNode(sn); + } else { + compileStringNode(sn); + } + break; + + case NodeType.CCLASS: + compileCClassNode((CClassNode)node); + break; + + case NodeType.CTYPE: + compileCTypeNode((CTypeNode)node); + break; + + case NodeType.CANY: + compileAnyCharNode(); + break; + + case NodeType.BREF: + compileBackrefNode((BackRefNode)node); + break; + + case NodeType.CALL: + if (Config.USE_SUBEXP_CALL) { + compileCallNode((CallNode)node); + break; + } // USE_SUBEXP_CALL + break; + + case NodeType.QTFR: + if (Config.USE_CEC) { + compileCECQuantifierNode((QuantifierNode)node); + } else { + compileNonCECQuantifierNode((QuantifierNode)node); + } + break; + + case NodeType.ENCLOSE: + EncloseNode enode = (EncloseNode)node; + if (enode.isOption()) { + compileOptionNode(enode); + } else { + compileEncloseNode(enode); + } + break; + + case NodeType.ANCHOR: + compileAnchorNode((AnchorNode)node); + break; + + default: + // undefined node type + newInternalException(PARSER_BUG); + } // switch + } + + protected final void compileTreeNTimes(Node node, int n) { + for (int i=0; i, \k */ + + boolean USE_MONOMANIAC_CHECK_CAPTURES_IN_ENDLESS_REPEAT = ConfigSupport.getBoolean("joni.use_monomaniac_check_captures_in_endless_repeat", true); /* /(?:()|())*\2/ */ + boolean USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE = ConfigSupport.getBoolean("joni.use_newline_at_end_of_string_has_empty_line", true); /* /\n$/ =~ "\n" */ + boolean USE_WARNING_REDUNDANT_NESTED_REPEAT_OPERATOR = ConfigSupport.getBoolean("joni.use_warning_redundant_nested_repeat_operator", true); + + boolean CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS = ConfigSupport.getBoolean("joni.case_fold_is_applied_inside_negative_cclass", true); + + boolean USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE = ConfigSupport.getBoolean("joni.use_match_range_must_be_inside_of_specified_range", false); + boolean USE_CAPTURE_HISTORY = ConfigSupport.getBoolean("joni.use_capture_history", false); + boolean USE_VARIABLE_META_CHARS = ConfigSupport.getBoolean("joni.use_variable_meta_chars", true); + boolean USE_WORD_BEGIN_END = ConfigSupport.getBoolean("joni.use_word_begin_end", true); /* "\<": word-begin, "\>": word-end */ + boolean USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE = ConfigSupport.getBoolean("joni.use_find_longest_search_all_of_range", true); + boolean USE_SUNDAY_QUICK_SEARCH = ConfigSupport.getBoolean("joni.use_sunday_quick_search", true); + boolean USE_CEC = ConfigSupport.getBoolean("joni.use_cec", false); + boolean USE_DYNAMIC_OPTION = ConfigSupport.getBoolean("joni.use_dynamic_option", false); + boolean USE_BYTE_MAP = ConfigSupport.getBoolean("joni.use_byte_map", OptExactInfo.OPT_EXACT_MAXLEN <= CHAR_TABLE_SIZE); + boolean USE_INT_MAP_BACKWARD = ConfigSupport.getBoolean("joni.use_int_map_backward", false); + + int NREGION = ConfigSupport.getInt("joni.nregion", 10); + int MAX_BACKREF_NUM = ConfigSupport.getInt("joni.max_backref_num", 1000); + int MAX_CAPTURE_GROUP_NUM = ConfigSupport.getInt("joni.max_capture_group_num", 32767); + int MAX_REPEAT_NUM = ConfigSupport.getInt("joni.max_multi_byte_ranges_num", 100000); + int MAX_MULTI_BYTE_RANGES_NUM = ConfigSupport.getInt("joni.max_multi_byte_ranges_num", 10000); + + // internal config + boolean USE_OP_PUSH_OR_JUMP_EXACT = ConfigSupport.getBoolean("joni.use_op_push_or_jump_exact", true); + boolean USE_QTFR_PEEK_NEXT = ConfigSupport.getBoolean("joni.use_qtfr_peek_next", true); + + int INIT_MATCH_STACK_SIZE = ConfigSupport.getInt("joni.init_match_stack_size", 64); + + boolean OPTIMIZE = ConfigSupport.getBoolean("joni.optimize", true); + @Deprecated boolean DONT_OPTIMIZE = !OPTIMIZE; + + // use embedded string templates in Regex object as byte arrays instead of compiling them into int bytecode array + boolean USE_STRING_TEMPLATES = ConfigSupport.getBoolean("joni.use_string_templates", true); + + + int MAX_CAPTURE_HISTORY_GROUP = ConfigSupport.getInt("joni.max_capture_history_group", 31); + + + int CHECK_STRING_THRESHOLD_LEN = ConfigSupport.getInt("joni.check_string_threshold_len", 7); + int CHECK_BUFF_MAX_SIZE = ConfigSupport.getInt("joni.check_buff_max_size", 0x4000); + + PrintStream log = System.out; + PrintStream err = System.err; + + boolean DEBUG_ALL = ConfigSupport.getBoolean("joni.debug.all", false); + + boolean DEBUG = ConfigSupport.getBoolean("joni.debug", false) || DEBUG_ALL; + boolean DEBUG_PARSE_TREE = ConfigSupport.getBoolean("joni.debug.parse.tree", false) || DEBUG_ALL; + boolean DEBUG_PARSE_TREE_RAW = ConfigSupport.getBoolean("joni.debug.parse.tree.raw", true) || DEBUG_ALL; + boolean DEBUG_COMPILE = ConfigSupport.getBoolean("joni.debug.compile", false) || DEBUG_ALL; + boolean DEBUG_COMPILE_BYTE_CODE_INFO = ConfigSupport.getBoolean("joni.debug.compile.bytecode.info", false) || DEBUG_ALL; + boolean DEBUG_SEARCH = ConfigSupport.getBoolean("joni.debug.search", false) || DEBUG_ALL; + boolean DEBUG_MATCH = ConfigSupport.getBoolean("joni.debug.match", false) || DEBUG_ALL; +} diff --git a/third_party/joni/src/org/joni/ConfigSupport.java b/third_party/joni/src/org/joni/ConfigSupport.java new file mode 100644 index 000000000..99e263352 --- /dev/null +++ b/third_party/joni/src/org/joni/ConfigSupport.java @@ -0,0 +1,13 @@ +package org.joni; + +public class ConfigSupport { + static boolean getBoolean(String property, boolean def) { + String value = System.getProperty(property, def ? "true" : "false"); + return !value.equals("false"); + } + + static int getInt(String property, int def) { + String value = System.getProperty(property); + return value != null ? Integer.parseInt(value) : def; + } +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java new file mode 100644 index 000000000..d9ed2b1cb --- /dev/null +++ b/third_party/joni/src/org/joni/Lexer.java @@ -0,0 +1,1336 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.Option.isAsciiRange; +import static org.joni.Option.isSingleline; +import static org.joni.Option.isWordBoundAllRange; +import static org.joni.ast.QuantifierNode.isRepeatInfinite; + +import org.jcodings.Encoding; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.jcodings.exception.CharacterPropertyException; +import org.jcodings.exception.EncodingError; +import org.joni.ast.QuantifierNode; +import org.joni.constants.MetaChar; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.TokenType; +import org.joni.exception.ErrorMessages; + +class Lexer extends ScannerSupport { + protected final Regex regex; + protected final ScanEnvironment env; + protected final Syntax syntax; // fast access to syntax + protected final Token token = new Token(); // current token + + protected Lexer(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex.enc, bytes, p, end); + this.regex = regex; + this.env = new ScanEnvironment(regex, syntax, warnings); + this.syntax = env.syntax; + } + + /** + * @return 0: normal {n,m}, 2: fixed {n} + * !introduce returnCode here + */ + private int fetchRangeQuantifier() { + mark(); + boolean synAllow = syntax.allowInvalidInterval(); + + if (!left()) { + if (synAllow) { + return 1; /* "....{" : OK! */ + } else { + newSyntaxException(END_PATTERN_AT_LEFT_BRACE); + } + } + + if (!synAllow) { + c = peek(); + if (c == ')' || c == '(' || c == '|') { + newSyntaxException(END_PATTERN_AT_LEFT_BRACE); + } + } + + int low = scanUnsignedNumber(); + if (low < 0) newSyntaxException(ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + if (low > Config.MAX_REPEAT_NUM) newSyntaxException(ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + + boolean nonLow = false; + if (p == _p) { /* can't read low */ + if (syntax.allowIntervalLowAbbrev()) { + low = 0; + nonLow = true; + } else { + return invalidRangeQuantifier(synAllow); + } + } + + if (!left()) return invalidRangeQuantifier(synAllow); + + fetch(); + int up; + int ret = 0; + if (c == ',') { + int prev = p; // ??? last + up = scanUnsignedNumber(); + if (up < 0) newValueException(TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + if (up > Config.MAX_REPEAT_NUM) newValueException(TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + + if (p == prev) { + if (nonLow) return invalidRangeQuantifier(synAllow); + up = QuantifierNode.REPEAT_INFINITE; /* {n,} : {n,infinite} */ + } + } else { + if (nonLow) return invalidRangeQuantifier(synAllow); + unfetch(); + up = low; /* {n} : exact n times */ + ret = 2; /* fixed */ + } + + if (!left()) return invalidRangeQuantifier(synAllow); + fetch(); + + if (syntax.opEscBraceInterval()) { + if (c != syntax.metaCharTable.esc) return invalidRangeQuantifier(synAllow); + if (!left()) return invalidRangeQuantifier(synAllow); + fetch(); + } + + if (c != '}') return invalidRangeQuantifier(synAllow); + + if (!isRepeatInfinite(up) && low > up) { + newValueException(UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE); + } + + token.type = TokenType.INTERVAL; + token.setRepeatLower(low); + token.setRepeatUpper(up); + + return ret; /* 0: normal {n,m}, 2: fixed {n} */ + } + + private int invalidRangeQuantifier(boolean synAllow) { + if (synAllow) { + restore(); + return 1; + } else { + newSyntaxException(INVALID_REPEAT_RANGE_PATTERN); + return 0; // not reached + } + } + + /* \M-, \C-, \c, or \... */ + private void fetchEscapedValue() { + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + fetch(); + + switch(c) { + + case 'M': + if (syntax.op2EscCapitalMBarMeta()) { + if (!left()) newSyntaxException(END_PATTERN_AT_META); + fetch(); + if (c != '-') newSyntaxException(META_CODE_SYNTAX); + if (!left()) newSyntaxException(END_PATTERN_AT_META); + fetch(); + if (c == syntax.metaCharTable.esc) fetchEscapedValue(); + c = ((c & 0xff) | 0x80); + } else { + fetchEscapedValueBackSlash(); + } + break; + + case 'C': + if (syntax.op2EscCapitalCBarControl()) { + if (!left()) newSyntaxException(END_PATTERN_AT_CONTROL); + fetch(); + if (c != '-') newSyntaxException(CONTROL_CODE_SYNTAX); + fetchEscapedValueControl(); + } else { + fetchEscapedValueBackSlash(); + } + break; + + case 'c': + if (syntax.opEscCControl()) { + fetchEscapedValueControl(); + } + /* fall through */ + + default: + fetchEscapedValueBackSlash(); + } // switch + } + + private void fetchEscapedValueBackSlash() { + c = env.convertBackslashValue(c); + } + + private void fetchEscapedValueControl() { + if (!left()) { + if (syntax.op3OptionECMAScript()) { + return; + } else { + newSyntaxException(END_PATTERN_AT_CONTROL); + } + } + fetch(); + if (c == '?') { + c = 0177; + } else { + if (c == syntax.metaCharTable.esc) fetchEscapedValue(); + c &= 0x9f; + } + } + + private int nameEndCodePoint(int start) { + switch(start) { + case '<': + return '>'; + case '\'': + return '\''; + case '(': + return ')'; + case '{': + return '}'; + default: + return 0; + } + } + + // USE_NAMED_GROUP && USE_BACKREF_AT_LEVEL + /* + \k, \k + \k, \k + \k<-num+n>, \k<-num-n> + */ + + // value implicit (rnameEnd) + private boolean fetchNameWithLevel(int startCode, Ptr rbackNum, Ptr rlevel) { + int src = p; + boolean existLevel = false; + int isNum = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + if (enc.isDigit(c)) { + isNum = 1; + } else if (c == '-') { + isNum = 2; + sign = -1; + pnumHead = p; + } + } + + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')' || c == '+' || c == '-') { + if (isNum == 2) err = INVALID_GROUP_NAME; + break; + } + + if (isNum != 0) { + if (enc.isDigit(c)) { + isNum = 1; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } + } + + boolean isEndCode = false; + if (err == null && c != endCode) { + if (c == '+' || c == '-') { + int flag = c == '-' ? -1 : 1; + + if (!left()) newValueException(INVALID_CHAR_IN_GROUP_NAME); + fetch(); + if (!enc.isDigit(c)) newValueException(INVALID_GROUP_NAME, src, stop); + unfetch(); + int level = scanUnsignedNumber(); + if (level < 0) newValueException(TOO_BIG_NUMBER); + rlevel.p = level * flag; + existLevel = true; + + if (left()) { + fetch(); + isEndCode = c == endCode; + } + } + + if (!isEndCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + } + } + + if (err == null) { + if (isNum != 0) { + mark(); + p = pnumHead; + int backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0) { + newValueException(INVALID_GROUP_NAME, src, stop); + } + rbackNum.p = backNum * sign; + } + value = nameEnd; + return existLevel; + } else { + newValueException(INVALID_GROUP_NAME, src, nameEnd); + return false; // not reached + } + } + + // USE_NAMED_GROUP + // ref: 0 -> define name (don't allow number name) + // 1 -> reference name (allow number name) + private int fetchNameForNamedGroup(int startCode, boolean ref) { + int src = p; + value = 0; + + int isNum = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + if (enc.isDigit(c)) { + if (ref) { + isNum = 1; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } else if (c == '-') { + if (ref) { + isNum = 2; + sign = -1; + pnumHead = p; + } else { + err = INVALID_GROUP_NAME; + // isNum = 0; + } + } + } + + if (err == null) { + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')') { + if (isNum == 2) { + err = INVALID_GROUP_NAME; + return fetchNameTeardown(src, endCode, nameEnd, err); + } + break; + } + + if (isNum != 0) { + if (enc.isDigit(c)) { + isNum = 1; + } else { + if (!enc.isWord(c)) { + err = INVALID_CHAR_IN_GROUP_NAME; + } else { + err = INVALID_GROUP_NAME; + } + return fetchNameTeardown(src, endCode, nameEnd, err); + } + } + } + + if (c != endCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + return fetchNameErr(src, nameEnd, err); + } + + int backNum = 0; + if (isNum != 0) { + mark(); + p = pnumHead; + backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0) { + newValueException(INVALID_GROUP_NAME, src, nameEnd); + } + backNum *= sign; + } + value = nameEnd; + return backNum; + } else { + return fetchNameTeardown(src, endCode, nameEnd, err); + } + } + + private int fetchNameErr(int src, int nameEnd, String err) { + newValueException(err, src, nameEnd); + return 0; // not reached + } + + private int fetchNameTeardown(int src, int endCode, int nameEnd, String err) { + while (left()) { + nameEnd = p; + fetch(); + if (c == endCode || c == ')') break; + } + if (!left()) nameEnd = stop; + return fetchNameErr(src, nameEnd, err); + } + + // #else USE_NAMED_GROUP + // make it return nameEnd! + private final int fetchNameForNoNamedGroup(int startCode, boolean ref) { + int src = p; + value = 0; + int sign = 1; + + int endCode = nameEndCodePoint(startCode); + int pnumHead = p; + int nameEnd = stop; + + String err = null; + if (!left()) { + newValueException(EMPTY_GROUP_NAME); + } else { + fetch(); + if (c == endCode) newValueException(EMPTY_GROUP_NAME); + + if (enc.isDigit(c)) { + } else if (c == '-') { + sign = -1; + pnumHead = p; + } else { + err = INVALID_CHAR_IN_GROUP_NAME; + } + } + + while(left()) { + nameEnd = p; + + fetch(); + if (c == endCode || c == ')') break; + if (!enc.isDigit(c)) err = INVALID_CHAR_IN_GROUP_NAME; + } + + if (err == null && c != endCode) { + err = INVALID_GROUP_NAME; + nameEnd = stop; + } + + if (err == null) { + mark(); + p = pnumHead; + int backNum = scanUnsignedNumber(); + restore(); + if (backNum < 0) { + newValueException(TOO_BIG_NUMBER); + } else if (backNum == 0){ + newValueException(INVALID_GROUP_NAME, src, nameEnd); + } + backNum *= sign; + + value = nameEnd; + return backNum; + } else { + newValueException(err, src, nameEnd); + return 0; // not reached + } + } + + protected final int fetchName(int startCode, boolean ref) { + if (Config.USE_NAMED_GROUP) { + return fetchNameForNamedGroup(startCode, ref); + } else { + return fetchNameForNoNamedGroup(startCode, ref); + } + } + + private boolean strExistCheckWithEsc(int[]s, int n, int bad) { + int p = this.p; + int to = this.stop; + + boolean inEsc = false; + int i; + while(p < to) { + if (inEsc) { + inEsc = false; + p += enc.length(bytes, p, to); + } else { + int x = enc.mbcToCode(bytes, p, to); + int q = p + enc.length(bytes, p, to); + if (x == s[0]) { + for (i=1; i= n) return true; + p += enc.length(bytes, p, to); + } else { + x = enc.mbcToCode(bytes, p, to); + if (x == bad) return false; + else if (x == syntax.metaCharTable.esc) inEsc = true; + p = q; + } + } + } + return false; + } + + private static final int[] send = new int[]{':', ']'}; + + private void fetchTokenInCCFor_charType(boolean flag, int type) { + token.type = TokenType.CHAR_TYPE; + token.setPropCType(type); + token.setPropNot(flag); + } + + private void fetchTokenInCCFor_p() { + int c2 = peek(); // !!! migrate to peekIs + if (c2 == '{' && syntax.op2EscPBraceCharProperty()) { + inc(); + token.type = TokenType.CHAR_PROPERTY; + token.setPropNot(c == 'P'); + + if (left() && syntax.op2EscPBraceCircumflexNot()) { + c2 = fetchTo(); + if (c2 == '^') { + token.setPropNot(!token.getPropNot()); + } else { + unfetch(); + } + } + } else { + syntaxWarn("invalid Unicode Property \\<%n>", (char)c); + } + } + + private void fetchTokenInCCFor_x() { + if (!left()) return; + int last = p; + + if (peekIs('{') && syntax.opEscXBraceHex8()) { + inc(); + int num = scanUnsignedHexadecimalNumber(0, 8); + if (num < 0) newValueException(ERR_TOO_BIG_WIDE_CHAR_VALUE); + if (left()) { + int c2 = peek(); + if (enc.isXDigit(c2)) newValueException(ERR_TOO_LONG_WIDE_CHAR_VALUE); + } + + if (p > last + enc.length(bytes, last, stop) && left() && peekIs('}')) { + inc(); + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } else { + /* can't read nothing or invalid format */ + p = last; + } + } else if (syntax.opEscXHex2()) { + int num = scanUnsignedHexadecimalNumber(0, 2); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 16; + token.setC(num); + } + } + + private void fetchTokenInCCFor_u() { + if (!left()) return; + int last = p; + + if (syntax.op2EscUHex4()) { + int num = scanUnsignedHexadecimalNumber(4, 4); + if (num < -1) newValueException(TOO_SHORT_DIGITS); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } + } + + private void fetchTokenInCCFor_digit() { + if (syntax.opEscOctal3()) { + unfetch(); + int last = p; + int num = scanUnsignedOctalNumber(3); + if (num < 0 || num > 0xff) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 8; + token.setC(num); + } + } + + private void fetchTokenInCCFor_posixBracket() { + if (syntax.opPosixBracket() && peekIs(':')) { + token.backP = p; /* point at '[' is readed */ + inc(); + if (strExistCheckWithEsc(send, send.length, ']')) { + token.type = TokenType.POSIX_BRACKET_OPEN; + } else { + unfetch(); + // remove duplication, goto cc_in_cc; + if (syntax.op2CClassSetOp()) { + token.type = TokenType.CC_CC_OPEN; + } else { + env.ccEscWarn("["); + } + } + } else { // cc_in_cc: + if (syntax.op2CClassSetOp()) { + token.type = TokenType.CC_CC_OPEN; + } else { + env.ccEscWarn("["); + } + } + } + + private void fetchTokenInCCFor_and() { + if (syntax.op2CClassSetOp() && left() && peekIs('&')) { + inc(); + token.type = TokenType.CC_AND; + } + } + + protected final TokenType fetchTokenInCC() { + if (!left()) { + token.type = TokenType.EOT; + return token.type; + } + + fetch(); + token.type = TokenType.CHAR; + token.base = 0; + token.setC(c); + token.escaped = false; + + if (c == ']') { + token.type = TokenType.CC_CLOSE; + } else if (c == '-') { + token.type = TokenType.CC_RANGE; + } else if (c == syntax.metaCharTable.esc) { + if (!syntax.backSlashEscapeInCC()) return token.type; + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + fetch(); + token.escaped = true; + token.setC(c); + + switch (c) { + case 'w': + fetchTokenInCCFor_charType(false, CharacterType.WORD); + break; + case 'W': + fetchTokenInCCFor_charType(true, CharacterType.WORD); + break; + case 'd': + fetchTokenInCCFor_charType(false, CharacterType.DIGIT); + break; + case 'D': + fetchTokenInCCFor_charType(true, CharacterType.DIGIT); + break; + case 's': + fetchTokenInCCFor_charType(false, CharacterType.SPACE); + break; + case 'S': + fetchTokenInCCFor_charType(true, CharacterType.SPACE); + break; + case 'h': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + break; + case 'H': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + break; + case 'p': + case 'P': + if (!left()) break; + fetchTokenInCCFor_p(); + break; + case 'x': + fetchTokenInCCFor_x(); + break; + case 'u': + fetchTokenInCCFor_u(); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + fetchTokenInCCFor_digit(); + break; + + default: + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.setCode(c); + token.type = TokenType.CODE_POINT; + } + break; + } // switch + + } else if (c == '[') { + fetchTokenInCCFor_posixBracket(); + } else if (c == '&') { + fetchTokenInCCFor_and(); + } + return token.type; + } + + protected final int backrefRelToAbs(int relNo) { + return env.numMem + 1 + relNo; + } + + private void fetchTokenFor_repeat(int lower, int upper) { + token.type = TokenType.OP_REPEAT; + token.setRepeatLower(lower); + token.setRepeatUpper(upper); + greedyCheck(); + } + + private void fetchTokenFor_openBrace() { + switch (fetchRangeQuantifier()) { + case 0: + greedyCheck(); + break; + case 2: + if (syntax.fixedIntervalIsGreedyOnly()) { + possessiveCheck(); + } else { + greedyCheck(); + } + break; + default: /* 1 : normal char */ + } // inner switch + } + + private void fetchTokenFor_anchor(int subType) { + token.type = TokenType.ANCHOR; + token.setAnchorSubtype(subType); + } + + private void fetchTokenFor_xBrace() { + if (!left()) return; + + int last = p; + if (peekIs('{') && syntax.opEscXBraceHex8()) { + inc(); + int num = scanUnsignedHexadecimalNumber(0, 8); + if (num < 0) newValueException(ERR_TOO_BIG_WIDE_CHAR_VALUE); + if (left()) { + if (enc.isXDigit(peek())) newValueException(ERR_TOO_LONG_WIDE_CHAR_VALUE); + } + + if (p > last + enc.length(bytes, last, stop) && left() && peekIs('}')) { + inc(); + token.type = TokenType.CODE_POINT; + token.setCode(num); + } else { + /* can't read nothing or invalid format */ + p = last; + } + } else if (syntax.opEscXHex2()) { + int num = scanUnsignedHexadecimalNumber(0, 2); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 16; + token.setC(num); + } + } + + private void fetchTokenFor_uHex() { + if (!left()) return; + int last = p; + + if (syntax.op2EscUHex4()) { + int num = scanUnsignedHexadecimalNumber(4, 4); + if (num < -1) newValueException(TOO_SHORT_DIGITS); + if (num < 0) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.CODE_POINT; + token.base = 16; + token.setCode(num); + } + } + + private void fetchTokenFor_digit() { + unfetch(); + int last = p; + int num = scanUnsignedNumber(); + if (num < 0 || num > Config.MAX_BACKREF_NUM) { // goto skip_backref + } else if (syntax.opDecimalBackref() && (num <= env.numMem || num <= 9)) { /* This spec. from GNU regex */ + if (syntax.strictCheckBackref()) { + if (num > env.numMem || env.memNodes == null || env.memNodes[num] == null) newValueException(INVALID_BACKREF); + } + token.type = TokenType.BACKREF; + token.setBackrefNum(1); + token.setBackrefRef1(num); + token.setBackrefByName(false); + if (Config.USE_BACKREF_WITH_LEVEL) token.setBackrefExistLevel(false); + return; + } + + if (c == '8' || c == '9') { /* normal char */ // skip_backref: + p = last; + inc(); + return; + } + p = last; + + fetchTokenFor_zero(); /* fall through */ + } + + private void fetchTokenFor_zero() { + if (syntax.opEscOctal3()) { + int last = p; + int num = scanUnsignedOctalNumber(c == '0' ? 2 : 3); + if (num < 0 || num > 0xff) newValueException(TOO_BIG_NUMBER); + if (p == last) { /* can't read nothing. */ + num = 0; /* but, it's not error */ + } + token.type = TokenType.RAW_BYTE; + token.base = 8; + token.setC(num); + } else if (c != '0') { + inc(); + } + } + + private void fetchTokenFor_NamedBackref() { + if (Config.USE_NAMED_GROUP) { + if (syntax.op2EscKNamedBackref() && left()) { + fetch(); + if (c =='<' || c == '\'') { + fetchNamedBackrefToken(); + } else { + unfetch(); + syntaxWarn("invalid back reference"); + } + } + } + } + + private void fetchTokenFor_subexpCall() { + if (Config.USE_NAMED_GROUP) { + if (syntax.op2EscGBraceBackref() && left()) { + fetch(); + if (c == '{') { + fetchNamedBackrefToken(); + } else { + unfetch(); + } + } + } + if (Config.USE_SUBEXP_CALL) { + if (syntax.op2EscGSubexpCall() && left()) { + fetch(); + if (c == '<' || c == '\'') { + int gNum = -1; + boolean rel = false; + int cnext = peek(); + int nameEnd = 0; + if (cnext == '0') { + inc(); + if (peekIs(nameEndCodePoint(c))) { /* \g<0>, \g'0' */ + inc(); + nameEnd = p; + gNum = 0; + } + } else if (cnext == '+') { + inc(); + rel = true; + } + int prev = p; + if (gNum < 0) { + gNum = fetchName(c, true); + nameEnd = value; + } + token.type = TokenType.CALL; + token.setCallNameP(prev); + token.setCallNameEnd(nameEnd); + token.setCallGNum(gNum); + token.setCallRel(rel); + } else { + syntaxWarn("invalid subexp call"); + unfetch(); + } + } + } + } + + protected void fetchNamedBackrefToken() { + int last = p; + int backNum; + if (Config.USE_BACKREF_WITH_LEVEL) { + Ptr rbackNum = new Ptr(); + Ptr rlevel = new Ptr(); + token.setBackrefExistLevel(fetchNameWithLevel(c, rbackNum, rlevel)); + token.setBackrefLevel(rlevel.p); + backNum = rbackNum.p; + } else { + backNum = fetchName(c, true); + } // USE_BACKREF_AT_LEVEL + int nameEnd = value; // set by fetchNameWithLevel/fetchName + + if (backNum != 0) { + if (backNum < 0) { + backNum = backrefRelToAbs(backNum); + if (backNum <= 0) newValueException(INVALID_BACKREF); + } + + if (syntax.strictCheckBackref() && (backNum > env.numMem || env.memNodes == null)) { + newValueException(INVALID_BACKREF); + } + token.type = TokenType.BACKREF; + token.setBackrefByName(false); + token.setBackrefNum(1); + token.setBackrefRef1(backNum); + } else { + NameEntry e = regex.nameToGroupNumbers(bytes, last, nameEnd); + if (e == null) newValueException(UNDEFINED_NAME_REFERENCE, last, nameEnd); + + if (syntax.strictCheckBackref()) { + if (e.backNum == 1) { + if (e.backRef1 > env.numMem || + env.memNodes == null || + env.memNodes[e.backRef1] == null) newValueException(INVALID_BACKREF); + } else { + for (int i=0; i env.numMem || + env.memNodes == null || + env.memNodes[e.backRefs[i]] == null) newValueException(INVALID_BACKREF); + } + } + } + + token.type = TokenType.BACKREF; + token.setBackrefByName(true); + + if (e.backNum == 1) { + token.setBackrefNum(1); + token.setBackrefRef1(e.backRef1); + } else { + token.setBackrefNum(e.backNum); + token.setBackrefRefs(e.backRefs); + } + } + } + + private void fetchTokenFor_charProperty() { + if (peekIs('{') && syntax.op2EscPBraceCharProperty()) { + inc(); + token.type = TokenType.CHAR_PROPERTY; + token.setPropNot(c == 'P'); + + if (left() && syntax.op2EscPBraceCircumflexNot()) { + fetch(); + if (c == '^') { + token.setPropNot(!token.getPropNot()); + } else { + unfetch(); + } + } + } else { + syntaxWarn("invalid Unicode Property \\<%n>", (char)c); + } + } + + private void fetchTokenFor_metaChars() { + if (c == syntax.metaCharTable.anyChar) { + token.type = TokenType.ANYCHAR; + } else if (c == syntax.metaCharTable.anyTime) { + fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + } else if (c == syntax.metaCharTable.zeroOrOneTime) { + fetchTokenFor_repeat(0, 1); + } else if (c == syntax.metaCharTable.oneOrMoreTime) { + fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + } else if (c == syntax.metaCharTable.anyCharAnyTime) { + token.type = TokenType.ANYCHAR_ANYTIME; + // goto out + } + } + + protected final void fetchToken() { + int src = p; + // mark(); // out + start: + while(true) { + if (!left()) { + token.type = TokenType.EOT; + return; + } + + token.type = TokenType.STRING; + token.base = 0; + token.backP = p; + + fetch(); + + if (c == syntax.metaCharTable.esc && !syntax.op2IneffectiveEscape()) { // IS_MC_ESC_CODE(code, syn) + if (!left()) newSyntaxException(END_PATTERN_AT_ESCAPE); + + token.backP = p; + fetch(); + + token.setC(c); + token.escaped = true; + switch(c) { + + case '*': + if (syntax.opEscAsteriskZeroInf()) fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + break; + case '+': + if (syntax.opEscPlusOneInf()) fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + break; + case '?': + if (syntax.opEscQMarkZeroOne()) fetchTokenFor_repeat(0, 1); + break; + case '{': + if (syntax.opEscBraceInterval()) fetchTokenFor_openBrace(); + break; + case '|': + if (syntax.opEscVBarAlt()) token.type = TokenType.ALT; + break; + case '(': + if (syntax.opEscLParenSubexp()) token.type = TokenType.SUBEXP_OPEN; + break; + case ')': + if (syntax.opEscLParenSubexp()) token.type = TokenType.SUBEXP_CLOSE; + break; + case 'w': + if (syntax.opEscWWord()) fetchTokenInCCFor_charType(false, CharacterType.WORD); + break; + case 'W': + if (syntax.opEscWWord()) fetchTokenInCCFor_charType(true, CharacterType.WORD); + break; + case 'b': + if (syntax.opEscBWordBound()) { + fetchTokenFor_anchor(AnchorType.WORD_BOUND); + token.setAnchorASCIIRange(isAsciiRange(env.option) && !isWordBoundAllRange(env.option)); + } + break; + case 'B': + if (syntax.opEscBWordBound()) { + fetchTokenFor_anchor(AnchorType.NOT_WORD_BOUND); + token.setAnchorASCIIRange(isAsciiRange(env.option) && !isWordBoundAllRange(env.option)); + } + break; + case '<': + if (Config.USE_WORD_BEGIN_END && syntax.opEscLtGtWordBeginEnd()) { + fetchTokenFor_anchor(AnchorType.WORD_BEGIN); + token.setAnchorASCIIRange(isAsciiRange(env.option)); + } + break; + case '>': + if (Config.USE_WORD_BEGIN_END && syntax.opEscLtGtWordBeginEnd()) { + fetchTokenFor_anchor(AnchorType.WORD_END); + token.setAnchorASCIIRange(isAsciiRange(env.option)); + } + break; + case 's': + if (syntax.opEscSWhiteSpace()) fetchTokenInCCFor_charType(false, CharacterType.SPACE); + break; + case 'S': + if (syntax.opEscSWhiteSpace()) fetchTokenInCCFor_charType(true, CharacterType.SPACE); + break; + case 'd': + if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(false, CharacterType.DIGIT); + break; + case 'D': + if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(true, CharacterType.DIGIT); + break; + case 'h': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + break; + case 'H': + if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + break; + case 'A': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); + break; + case 'Z': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.SEMI_END_BUF); + break; + case 'z': + if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.END_BUF); + break; + case 'G': + if (syntax.opEscCapitalGBeginAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_POSITION); + break; + case '`': + if (syntax.op2EscGnuBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); + break; + case '\'': + if (syntax.op2EscGnuBufAnchor()) fetchTokenFor_anchor(AnchorType.END_BUF); + break; + case 'x': + fetchTokenFor_xBrace(); + break; + case 'u': + fetchTokenFor_uHex(); + break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + fetchTokenFor_digit(); + break; + case '0': + fetchTokenFor_zero(); + break; + case 'k': + fetchTokenFor_NamedBackref(); + break; + case 'g': + fetchTokenFor_subexpCall(); + break; + case 'Q': + if (syntax.op2EscCapitalQQuote()) token.type = TokenType.QUOTE_OPEN; + break; + case 'p': + case 'P': + fetchTokenFor_charProperty(); + break; + case 'R': + if (syntax.op2EscCapitalRLinebreak()) token.type = TokenType.LINEBREAK; + break; + case 'X': + if (syntax.op2EscCapitalXExtendedGraphemeCluster()) token.type = TokenType.EXTENDED_GRAPHEME_CLUSTER; + break; + case 'K': + if (syntax.op2EscCapitalKKeep()) token.type = TokenType.KEEP; + break; + default: + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { /* set_raw: */ + token.type = TokenType.CODE_POINT; + token.setCode(c); + } else { /* string */ + int encLength = enc.length(bytes, token.backP, stop); + /* Check and ensure the encoded character is valid */ + if (encLength == Encoding.CHAR_INVALID) { + throw new IllegalArgumentException("Invalid character found."); + } + p = token.backP + encLength; + } + break; + } // switch (c) + } else { + token.setC(c); + token.escaped = false; + + if (Config.USE_VARIABLE_META_CHARS && (c != MetaChar.INEFFECTIVE_META_CHAR && syntax.opVariableMetaCharacters())) { + fetchTokenFor_metaChars(); + break; + } + + { + switch(c) { + case '.': + if (syntax.opDotAnyChar()) token.type = TokenType.ANYCHAR; + break; + case '*': + if (syntax.opAsteriskZeroInf()) fetchTokenFor_repeat(0, QuantifierNode.REPEAT_INFINITE); + break; + case '+': + if (syntax.opPlusOneInf()) fetchTokenFor_repeat(1, QuantifierNode.REPEAT_INFINITE); + break; + case '?': + if (syntax.opQMarkZeroOne()) fetchTokenFor_repeat(0, 1); + break; + case '{': + if (syntax.opBraceInterval()) fetchTokenFor_openBrace(); + break; + case '|': + if (syntax.opVBarAlt()) token.type = TokenType.ALT; + break; + + case '(': + if (peekIs('?') && syntax.op2QMarkGroupEffect()) { + inc(); + if (peekIs('#')) { + fetch(); + while (true) { + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c == syntax.metaCharTable.esc) { + if (left()) fetch(); + } else { + if (c == ')') break; + } + } + continue start; // goto start + } + unfetch(); + } + + if (syntax.opLParenSubexp()) token.type = TokenType.SUBEXP_OPEN; + break; + case ')': + if (syntax.opLParenSubexp()) token.type = TokenType.SUBEXP_CLOSE; + break; + case '^': + if (syntax.opLineAnchor()) fetchTokenFor_anchor(isSingleline(env.option) ? AnchorType.BEGIN_BUF : AnchorType.BEGIN_LINE); + break; + case '$': + if (syntax.opLineAnchor()) fetchTokenFor_anchor(isSingleline(env.option) ? AnchorType.SEMI_END_BUF : AnchorType.END_LINE); + break; + case '[': + if (syntax.opBracketCC()) token.type = TokenType.CC_OPEN; + break; + case ']': + if (src > getBegin()) { /* /].../ is allowed. */ + env.closeBracketWithoutEscapeWarn("]"); + } + break; + case '#': + if (Option.isExtend(env.option)) { + while (left()) { + fetch(); + if (enc.isNewLine(c)) break; + } + continue start; // goto start + } + break; + + case ' ': + case '\t': + case '\n': + case '\r': + case '\f': + if (Option.isExtend(env.option)) continue start; // goto start + break; + + default: // string + break; + + } // switch + } + } + + break; + } // while + } + + private void greedyCheck() { + if (left() && peekIs('?') && syntax.opQMarkNonGreedy()) { + + fetch(); + + token.setRepeatGreedy(false); + token.setRepeatPossessive(false); + } else { + possessiveCheck(); + } + } + + private void possessiveCheck() { + if (left() && peekIs('+') && + (syntax.op2PlusPossessiveRepeat() && token.type != TokenType.INTERVAL || + syntax.op2PlusPossessiveInterval() && token.type == TokenType.INTERVAL)) { + + fetch(); + + token.setRepeatGreedy(true); + token.setRepeatPossessive(true); + } else { + token.setRepeatGreedy(true); + token.setRepeatPossessive(false); + } + } + + protected final int fetchCharPropertyToCType() { + mark(); + + while (left()) { + int last = p; + fetch(); + if (c == '}') { + return enc.propertyNameToCType(bytes, _p, last); + } else if (c == '(' || c == ')' || c == '{' || c == '|') { + throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); + } + } + newValueException(PROPERTY_NAME_NEVER_TERMINATED, _p, stop); + return 0; // not reached + } + + protected final void syntaxWarn(String message, char c) { + syntaxWarn(message.replace("<%n>", Character.toString(c))); + } + + protected final void syntaxWarn(String message) { + if (env.warnings != WarnCallback.NONE) { + env.warnings.warn(message + ": /" + new String(bytes, getBegin(), getEnd() - getBegin()) + "/"); + } + } +} diff --git a/third_party/joni/src/org/joni/MatchView.java b/third_party/joni/src/org/joni/MatchView.java new file mode 100644 index 000000000..46227575a --- /dev/null +++ b/third_party/joni/src/org/joni/MatchView.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** Read-only provisional matcher state, valid only during a callout. */ +public interface MatchView { + int currentBytePosition(); + + int captureCount(); + + int captureBegin(int capture); + + int captureEnd(int capture); +} diff --git a/third_party/joni/src/org/joni/Matcher.java b/third_party/joni/src/org/joni/Matcher.java new file mode 100644 index 000000000..79abcc291 --- /dev/null +++ b/third_party/joni/src/org/joni/Matcher.java @@ -0,0 +1,672 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.joni; + +import static org.joni.Option.isFindLongest; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; +import org.jcodings.constants.CharacterType; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.constants.internal.AnchorType; +import org.joni.exception.TimeoutException; + +public abstract class Matcher extends IntHolder { + static final InterruptedException INTERRUPTED_EXCEPTION = new InterruptedException(); + static final InterruptedException TIMEOUT_EXCEPTION = new TimeoutException(); + public static final int FAILED = -1; + public static final int INTERRUPTED = -2; + + protected final Regex regex; + protected final Encoding enc; + + protected final byte[]bytes; + protected final int str; + protected final int end; + + protected int msaStart; + protected int msaOptions; + protected final Region msaRegion; + protected int msaBestLen; + protected int msaBestS; + protected int msaGpos; + + protected int msaBegin; + protected int msaEnd; + + protected long timeout; // nanoseconds + private CalloutHandler calloutHandler; + + // nanoseconds since entering searchCommon (underlying machines will check during interrupt checks + // which will cheapen how often we look but also it should be granular enough to not matter). + protected long startTime; + + Matcher(Regex regex, Region region, byte[]bytes, int p, int end) { + this(regex, region, bytes, p, end, -1); + } + + // FIXME: For next major version this should be the main constructor and MatcherFactory should + // have the abstract method be for this. + Matcher(Regex regex, Region region, byte[]bytes, int p, int end, long timeout) { + this.regex = regex; + this.enc = regex.enc; + this.bytes = bytes; + this.str = p; + this.end = end; + this.msaRegion = region; + this.timeout = timeout; + } + + // main matching method + protected abstract int matchAt(int range, int sstart, int sprev, boolean interrupt) throws InterruptedException; + + protected abstract void stateCheckBuffInit(int strLength, int offset, int stateNum); + protected abstract void stateCheckBuffClear(); + + public abstract void interrupt(); + + public final Region getRegion() { + return msaRegion; + } + + public final Region getEagerRegion() { + return msaRegion != null ? msaRegion : Region.newRegion(msaBegin, msaEnd); + } + + public final int getBegin() { + return msaBegin; + } + + public final int getEnd() { + return msaEnd; + } + + protected final void msaInit(int option, int start, int gpos) { + msaOptions = option; + msaStart = start; + msaGpos = gpos; + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) msaBestLen = -1; + } + + public final int match(int at, int range, int option) { + try { + return matchCommon(at, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int matchInterruptible(int at, int range, int option) throws InterruptedException { + return matchCommon(at, range, option, true); + } + + private final int matchCommon(int at, int range, int option, boolean interrupt) throws InterruptedException { + msaInit(option, at, at); + + if (Config.USE_CEC) { + int offset = at = str; + stateCheckBuffInit(end - str, offset, regex.numCombExpCheck); // move it to construction? + } // USE_COMBINATION_EXPLOSION_CHECK + + int prev = enc.prevCharHead(bytes, str, at, end); + + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + return matchAt(end /*range*/, at, prev, interrupt); + } else { + return matchAt(range /*range*/, at, prev, interrupt); + } + } + + int low, high; // these are the return values + private final boolean forwardSearchRange(byte[]bytes, int str, int end, int s, int range, IntHolder lowPrev) { + int pprev = -1; + int p = s; + + if (Config.DEBUG_SEARCH) debugForwardSearchRange(str, end, s, range); + + if (regex.dMin > 0) { + if (enc.isSingleByte()) { + p += regex.dMin; + } else { + int q = p + regex.dMin; + while (p < q && p < end) p += enc.length(bytes, p, end); + } + } + + retry:while (true) { + if (Config.DEBUG_SEARCH) debugSearch(regex.forward.getName(), p, end, range); + p = regex.forward.search(this, bytes, p, end, range); + + if (p != -1 && p < range) { + if (p - regex.dMin < s) { + // retry_gate: + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + + if (regex.subAnchor != 0) { + switch (regex.subAnchor) { + case AnchorType.BEGIN_LINE: + if (p != str) { + int prev = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + if (!enc.isNewLine(bytes, prev, end)) { + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + } + break; + + case AnchorType.END_LINE: + if (p == end) { + if (!Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + int prev = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + if (prev != -1 && enc.isNewLine(bytes, prev, end)) { + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + } + } else if (!enc.isNewLine(bytes, p, end) && (!Config.USE_CRNL_AS_LINE_TERMINATOR || !enc.isMbcCrnl(bytes, p, end))) { + //if () break; + // goto retry_gate; + pprev = p; + p += enc.length(bytes, p, end); + continue retry; + } + break; + } // switch + } + + if (regex.dMax == 0) { + low = p; + if (lowPrev != null) { // ??? // remove null checks + if (low > s) { + lowPrev.value = enc.prevCharHead(bytes, s, p, end); + } else { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, p, end); + } + } + } else { + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + low = p - regex.dMax; + + if (low > s) { + low = enc.rightAdjustCharHeadWithPrev(bytes, s, low, end, lowPrev); + if (lowPrev != null && lowPrev.value == -1) { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : s, low, end); + } + } else { + if (lowPrev != null) { + lowPrev.value = enc.prevCharHead(bytes, (pprev != -1) ? pprev : str, low, end); + } + } + } + } + /* no needs to adjust *high, *high is used as range check only */ + high = p - regex.dMin; + if (Config.DEBUG_SEARCH) debugForwardSearchRangeSuccess(str, low, high); + return true; /* success */ + } + return false; /* fail */ + } //while + } + + // low, high + private final boolean backwardSearchRange(byte[]bytes, int str, int end, int s, int range, int adjrange) { + range += regex.dMin; + int p = s; + + retry:while (true) { + p = regex.backward.search(this, bytes, range, adjrange, end, p, s, range); + + if (p != -1) { + if (regex.subAnchor != 0) { + switch (regex.subAnchor) { + case AnchorType.BEGIN_LINE: + if (p != str) { + int prev = enc.prevCharHead(bytes, str, p, end); + if (!enc.isNewLine(bytes, prev, end)) { + p = prev; + continue retry; + } + } + break; + + case AnchorType.END_LINE: + if (p == end) { + if (!Config.USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE) { + int prev = enc.prevCharHead(bytes, adjrange, p, end); + if (prev == -1) return false; + if (enc.isNewLine(bytes, prev, end)) { + p = prev; + continue retry; + } + } + } else if (!enc.isNewLine(bytes, p, end) && (!Config.USE_CRNL_AS_LINE_TERMINATOR || !enc.isMbcCrnl(bytes, p, end))) { + p = enc.prevCharHead(bytes, adjrange, p, end); + if (p == -1) return false; + continue retry; + } + break; + } // switch + } + + /* no needs to adjust *high, *high is used as range check only */ + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + low = p - regex.dMax; + high = p - regex.dMin; + high = enc.rightAdjustCharHead(bytes, adjrange, high, end); + } + + if (Config.DEBUG_SEARCH) debugBackwardSearchRange(str, low, high); + return true; + } + + if (Config.DEBUG_SEARCH) Config.log.println("backward_search_range: fail."); + return false; + } // while + } + + // MATCH_AND_RETURN_CHECK + private boolean matchCheck(int upperRange, int s, int prev, boolean interrupt) throws InterruptedException { + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + //range = upperRange; + if (matchAt(upperRange, s, prev, interrupt) != -1) { + if (!isFindLongest(regex.options)) return true; + } + } else { + //range = upperRange; + if (matchAt(upperRange, s, prev, interrupt) != -1) return true; + } + } else { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (matchAt(end, s, prev, interrupt) != -1) { + //range = upperRange; + if (!isFindLongest(regex.options)) return true; + } + } else { + //range = upperRange; + if (matchAt(end, s, prev, interrupt) != -1) return true; + } + } + return false; + } + + public final int search(int start, int range, int option) { + try { + return searchCommon(start, start, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int search(int gpos, int start, int range, int option) { + try { + return searchCommon(gpos, start, range, option, false); + } catch (InterruptedException ex) { + return INTERRUPTED; + } + } + + public final int searchInterruptible(int start, int range, int option) throws InterruptedException { + return searchCommon(start, start, range, option, true); + } + + public final int searchInterruptible(int gpos, int start, int range, int option) throws InterruptedException { + return searchCommon(gpos, start, range, option, true); + } + + private final int searchCommon(int gpos, int start, int range, int option, boolean interrupt) throws InterruptedException { + if (timeout != -1) startTime = System.nanoTime(); + int s, prev; + int origStart = start; + int origRange = range; + + if (Config.DEBUG_SEARCH) debugSearch(str, end, start, range); + + if (start > end || start < str) return FAILED; + + /* anchor optimize: resume search range */ + if (regex.anchor != 0 && str < end) { + int minSemiEnd, maxSemiEnd; + + if ((regex.anchor & AnchorType.BEGIN_POSITION) != 0) { + /* search start-position only */ + // !begin_position:! + if (range > start) { + if (gpos > start) { + if (gpos < range) range = gpos + 1; + } else { + range = start + 1; + } + } else { + range = start; + } + } else if ((regex.anchor & AnchorType.BEGIN_BUF) != 0) { + /* search str-position only */ + if (range > start) { + if (start != str) return FAILED; // mismatch_no_msa; + range = str + 1; + } else { + if (range <= str) { + start = str; + range = str; + } else { + return FAILED; // mismatch_no_msa; + } + } + } else if ((regex.anchor & AnchorType.END_BUF) != 0) { + minSemiEnd = maxSemiEnd = end; + // !end_buf:! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } else if ((regex.anchor & AnchorType.SEMI_END_BUF) != 0) { + int preEnd = enc.stepBack(bytes, str, end, end, 1); + maxSemiEnd = end; + if (enc.isNewLine(bytes, preEnd, end)) { + minSemiEnd = preEnd; + if (Config.USE_CRNL_AS_LINE_TERMINATOR) { + preEnd = enc.stepBack(bytes, str, preEnd, end, 1); + if (preEnd != -1 && enc.isMbcCrnl(bytes, preEnd, end)) { + minSemiEnd = preEnd; + } + } + if (minSemiEnd > str && start <= minSemiEnd) { + // !goto end_buf;! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } + } else { + minSemiEnd = end; + // !goto end_buf;! + if (endBuf(start, range, minSemiEnd, maxSemiEnd)) return FAILED; // mismatch_no_msa; + } + } else if ((regex.anchor & AnchorType.ANYCHAR_STAR_ML) != 0) { + // goto !begin_position;! + if (range > start) { + if (gpos > start) { + if (gpos < range) range = gpos + 1; + } else { + range = start + 1; + } + } else { + range = start; + } + } + + } else if (str == end) { /* empty string */ + // empty address ? + if (Config.DEBUG_SEARCH) Config.log.println("onig_search: empty string."); + + if (regex.thresholdLength == 0) { + s = start = str; + prev = -1; + msaInit(option, start, start); + + if (Config.USE_CEC) stateCheckBuffClear(); + + if (matchCheck(end, s, prev, interrupt)) return match(s); + return mismatch(); + } + return FAILED; // goto mismatch_no_msa; + } + + if (Config.DEBUG_SEARCH) debugSearch(str, end, start, range); + + msaInit(option, origStart, gpos); + if (Config.USE_CEC) { + int offset = Math.min(start, range) - str; + stateCheckBuffInit(end - str, offset, regex.numCombExpCheck); + } + + s = start; + if (range > start) { /* forward search */ + if (s > str) { + prev = enc.prevCharHead(bytes, str, s, end); + } else { + prev = 0; // -1 + } + + if (regex.forward != null) { + int schRange = range; + if (regex.dMax != 0) { + if (regex.dMax == MinMaxLen.INFINITE_DISTANCE) { + schRange = end; + } else { + schRange += regex.dMax; + if (schRange > end) schRange = end; + } + } + if ((end - start) < regex.thresholdLength) return mismatch(); + + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE) { + do { + if (!forwardSearchRange(bytes, str, end, s, schRange, this)) return mismatch(); // low, high, lowPrev + if (s < low) { + s = low; + prev = value; + } + while (s <= high) { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); // ??? + prev = s; + s += enc.length(bytes, s, end); + } + } while (s < range); + return mismatch(); + + } else { /* check only. */ + if (!forwardSearchRange(bytes, str, end, s, schRange, null)) return mismatch(); + + if ((regex.anchor & AnchorType.ANYCHAR_STAR) != 0) { + do { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + prev = s; + s += enc.length(bytes, s, end); + + if ((regex.anchor & (AnchorType.LOOK_BEHIND | AnchorType.PREC_READ_NOT)) == 0) { + while (!enc.isNewLine(bytes, prev, end) && s < range) { + prev = s; + s += enc.length(bytes, s, end); + } + } + } while (s < range); + return mismatch(); + } + } + } + + do { + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + prev = s; + s += enc.length(bytes, s, end); + } while (s < range); + + if (s == range) { /* because empty match with /$/. */ + if (matchCheck(origRange, s, prev, interrupt)) return match(s); + } + } else { /* backward search */ + if (Config.USE_MATCH_RANGE_MUST_BE_INSIDE_OF_SPECIFIED_RANGE) { + if (origStart < end) { + origStart += enc.length(bytes, origStart, end); // /* is upper range */ + } + } + + if (regex.backward != null) { + int adjrange; + if (range < end) { + adjrange = enc.leftAdjustCharHead(bytes, str, range, end); + } else { + adjrange = end; + } + if (regex.dMax != MinMaxLen.INFINITE_DISTANCE && (end - range) >= regex.thresholdLength) { + do { + int schStart = s + regex.dMax; + if (schStart > end) schStart = end; + if (!backwardSearchRange(bytes, str, end, schStart, range, adjrange)) return mismatch(); // low, high + if (s > high) s = high; + while (s != -1 && s >= low) { + prev = enc.prevCharHead(bytes, str, s, end); + if (matchCheck(origStart, s, prev, interrupt)) return match(s); + s = prev; + } + } while (s >= range); + return mismatch(); + } else { /* check only. */ + if ((end - range) < regex.thresholdLength) return mismatch(); + + int schStart = s; + if (regex.dMax != 0) { + if (regex.dMax == MinMaxLen.INFINITE_DISTANCE) { + schStart = end; + } else { + schStart += regex.dMax; + if (schStart > end) { + schStart = end; + } else { + schStart = enc.leftAdjustCharHead(bytes, start, schStart, end); + } + } + } + if (!backwardSearchRange(bytes, str, end, schStart, range, adjrange)) return mismatch(); + } + } + + do { + prev = enc.prevCharHead(bytes, str, s, end); + if (matchCheck(origStart, s, prev, interrupt)) return match(s); + s = prev; + } while (s >= range); + + } + return mismatch(); + } + + private final boolean endBuf(int start, int range, int minSemiEnd, int maxSemiEnd) { + if ((maxSemiEnd - str) < regex.anchorDmin) return true; // mismatch_no_msa; + + if (range > start) { + if ((minSemiEnd - start) > regex.anchorDmax) { + start = minSemiEnd - regex.anchorDmax; + if (start < end) { + start = enc.rightAdjustCharHead(bytes, str, start, end); + } else { /* match with empty at end */ + start = enc.prevCharHead(bytes, str, end, end); + } + } + if ((maxSemiEnd - (range - 1)) < regex.anchorDmin) { + range = maxSemiEnd - regex.anchorDmin + 1; + } + if (start >= range) return true; // mismatch_no_msa; + } else { + if ((minSemiEnd - range) > regex.anchorDmax) { + range = minSemiEnd - regex.anchorDmax; + } + if ((maxSemiEnd - start) < regex.anchorDmin) { + start = maxSemiEnd - regex.anchorDmin; + start = enc.leftAdjustCharHead(bytes, str, start, end); + } + if (range > start) return true; // mismatch_no_msa; + } + return false; + } + + private final int match(int s) { + return s - str; // sstart ??? + } + + private final int mismatch() { + if (Config.USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE) { + if (msaBestLen >= 0) { + int s = msaBestS; + return match(s); + } + } + // falls through finish: + return FAILED; + } + + private byte[]icbuf; + + protected final byte[]icbuf() { + return icbuf == null ? icbuf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN] : icbuf; + } + + static boolean isMbcAsciiWord(Encoding enc, byte[]bytes, int p, int end) { // ONIGENC_IS_MBC_ASCII_WORD + return ASCIIEncoding.INSTANCE.isCodeCType(enc.mbcToCode(bytes, p, end), CharacterType.WORD); + } + + private final void debugForwardSearchRange(int str, int end, int s, int range) { + if (Config.DEBUG_SEARCH) { + Config.log.println("forward_search_range: " + + "str: " + str + + ", end: " + end + + ", s: " + s + + ", range: " + range); + } + } + + private final void debugForwardSearchRangeSuccess(int str, int low, int high) { + if (Config.DEBUG_SEARCH) { + Config.log.println("forward_search_range success: " + + "low: " + (low - str) + + ", high: " + (high - str) + + ", dmin: " + regex.dMin + + ", dmax: " + regex.dMax); + } + } + + private final void debugSearch(int str, int end, int start, int range) { + if (Config.DEBUG_SEARCH) { + Config.log.println("onig_search (entry point): " + + "str: " + str + + ", end: " + (end - str) + + ", start: " + (start - str) + + ", range " + (range - str)); + } + } + + private final void debugBackwardSearchRange(int str, int low, int high) { + if (Config.DEBUG_SEARCH) { + Config.log.println("backward_search_range: "+ + "low: " + (low - str) + + ", high: " + (high - str)); + } + } + + static void debugSearch(String name, int textP, int textEnd, int textRange) { + Config.log.println(name + ": text: " + textP + ", text_end: " + textEnd + ", text_range: " + textRange); + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public final void setCalloutHandler(CalloutHandler handler) { + calloutHandler = handler; + } + + final CalloutHandler getCalloutHandler() { + return calloutHandler; + } +} diff --git a/third_party/joni/src/org/joni/MatcherFactory.java b/third_party/joni/src/org/joni/MatcherFactory.java new file mode 100644 index 000000000..48f3ae445 --- /dev/null +++ b/third_party/joni/src/org/joni/MatcherFactory.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +abstract class MatcherFactory { + abstract Matcher create(Regex regex, Region region, byte[]bytes, int p, int end); + + public Matcher create(Regex regex, Region region, byte[]bytes, int p, int end, long timeout) { + Matcher matcher = create(regex, region, bytes, p, end); + matcher.setTimeout(timeout); + return matcher; + } + + static final MatcherFactory DEFAULT = new MatcherFactory() { + @Override + Matcher create(Regex regex, Region region, byte[]bytes, int p, int end) { + return new ByteCodeMachine(regex, region, bytes, p, end); + } + }; +} diff --git a/third_party/joni/src/org/joni/MinMaxLen.java b/third_party/joni/src/org/joni/MinMaxLen.java new file mode 100644 index 000000000..0c6d18f46 --- /dev/null +++ b/third_party/joni/src/org/joni/MinMaxLen.java @@ -0,0 +1,129 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +final class MinMaxLen { + int min; /* min byte length */ + int max; /* max byte length */ + + /* 1000 / (min-max-dist + 1) */ + private static final short[] distValues = { + 1000, 500, 333, 250, 200, 167, 143, 125, 111, 100, + 91, 83, 77, 71, 67, 63, 59, 56, 53, 50, + 48, 45, 43, 42, 40, 38, 37, 36, 34, 33, + 32, 31, 30, 29, 29, 28, 27, 26, 26, 25, + 24, 24, 23, 23, 22, 22, 21, 21, 20, 20, + 20, 19, 19, 19, 18, 18, 18, 17, 17, 17, + 16, 16, 16, 16, 15, 15, 15, 15, 14, 14, + 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, + 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 10, 10, 10, 10, 10 + }; + + int distanceValue() { + if (max == INFINITE_DISTANCE) return 0; + int d = max - min; + /* return dist_vals[d] * 16 / (mm->min + 12); */ + return d < distValues.length ? distValues[d] : 1; + } + + int compareDistanceValue(MinMaxLen other, int v1, int v2) { + if (v2 <= 0) return -1; + if (v1 <= 0) return 1; + + v1 *= distanceValue(); + v2 *= other.distanceValue(); + + if (v2 > v1) return 1; + if (v2 < v1) return -1; + + return Integer.compare(min, other.min); + } + + boolean equal(MinMaxLen other) { + return min == other.min && max == other.max; + } + + void set(int min, int max) { + this.min = min; + this.max = max; + } + + void clear() { + min = max = 0; + } + + void copy(MinMaxLen other) { + min = other.min; + max = other.max; + } + + void add(MinMaxLen other) { + min = distanceAdd(min, other.min); + max = distanceAdd(max, other.max); + } + + void addLength(int len) { + min = distanceAdd(min, len); + max = distanceAdd(max, len); + } + + void altMerge(MinMaxLen other) { + if (min > other.min) min = other.min; + if (max < other.max) max = other.max; + } + + static final int INFINITE_DISTANCE = 0x7FFFFFFF; + static int distanceAdd(int d1, int d2) { + if (d1 == INFINITE_DISTANCE || d2 == INFINITE_DISTANCE) { + return INFINITE_DISTANCE; + } else { + if (d1 <= INFINITE_DISTANCE - d2) return d1 + d2; + else return INFINITE_DISTANCE; + } + } + + static int distanceMultiply(int d, int m) { + if (m == 0) return 0; + if (d < INFINITE_DISTANCE / m) { + return d * m; + } else { + return INFINITE_DISTANCE; + } + } + + static String distanceRangeToString(int a, int b) { + String s = ""; + if (a == INFINITE_DISTANCE) { + s += "inf"; + } else { + s += "(" + a + ")"; + } + + s += "-"; + + if (b == INFINITE_DISTANCE) { + s += "inf"; + } else { + s += "(" + b + ")"; + } + return s; + } +} diff --git a/third_party/joni/src/org/joni/MultiRegion.java b/third_party/joni/src/org/joni/MultiRegion.java new file mode 100644 index 000000000..baf0eb407 --- /dev/null +++ b/third_party/joni/src/org/joni/MultiRegion.java @@ -0,0 +1,72 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import java.util.Arrays; + +public final class MultiRegion extends Region { + private final int[] begEnd; + + public MultiRegion(int num) { + this.begEnd = new int[num * 2]; + } + + public MultiRegion(int begin, int end) { + this.begEnd = new int[]{begin, end}; + } + + @Override + public final int getNumRegs() { + return begEnd.length / 2; + } + + @Override + public MultiRegion clone() { + MultiRegion region = new MultiRegion(getNumRegs()); + System.arraycopy(begEnd, 0, region.begEnd, 0, begEnd.length); + if (getCaptureTree() != null) region.setCaptureTree(getCaptureTree().cloneTree()); + return region; + } + + @Override + public int getBeg(int index) { + return begEnd[index * 2]; + } + + @Override + public int setBeg(int index, int value) { + return begEnd[index * 2] = value; + } + + @Override + public int getEnd(int index) { + return begEnd[index * 2 + 1]; + } + + @Override + public int setEnd(int index, int value) { + return begEnd[index * 2 + 1] = value; + } + + @Override + void clear() { + Arrays.fill(begEnd, REGION_NOTPOS); + } +} diff --git a/third_party/joni/src/org/joni/NameEntry.java b/third_party/joni/src/org/joni/NameEntry.java new file mode 100644 index 000000000..2b6251305 --- /dev/null +++ b/third_party/joni/src/org/joni/NameEntry.java @@ -0,0 +1,98 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class NameEntry { + static final int INIT_NAME_BACKREFS_ALLOC_NUM = 8; + + public final byte[]name; + public final int nameP; + public final int nameEnd; + + int backNum; + int backRef1; + int[] backRefs; + + public NameEntry(byte[]bytes, int p, int end) { + name = bytes; + nameP = p; + nameEnd = end; + } + + public int[] getBackRefs() { + switch (backNum) { + case 0: + return new int[]{}; + case 1: + return new int[]{backRef1}; + default: + int[]result = new int[backNum]; + System.arraycopy(backRefs, 0, result, 0, backNum); + return result; + } + } + + private void alloc() { + backRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; + } + + private void ensureSize() { + if (backNum > backRefs.length) { + int[]tmp = new int[backRefs.length << 1]; + System.arraycopy(backRefs, 0, tmp, 0, backRefs.length); + backRefs = tmp; + } + } + + public void addBackref(int backRef) { + backNum++; + + switch (backNum) { + case 1: + backRef1 = backRef; + break; + case 2: + alloc(); + backRefs[0] = backRef1; + backRefs[1] = backRef; + break; + default: + ensureSize(); + backRefs[backNum - 1] = backRef; + } + } + + @Override + public String toString() { + StringBuilder buff = new StringBuilder(new String(name, nameP, nameEnd - nameP) + " "); + if (backNum == 0) { + buff.append("-"); + } else if (backNum == 1){ + buff.append(backRef1); + } else { + for (int i=0; i 0) buff.append(", "); + buff.append(backRefs[i]); + } + } + return buff.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/NativeMachine.java b/third_party/joni/src/org/joni/NativeMachine.java new file mode 100644 index 000000000..d086123a5 --- /dev/null +++ b/third_party/joni/src/org/joni/NativeMachine.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +abstract class NativeMachine extends Matcher { + + protected NativeMachine(Regex regex, Region region, byte[]bytes, int p, int end) { + super(regex, region, bytes, p, end); + } +} diff --git a/third_party/joni/src/org/joni/NodeOptInfo.java b/third_party/joni/src/org/joni/NodeOptInfo.java new file mode 100644 index 000000000..9eb737577 --- /dev/null +++ b/third_party/joni/src/org/joni/NodeOptInfo.java @@ -0,0 +1,127 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +final class NodeOptInfo { + final MinMaxLen length = new MinMaxLen(); + final OptAnchorInfo anchor = new OptAnchorInfo(); + final OptExactInfo exb = new OptExactInfo(); /* boundary */ + final OptExactInfo exm = new OptExactInfo(); /* middle */ + final OptExactInfo expr = new OptExactInfo(); /* prec read (?=...) */ + final OptMapInfo map = new OptMapInfo(); /* boundary */ + + public void setBoundNode(MinMaxLen mmd) { + exb.mmd.copy(mmd); + expr.mmd.copy(mmd); + map.mmd.copy(mmd); + } + + public void clear() { + length.clear(); + anchor.clear(); + exb.clear(); + exm.clear(); + expr.clear(); + map.clear(); + } + + public void copy(NodeOptInfo other) { + length.copy(other.length); + anchor.copy(other.anchor); + exb.copy(other.exb); + exm.copy(other.exm); + expr.copy(other.expr); + map.copy(other.map); + } + + public void concatLeftNode(NodeOptInfo other, Encoding enc) { + OptAnchorInfo tanchor = new OptAnchorInfo(); // remove it somehow ? + tanchor.concat(anchor, other.anchor, length.max, other.length.max); + anchor.copy(tanchor); + + if (other.exb.length > 0 && length.max == 0) { + tanchor.concat(anchor, other.exb.anchor, length.max, other.length.max); + other.exb.anchor.copy(tanchor); + } + + if (other.map.value > 0 && length.max == 0) { + if (other.map.mmd.max == 0) { + other.map.anchor.leftAnchor |= anchor.leftAnchor; + } + } + + boolean exbReach = exb.reachEnd; + boolean exmReach = exm.reachEnd; + + if (other.length.max != 0) { + exb.reachEnd = exm.reachEnd = false; + } + + if (other.exb.length > 0) { + if (exbReach) { + exb.concat(other.exb, enc); + other.exb.clear(); + } else if (exmReach) { + exm.concat(other.exb, enc); + other.exb.clear(); + } + } + + exm.select(other.exb, enc); + exm.select(other.exm, enc); + + if (expr.length > 0) { + if (other.length.max > 0) { + // TODO: make sure it is not an Oniguruma bug (casting unsigned int to int for arithmetic comparison) + int otherLengthMax = other.length.max; + if (otherLengthMax == MinMaxLen.INFINITE_DISTANCE) otherLengthMax = -1; + if (expr.length > otherLengthMax) expr.length = otherLengthMax; + if (expr.mmd.max == 0) { + exb.select(expr, enc); + } else { + exm.select(expr, enc); + } + } + } else if (other.expr.length > 0) { + expr.copy(other.expr); + } + + map.select(other.map); + length.add(other.length); + } + + public void altMerge(NodeOptInfo other, OptEnvironment env) { + anchor.altMerge(other.anchor); + exb.altMerge(other.exb, env); + exm.altMerge(other.exm, env); + expr.altMerge(other.expr, env); + map.altMerge(other.map, env.enc); + length.altMerge(other.length); + } + + public void setBound(MinMaxLen mmd) { + exb.mmd.copy(mmd); + expr.mmd.copy(mmd); + map.mmd.copy(mmd); + } + +} diff --git a/third_party/joni/src/org/joni/OptAnchorInfo.java b/third_party/joni/src/org/joni/OptAnchorInfo.java new file mode 100644 index 000000000..6774e20af --- /dev/null +++ b/third_party/joni/src/org/joni/OptAnchorInfo.java @@ -0,0 +1,96 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.constants.internal.AnchorType; + +final class OptAnchorInfo implements AnchorType { + int leftAnchor; + int rightAnchor; + + void clear() { + leftAnchor = rightAnchor = 0; + } + + void copy(OptAnchorInfo other) { + leftAnchor = other.leftAnchor; + rightAnchor = other.rightAnchor; + } + + void concat(OptAnchorInfo left, OptAnchorInfo right, int leftLength, int rightLength) { + leftAnchor = left.leftAnchor; + if (leftLength == 0) leftAnchor |= right.leftAnchor; + + rightAnchor = right.rightAnchor; + if (rightLength == 0) { + rightAnchor |= left.rightAnchor; + } else { + rightAnchor |= left.rightAnchor & AnchorType.PREC_READ_NOT; + } + } + + boolean isSet(int anchor) { + if ((leftAnchor & anchor) != 0) return true; + return (rightAnchor & anchor) != 0; + } + + void add(int anchor) { + if (isLeftAnchor(anchor)) { + leftAnchor |= anchor; + } else { + rightAnchor |= anchor; + } + } + + void remove(int anchor) { + if (isLeftAnchor(anchor)) { + leftAnchor &= ~anchor; + } else { + rightAnchor &= ~anchor; + } + } + + void altMerge(OptAnchorInfo other) { + leftAnchor &= other.leftAnchor; + rightAnchor &= other.rightAnchor; + } + + static boolean isLeftAnchor(int anchor) { // make a mask for it ? + return !(anchor == END_BUF || anchor == SEMI_END_BUF || + anchor == END_LINE || anchor == PREC_READ || + anchor == PREC_READ_NOT); + } + + static String anchorToString(int anchor) { + StringBuilder s = new StringBuilder("["); + + if ((anchor & AnchorType.BEGIN_BUF) !=0 ) s.append("begin-buf "); + if ((anchor & AnchorType.BEGIN_LINE) !=0 ) s.append("begin-line "); + if ((anchor & AnchorType.BEGIN_POSITION) !=0 ) s.append("begin-pos "); + if ((anchor & AnchorType.END_BUF) !=0 ) s.append("end-buf "); + if ((anchor & AnchorType.SEMI_END_BUF) !=0 ) s.append("semi-end-buf "); + if ((anchor & AnchorType.END_LINE) !=0 ) s.append("end-line "); + if ((anchor & AnchorType.ANYCHAR_STAR) !=0 ) s.append("anychar-star "); + if ((anchor & AnchorType.ANYCHAR_STAR_ML) !=0 ) s.append("anychar-star-pl "); + s.append("]"); + + return s.toString(); + } +} diff --git a/third_party/joni/src/org/joni/OptEnvironment.java b/third_party/joni/src/org/joni/OptEnvironment.java new file mode 100644 index 000000000..bc627508d --- /dev/null +++ b/third_party/joni/src/org/joni/OptEnvironment.java @@ -0,0 +1,39 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +// remove this one in future and pass mmd directly +final class OptEnvironment { + final MinMaxLen mmd = new MinMaxLen(); + Encoding enc; + int options; + int caseFoldFlag; + ScanEnvironment scanEnv; + + void copy(OptEnvironment other) { + mmd.copy(other.mmd); + enc = other.enc; + options = other.options; + caseFoldFlag = other.caseFoldFlag; + scanEnv = other.scanEnv; + } +} diff --git a/third_party/joni/src/org/joni/OptExactInfo.java b/third_party/joni/src/org/joni/OptExactInfo.java new file mode 100644 index 000000000..ab975bbbb --- /dev/null +++ b/third_party/joni/src/org/joni/OptExactInfo.java @@ -0,0 +1,170 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +final class OptExactInfo { + static final int OPT_EXACT_MAXLEN = 24; + + final MinMaxLen mmd = new MinMaxLen(); + final OptAnchorInfo anchor = new OptAnchorInfo(); + boolean reachEnd; + int ignoreCase; /* -1: unset, 0: case sensitive, 1: ignore case */ + final byte[] bytes = new byte[OPT_EXACT_MAXLEN]; + int length; + + boolean isFull() { + return length >= OPT_EXACT_MAXLEN; + } + + void clear() { + mmd.clear(); + anchor.clear(); + reachEnd = false; + ignoreCase = -1; + length = 0; + } + + void copy(OptExactInfo other) { + mmd.copy(other.mmd); + anchor.copy(other.anchor); + reachEnd = other.reachEnd; + ignoreCase = other.ignoreCase; + length = other.length; + + System.arraycopy(other.bytes, 0, bytes, 0, OPT_EXACT_MAXLEN); + } + + void concat(OptExactInfo other, Encoding enc) { + if (ignoreCase < 0) { + ignoreCase = other.ignoreCase; + } else if (ignoreCase != other.ignoreCase) { + return; + } + + int p = 0; // add->s; + int end = p + other.length; + + int i; + for (i = length; p < end;) { + int len = enc.length(other.bytes, p, end); + if (i + len > OPT_EXACT_MAXLEN) break; + for (int j = 0; j < len && p < end; j++) { + bytes[i++] = other.bytes[p++]; // arraycopy or even don't copy anything ?? + } + } + + length = i; + reachEnd = (p == end && other.reachEnd); + + OptAnchorInfo tmp = new OptAnchorInfo(); + tmp.concat(anchor, other.anchor, 1, 1); + if (!reachEnd) tmp.rightAnchor = 0; + anchor.copy(tmp); + } + + void concatStr(byte[]lbytes, int p, int end, boolean raw, Encoding enc) { + int i; + for (i = length; p < end && i < OPT_EXACT_MAXLEN;) { + int len = enc.length(lbytes, p, end); + if (i + len > OPT_EXACT_MAXLEN) break; + for (int j = 0; j < len && p < end; j++) { + bytes[i++] = lbytes[p++]; + } + } + length = i; + } + + void altMerge(OptExactInfo other, OptEnvironment env) { + if (other.length == 0 || length == 0) { + clear(); + return; + } + + if (!mmd.equal(other.mmd)) { + clear(); + return; + } + + int i; + for (i=0; i= 0) { + ignoreCase |= other.ignoreCase; + } + + anchor.altMerge(other.anchor); + + if (!reachEnd) anchor.rightAnchor = 0; + } + + + void select(OptExactInfo alt, Encoding enc) { + int v1 = length; + int v2 = alt.length; + + if (v2 == 0) { + return; + } else if (v1 == 0) { + copy(alt); + return; + } else if (v1 <= 2 && v2 <= 2) { + /* ByteValTable[x] is big value --> low price */ + v2 = OptMapInfo.positionValue(enc, bytes[0] & 0xff); + v1 = OptMapInfo.positionValue(enc, alt.bytes[0] & 0xff); + + if (length > 1) v1 += 5; + if (alt.length > 1) v2 += 5; + } + + if (ignoreCase <= 0) v1 *= 2; + if (alt.ignoreCase <= 0) v2 *= 2; + + if (mmd.compareDistanceValue(alt.mmd, v1, v2) > 0) copy(alt); + } + + // comp_opt_exact_or_map_info + private static final int COMP_EM_BASE = 20; + int compare(OptMapInfo m) { + if (m.value <= 0) return -1; + + int ve = COMP_EM_BASE * length * (ignoreCase > 0 ? 1 : 2); + int vm = COMP_EM_BASE * 5 * 2 / m.value; + + return mmd.compareDistanceValue(m.mmd, ve, vm); + } +} diff --git a/third_party/joni/src/org/joni/OptMapInfo.java b/third_party/joni/src/org/joni/OptMapInfo.java new file mode 100644 index 000000000..0aeaec9e1 --- /dev/null +++ b/third_party/joni/src/org/joni/OptMapInfo.java @@ -0,0 +1,126 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.Encoding; + +final class OptMapInfo { + final MinMaxLen mmd = new MinMaxLen(); /* info position */ + final OptAnchorInfo anchor = new OptAnchorInfo(); + int value; /* weighted value */ + final byte[] map = new byte[Config.CHAR_TABLE_SIZE]; + + void clear() { + mmd.clear(); + anchor.clear(); + value = 0; + for (int i=0; i 0) copy(alt); + } + + // alt_merge_opt_map_info + void altMerge(OptMapInfo other, Encoding enc) { + /* if (! is_equal_mml(&to->mmd, &add->mmd)) return ; */ + if (value == 0) return; + if (other.value == 0 || mmd.max < other.mmd.max) { + clear(); + return; + } + + mmd.altMerge(other.mmd); + + int val = 0; + for (int i=0; i 1) { + return 20; + } else { + return ByteValTable[i]; + } + } else { + return 4; /* Take it easy. */ + } + } + +} diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java new file mode 100644 index 000000000..1747738a7 --- /dev/null +++ b/third_party/joni/src/org/joni/Option.java @@ -0,0 +1,149 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public final class Option { + /* options */ + public static final int NONE = 0; + public static final int IGNORECASE = (1 << 0); + public static final int EXTEND = (1 << 1); + public static final int MULTILINE = (1 << 2); + public static final int SINGLELINE = (1 << 3); + public static final int FIND_LONGEST = (1 << 4); + public static final int FIND_NOT_EMPTY = (1 << 5); + public static final int NEGATE_SINGLELINE = (1 << 6); + public static final int DONT_CAPTURE_GROUP = (1 << 7); + public static final int CAPTURE_GROUP = (1 << 8); + + /* options (search time) */ + public static final int NOTBOL = (1 << 9); + public static final int NOTEOL = (1 << 10); + public static final int POSIX_REGION = (1 << 11); + + /* options (ctype range) */ + public static final int ASCII_RANGE = (1 << 12); + public static final int POSIX_BRACKET_ALL_RANGE = (1 << 13); + public static final int WORD_BOUND_ALL_RANGE = (1 << 14); + /* options (newline) */ + public static final int NEWLINE_CRLF = (1 << 15); + public static final int NOTBOS = (1 << 16); + public static final int NOTEOS = (1 << 17); + public static final int CR_7_BIT = (1 << 18); + + public static final int MAXBIT = (1 << 19); /* limit */ + + public static final int DEFAULT = NONE; + + public static String toString(int option) { + String options = ""; + if (isIgnoreCase(option)) options += "IGNORECASE"; + if (isExtend(option)) options += "EXTEND"; + if (isMultiline(option)) options += "MULTILINE"; + if (isSingleline(option)) options += "SINGLELINE"; + if (isFindLongest(option)) options += "FIND_LONGEST"; + if (isFindNotEmpty(option)) options += "FIND_NOT_EMPTY"; + if (isNegateSingleline(option)) options += "NEGATE_SINGLELINE"; + if (isDontCaptureGroup(option)) options += "DONT_CAPTURE_GROUP"; + if (isCaptureGroup(option)) options += "CAPTURE_GROUP"; + if (isNotBol(option)) options += "NOTBOL"; + if (isNotEol(option)) options += "NOTEOL"; + if (isPosixRegion(option)) options += "POSIX_REGION"; + if (isCR7Bit(option)) options += "CR_7_BIT"; + return options; + } + + public static boolean isIgnoreCase(int option) { + return (option & IGNORECASE) != 0; + } + + public static boolean isExtend(int option) { + return (option & EXTEND) != 0; + } + + public static boolean isSingleline(int option) { + return (option & SINGLELINE) != 0; + } + + public static boolean isMultiline(int option) { + return (option & MULTILINE) != 0; + } + + public static boolean isFindLongest(int option) { + return (option & FIND_LONGEST) != 0; + } + + public static boolean isFindNotEmpty(int option) { + return (option & FIND_NOT_EMPTY) != 0; + } + + public static boolean isFindCondition(int option) { + return (option & (FIND_LONGEST | FIND_NOT_EMPTY)) != 0; + } + + public static boolean isNegateSingleline(int option) { + return (option & NEGATE_SINGLELINE) != 0; + } + + public static boolean isDontCaptureGroup(int option) { + return (option & DONT_CAPTURE_GROUP) != 0; + } + + public static boolean isCaptureGroup(int option) { + return (option & CAPTURE_GROUP) != 0; + } + + public static boolean isNotBol(int option) { + return (option & NOTBOL) != 0; + } + + public static boolean isNotEol(int option) { + return (option & NOTEOL) != 0; + } + + public static boolean isPosixRegion(int option) { + return (option & POSIX_REGION) != 0; + } + + public static boolean isAsciiRange(int option) { + return (option & ASCII_RANGE) != 0; + } + + public static boolean isPosixBracketAllRange(int option) { + return (option & POSIX_BRACKET_ALL_RANGE) != 0; + } + + public static boolean isWordBoundAllRange(int option) { + return (option & WORD_BOUND_ALL_RANGE) != 0; + } + + public static boolean isNewlineCRLF(int option) { + return (option & NEWLINE_CRLF) != 0; + } + + public static boolean isCR7Bit(int option) { + return (option & CR_7_BIT) != 0; + } + + public static boolean isDynamic(int option) { + // ignore-case and multibyte status are included in compiled code + // return (option & (MULTILINE | IGNORECASE)) != 0; + return false; + } +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java new file mode 100644 index 000000000..cd2be3dda --- /dev/null +++ b/third_party/joni/src/org/joni/Parser.java @@ -0,0 +1,1442 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsOnAtSimple; +import static org.joni.BitStatus.bsOnOff; +import static org.joni.Option.isAsciiRange; +import static org.joni.Option.isDontCaptureGroup; +import static org.joni.Option.isIgnoreCase; +import static org.joni.Option.isPosixBracketAllRange; + +import org.jcodings.Encoding; +import org.jcodings.ObjPtr; +import org.jcodings.Ptr; +import org.jcodings.constants.CharacterType; +import org.jcodings.constants.PosixBracket; +import org.jcodings.exception.InternalException; +import org.jcodings.unicode.UnicodeCodeRange; +import org.joni.ast.AnchorNode; +import org.joni.ast.AnyCharNode; +import org.joni.ast.BackRefNode; +import org.joni.ast.CClassNode; +import org.joni.ast.CClassNode.CCSTATE; +import org.joni.ast.CClassNode.CCStateArg; +import org.joni.ast.CClassNode.CCVALTYPE; +import org.joni.ast.CTypeNode; +import org.joni.ast.CallNode; +import org.joni.ast.CalloutNode; +import org.joni.ast.EncloseNode; +import org.joni.ast.ListNode; +import org.joni.ast.Node; +import org.joni.ast.QuantifierNode; +import org.joni.ast.StringNode; +import org.joni.constants.internal.AnchorType; +import org.joni.constants.internal.EncloseType; +import org.joni.constants.internal.NodeType; +import org.joni.constants.internal.TokenType; +import org.joni.exception.ErrorMessages; + +class Parser extends Lexer { + protected int returnCode; // return code used by parser methods (they itself return parsed nodes) + // this approach will not affect recursive calls + + protected Parser(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { + super(regex, syntax, bytes, p, end, warnings); + } + + private static final int POSIX_BRACKET_NAME_MIN_LEN = 4; + private static final int POSIX_BRACKET_CHECK_LIMIT_LENGTH = 20; + private static final byte[] BRACKET_END = ":]".getBytes(); + private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc) { + mark(); + + boolean not; + if (peekIs('^')) { + inc(); + not = true; + } else { + not = false; + } + if (enc.strLength(bytes, p, stop) >= POSIX_BRACKET_NAME_MIN_LEN + 3) { // else goto not_posix_bracket + boolean asciiRange = isAsciiRange(env.option) && !isPosixBracketAllRange(env.option); + + for (int i=0; i POSIX_BRACKET_CHECK_LIMIT_LENGTH) break; + } + + if (c == ':' && left()) { + inc(); + if (left()) { + fetch(); + if (c == ']') newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + } + } + restore(); + return true; /* 1: is not POSIX bracket, but no error. */ + } + + private boolean codeExistCheck(int code, boolean ignoreEscaped) { + mark(); + + boolean inEsc = false; + while (left()) { + if (ignoreEscaped && inEsc) { + inEsc = false; + } else { + fetch(); + if (c == code) { + restore(); + return true; + } + if (c == syntax.metaCharTable.esc) inEsc = true; + } + } + + restore(); + return false; + } + + private CClassNode parseCharClass(ObjPtr ascNode) { + final boolean neg; + CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null; + CCStateArg arg = new CCStateArg(); + + fetchTokenInCC(); + if (token.type == TokenType.CHAR && token.getC() == '^' && !token.escaped) { + neg = true; + fetchTokenInCC(); + } else { + neg = false; + } + + if (token.type == TokenType.CC_CLOSE && !syntax.op3OptionECMAScript()) { + if (!codeExistCheck(']', true)) newSyntaxException(EMPTY_CHAR_CLASS); + env.ccEscWarn("]"); + token.type = TokenType.CHAR; /* allow []...] */ + } + + cc = new CClassNode(); + if (isIgnoreCase(env.option)) { + ascCc = ascNode.p = new CClassNode(); + } + + boolean andStart = false; + arg.state = CCSTATE.START; + while (token.type != TokenType.CC_CLOSE) { + boolean fetched = false; + + switch (token.type) { + case CHAR: + final int len; + if (token.getCode() >= BitSet.SINGLE_BYTE_SIZE || (len = enc.codeToMbcLength(token.getC())) > 1) { + arg.inType = CCVALTYPE.CODE_POINT; + } else { + arg.inType = CCVALTYPE.SB; // sb_char: + } + arg.to = token.getC(); + arg.toIsRaw = false; + parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + break; + + case RAW_BYTE: + if (!enc.isSingleByte() && token.base != 0) { /* tok->base != 0 : octal or hexadec. */ + byte[]buf = new byte[Config.ENC_MBC_CASE_FOLD_MAXLEN]; + int psave = p; + int base = token.base; + buf[0] = (byte)token.getC(); + int i; + for (i=1; i len) { /* fetch back */ + p = psave; + for (i=1; i ascPtr = new ObjPtr<>(); + CClassNode acc = parseCharClass(ascPtr); + cc.or(acc, env); + if (ascPtr.p != null) { + ascCc.or(ascPtr.p, env); + } + break; + + case CC_AND: /* && */ + if (arg.state == CCSTATE.VALUE) { + arg.to = 0; + arg.toIsRaw = false; + cc.nextStateValue(arg, ascCc, env); + } + /* initialize local variables */ + andStart = true; + arg.state = CCSTATE.START; + if (prevCc != null) { + prevCc.and(cc, env); + if (ascCc != null) { + ascPrevCc.and(ascCc, env); + } + } else { + prevCc = cc; + if (workCc == null) workCc = new CClassNode(); + cc = workCc; + if (ascCc != null) { + ascPrevCc = ascCc; + if (ascWorkCc == null) ascWorkCc = new CClassNode(); + ascCc = ascWorkCc; + } + } + cc.clear(); + if (ascCc != null) ascCc.clear(); + break; + + case EOT: + newSyntaxException(PREMATURE_END_OF_CHAR_CLASS); + + default: + newInternalException(PARSER_BUG); + } // switch + + if (!fetched) fetchTokenInCC(); + + } // while + + if (arg.state == CCSTATE.VALUE) { + arg.to = 0; + arg.toIsRaw = false; + cc.nextStateValue(arg, ascCc, env); + } + + if (prevCc != null) { + prevCc.and(cc, env); + cc = prevCc; + if (ascCc != null) { + ascPrevCc.and(ascCc, env); + ascCc = ascPrevCc; + } + } + + if (neg) { + cc.setNot(); + if (ascCc != null) ascCc.setNot(); + } else { + cc.clearNot(); + if (ascCc != null) ascCc.clearNot(); + } + + if (cc.isNot() && syntax.notNewlineInNegativeCC()) { + if (!cc.isEmpty()) { // ??? + final int NEW_LINE = 0x0a; + if (enc.isNewLine(NEW_LINE)) { + if (enc.codeToMbcLength(NEW_LINE) == 1) { + cc.bs.set(env, NEW_LINE); + } else { + cc.addCodeRange(env, NEW_LINE, NEW_LINE); + } + } + } + } + + return cc; + } + + private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + arg.inType = CCVALTYPE.SB; + arg.to = token.getC(); + arg.toIsRaw = false; + parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + } + + private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + arg.to = '-'; + arg.toIsRaw = false; + parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + } + + private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + int len = enc.codeToMbcLength(arg.to); + arg.inType = len == 1 ? CCVALTYPE.SB : CCVALTYPE.CODE_POINT; + parseCharClassValEntry2(cc, ascCc, arg); // val_entry2: + } + + private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + cc.nextStateValue(arg, ascCc, env); + } + + private Node parseEnclose(TokenType term) { + Node node = null; + + if (!left()) newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + + int option = env.option; + + if (peekIs('?') && syntax.op2QMarkGroupEffect()) { + inc(); + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + + boolean listCapture = false; + + fetch(); + switch(c) { + case '{': + return parseInternalCallout(); + case ':': /* (?:...) grouping only */ + fetchToken(); // group: + node = parseSubExp(term); + returnCode = 1; /* group */ + return node; + case '=': + node = new AnchorNode(AnchorType.PREC_READ); + break; + case '!': /* preceding read */ + node = new AnchorNode(AnchorType.PREC_READ_NOT); + if (syntax.op3OptionECMAScript()) { + env.pushPrecReadNotNode(node); + } + break; + case '>': /* (?>...) stop backtrack */ + node = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + break; + case '~': /* (?~...) absent operator */ + if (syntax.op2QMarkTildeAbsent()) { + node = new EncloseNode(EncloseType.ABSENT); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + case '\'': + if (Config.USE_NAMED_GROUP) { + if (syntax.op2QMarkLtNamedGroup()) { + listCapture = false; // goto named_group1 + node = parseEncloseNamedGroup2(listCapture); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + } // USE_NAMED_GROUP + break; + case '<': /* look behind (?<=...), (?...) */ + } + unfetch(); + } + } // USE_NAMED_GROUP + EncloseNode en = EncloseNode.newMemory(env.option, false); + int num = env.addMemEntry(); + if (num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); + en.regNum = num; + node = en; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case '(': /* conditional expression: (?(cond)yes), (?(cond)yes|no) */ + if (left() && syntax.op2QMarkLParenCondition()) { + int num = -1; + int name = -1; + int calloutConditionId = -1; + fetch(); + if (c == '?' && left() && peekIs('{')) { + fetch(); + calloutConditionId = parseInternalCalloutId(); + } else if (enc.isDigit(c)) { /* (n) */ + unfetch(); + num = fetchName('(', true); + if (syntax.strictCheckBackref()) { + if (num > env.numMem || env.memNodes == null || env.memNodes[num] == null) newValueException(INVALID_BACKREF); + } + } else { + if (Config.USE_NAMED_GROUP) { + if (c == '<' || c == '\'') { /* (), ('name') */ + name = p; + fetchNamedBackrefToken(); + inc(); + num = token.getBackrefNum() > 1 ? token.getBackrefRefs()[0] : token.getBackrefRef1(); + } + } else { // USE_NAMED_GROUP + newSyntaxException(INVALID_CONDITION_PATTERN); + } + } + EncloseNode en = new EncloseNode(EncloseType.CONDITION); + en.regNum = num; + en.calloutConditionId = calloutConditionId; + if (name != -1) en.setNameRef(); + node = en; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case '^': /* loads default options */ + if (left() && syntax.op2OptionPerl()) { + /* d-imsx */ + option = bsOnOff(option, Option.ASCII_RANGE, true); + option = bsOnOff(option, Option.IGNORECASE, true); + option = bsOnOff(option, Option.SINGLELINE, false); + option = bsOnOff(option, Option.MULTILINE, true); + option = bsOnOff(option, Option.EXTEND, true); + fetch(); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + // case 'p': #ifdef USE_POSIXLINE_OPTION + case '-': + case 'i': + case 'm': + case 's': + case 'x': + case 'a': + case 'd': + case 'l': + case 'u': + boolean neg = false; + while (true) { + switch(c) { + case ':': + case ')': + break; + case '-': + neg = true; + break; + case 'x': + option = bsOnOff(option, Option.EXTEND, neg); + break; + case 'i': + option = bsOnOff(option, Option.IGNORECASE, neg); + break; + case 's': + if (syntax.op2OptionPerl()) { + option = bsOnOff(option, Option.MULTILINE, neg); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + case 'm': + if (syntax.op2OptionPerl()) { + option = bsOnOff(option, Option.SINGLELINE, !neg); + } else if (syntax.op2OptionRuby()) { + option = bsOnOff(option, Option.MULTILINE, neg); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + // case 'p': #ifdef USE_POSIXLINE_OPTION // not defined + // option = bsOnOff(option, Option.MULTILINE|Option.SINGLELINE, neg); + // break; + + case 'a': /* limits \d, \s, \w and POSIX brackets to ASCII range */ + if ((syntax.op2OptionPerl() || syntax.op2OptionRuby()) && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, false); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, true); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, true); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + case 'u': + if ((syntax.op2OptionPerl() || syntax.op2OptionRuby()) && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, true); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, true); + break; + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + case 'd': + if (syntax.op2OptionPerl() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + } else if (syntax.op2OptionRuby() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, false); + option = bsOnOff(option, Option.POSIX_BRACKET_ALL_RANGE, false); + option = bsOnOff(option, Option.WORD_BOUND_ALL_RANGE, false); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + + case 'l': + if (syntax.op2OptionPerl() && !neg) { + option = bsOnOff(option, Option.ASCII_RANGE, true); + } else { + newSyntaxException(UNDEFINED_GROUP_OPTION); + } + break; + default: + newSyntaxException(UNDEFINED_GROUP_OPTION); + } // switch + + if (c == ')') { + node = EncloseNode.newOption(option); + returnCode = 2; /* option only */ + return node; + } else if (c == ':') { + int prev = env.option; + env.option = option; + fetchToken(); + Node target = parseSubExp(term); + env.option = prev; + EncloseNode en = EncloseNode.newOption(option); + en.setTarget(target); + node = en; + returnCode = 0; + return node; + } + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + } // while + + default: + newSyntaxException(UNDEFINED_GROUP_OPTION); + } // switch + + } else { + if (isDontCaptureGroup(env.option)) { + fetchToken(); // goto group + node = parseSubExp(term); + returnCode = 1; /* group */ + return node; + } + EncloseNode en = EncloseNode.newMemory(env.option, false); + en.regNum = env.addMemEntry(); + node = en; + } + + fetchToken(); + Node target = parseSubExp(term); + + if (node.getType() == NodeType.ANCHOR) { + AnchorNode an = (AnchorNode)node; + an.setTarget(target); + if (syntax.op3OptionECMAScript() && an.type == AnchorType.PREC_READ_NOT) { + env.popPrecReadNotNode(an); + } + } else { + EncloseNode en = (EncloseNode)node; + en.setTarget(target); + if (en.type == EncloseType.MEMORY) { + if (syntax.op3OptionECMAScript()) { + en.containingAnchor = env.currentPrecReadNotNode(); + } + /* Don't move this to previous of parse_subexp() */ + env.setMemNode(en.regNum, en); + } else if (en.type == EncloseType.CONDITION) { + if (target.getType() != NodeType.ALT) { /* convert (?(cond)yes) to (?(cond)yes|empty) */ + en.setTarget(ListNode.newAlt(target, ListNode.newAlt(StringNode.EMPTY, null))); + } + } + } + returnCode = 0; + return node; // ?? + } + + private Node parseInternalCallout() { + return new CalloutNode(parseInternalCalloutId()); + } + + private int parseInternalCalloutId() { + final String prefix = "=CALL:"; + for (int i = 0; i < prefix.length(); i++) { + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c != prefix.charAt(i)) newSyntaxException(UNDEFINED_GROUP_OPTION); + } + + if (!left() || !enc.isDigit(peek())) newSyntaxException(UNDEFINED_GROUP_OPTION); + fetch(); + int calloutId = Encoding.digitVal(c); + while (left() && enc.isDigit(peek())) { + fetch(); + int digit = Encoding.digitVal(c); + if (calloutId > (Integer.MAX_VALUE - digit) / 10) { + newValueException("callout id is too large"); + } + calloutId = calloutId * 10 + digit; + } + + if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + fetch(); + if (c != '}') newSyntaxException(UNDEFINED_GROUP_OPTION); + if (!left()) newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + fetch(); + if (c != ')') newSyntaxException(UNMATCHED_CLOSE_PARENTHESIS); + + returnCode = 0; + return calloutId; + } + + private Node parseEncloseNamedGroup2(boolean listCapture) { + int nm = p; + fetchName(c, false); + int nameEnd = value; + int num = env.addMemEntry(); + if (listCapture && num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); + + regex.nameAdd(bytes, nm, nameEnd, num, syntax); + EncloseNode en = EncloseNode.newMemory(env.option, true); + en.regNum = num; + + if (listCapture) env.captureHistory = bsOnAtSimple(env.captureHistory, num); + env.numNamed++; + return en; + } + + private int findStrPosition(int[]s, int n, int from, int to, Ptr nextChar) { + int x; + int q; + int p = from; + int i; + while (p < to) { + x = enc.mbcToCode(bytes, p, to); + q = p + enc.length(bytes, p, to); + if (x == s[0]) { + for (i=1; i= n) { + if (bytes[nextChar.p] != 0) nextChar.p = q; // we may need zero term semantics... + return p; + } + } + p = q; + } + return -1; + } + + private Node parseExp(TokenType term) { + if (token.type == term) return StringNode.EMPTY; // goto end_of_token + Node node = null; + boolean group = false; + + switch(token.type) { + case ALT: + case EOT: + return StringNode.EMPTY; // end_of_token:, node_new_empty + + case SUBEXP_OPEN: + node = parseEnclose(TokenType.SUBEXP_CLOSE); + if (returnCode == 1) { + group = true; + } else if (returnCode == 2) { /* option only */ + int prev = env.option; + EncloseNode en = (EncloseNode)node; + env.option = en.option; + fetchToken(); + Node target = parseSubExp(term); + env.option = prev; + en.setTarget(target); + return node; + } + break; + case SUBEXP_CLOSE: + if (!syntax.allowUnmatchedCloseSubexp()) newSyntaxException(UNMATCHED_CLOSE_PARENTHESIS); + if (token.escaped) { + return parseExpTkRawByte(group); // goto tk_raw_byte + } else { + return parseExpTkByte(group); // goto tk_byte + } + case LINEBREAK: + node = parseLineBreak(); + break; + + case EXTENDED_GRAPHEME_CLUSTER: + node = parseExtendedGraphemeCluster(); + break; + + case KEEP: + node = new AnchorNode(AnchorType.KEEP); + break; + + case STRING: + return parseExpTkByte(group); // tk_byte: + + case RAW_BYTE: + return parseExpTkRawByte(group); // tk_raw_byte: + + case CODE_POINT: + return parseStringLoop(StringNode.fromCodePoint(token.getCode(), enc), group); + + case QUOTE_OPEN: + node = parseQuoteOpen(); + break; + + case CHAR_TYPE: + node = parseCharType(node); + break; + + case CHAR_PROPERTY: + node = parseCharProperty(); + break; + + case CC_OPEN: { + ObjPtr ascPtr = new ObjPtr<>(); + CClassNode cc = parseCharClass(ascPtr); + int code = cc.isOneChar(); + if (code != -1) return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + + node = cc; + if (isIgnoreCase(env.option)) node = cClassCaseFold(node, cc, ascPtr.p); + break; + } + + case ANYCHAR: + node = new AnyCharNode(); + break; + + case ANYCHAR_ANYTIME: + node = parseAnycharAnytime(); + break; + + case BACKREF: + node = parseBackref(); + break; + + case CALL: + if (Config.USE_SUBEXP_CALL) node = parseCall(); + break; + + case ANCHOR: + node = new AnchorNode(token.getAnchorSubtype(), token.getAnchorASCIIRange()); + break; + + case OP_REPEAT: + case INTERVAL: + if (syntax.contextIndepRepeatOps()) { + if (syntax.contextInvalidRepeatOps()) { + newSyntaxException(TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + } else { + node = StringNode.EMPTY; // node_new_empty + } + } else { + return parseExpTkByte(group); // goto tk_byte + } + break; + + default: + newInternalException(PARSER_BUG); + } //switch + + //targetp = node; + + fetchToken(); // re_entry: + + return parseExpRepeat(node, group); // repeat: + } + + private Node parseLineBreak() { + byte[]buflb = new byte[Config.ENC_CODE_TO_MBC_MAXLEN * 2]; + int len1 = enc.codeToMbc(0x0D, buflb, 0); + int len2 = enc.codeToMbc(0x0A, buflb, len1); + StringNode left = new StringNode(buflb, 0, len1 + len2); + left.setRaw(); + /* [\x0A-\x0D] or [\x0A-\x0D\x{85}\x{2028}\x{2029}] */ + CClassNode right = new CClassNode(); + if (enc.minLength() > 1) { + right.addCodeRange(env, 0x0A, 0x0D); + } else { + right.bs.setRange(env, 0x0A, 0x0D); + } + + if (enc.isUnicode()) { + /* UTF-8, UTF-16BE/LE, UTF-32BE/LE */ + right.addCodeRange(env, 0x85, 0x85); + right.addCodeRange(env, 0x2028, 0x2029); + } + /* (?>...) */ + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); + en.setTarget(ListNode.newAlt(left, ListNode.newAlt(right, null))); + return en; + } + + private void addPropertyToCC(CClassNode cc, UnicodeCodeRange range, boolean not) { + cc.addCType(range.getCType(), not, false, env, this); + } + + private void createPropertyNode(Node[]nodes, int np, UnicodeCodeRange range) { + CClassNode cc = new CClassNode(); + addPropertyToCC(cc, range, false); + nodes[np] = cc; + } + + private void quantifierNode(Node[]nodes, int np, int lower, int upper) { + QuantifierNode qnf = new QuantifierNode(lower, upper, false); + qnf.setTarget(nodes[np]); + nodes[np] = qnf; + } + + private void quantifierPropertyNode(Node[]nodes, int np, UnicodeCodeRange range, char repetitions) { + int lower = 0; + int upper = QuantifierNode.REPEAT_INFINITE; + + createPropertyNode(nodes, np, range); + switch (repetitions) { + case '?': upper = 1; break; + case '+': lower = 1; break; + case '*': break; + case '2': lower = upper = 2; break; + default : throw new InternalException(ErrorMessages.PARSER_BUG); + } + + quantifierNode(nodes, np, lower, upper); + } + + private void createNodeFromArray(boolean list, Node[] nodes, int np, int nodeArray) { + int i = 0; + ListNode tmp = null; + while (nodes[nodeArray + i] != null) i++; + while (--i >= 0) { + nodes[np] = list ? ListNode.newList(nodes[nodeArray + i], tmp) : ListNode.newAlt(nodes[nodeArray + i], tmp); + nodes[nodeArray + i] = null; + tmp = (ListNode)nodes[np]; + } + } + + private ListNode createNodeFromArray(Node[]nodes, int nodeArray) { + int i = 0; + ListNode np = null, tmp = null; + while (nodes[nodeArray + i] != null) i++; + while (--i >= 0) { + np = ListNode.newAlt(nodes[nodeArray + i], tmp); + nodes[nodeArray + i] = null; + tmp = np; + } + return np; + } + + private static final int NODE_COMMON_SIZE = 16; + private Node parseExtendedGraphemeCluster() { + final Node[] nodes = new Node[NODE_COMMON_SIZE]; + final int anyTargetPosition; + int alts = 0; + + StringNode strNode = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN * 2); + strNode.setRaw(); + strNode.catCode(0x0D, enc); + strNode.catCode(0x0A, enc); + nodes[alts] = strNode; + + if (Config.USE_UNICODE_PROPERTIES && enc.isUnicode()) { + CClassNode cc; + cc = new CClassNode(); + nodes[alts + 1] = cc; + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, false); + if (enc.minLength() > 1) { + cc.addCodeRange(env, 0x000A, 0x000A); + cc.addCodeRange(env, 0x000D, 0x000D); + } else { + cc.bs.set(0x0A); + cc.bs.set(0x0D); + } + + { + int list = alts + 3; + quantifierPropertyNode(nodes, list + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_PREPEND, '*'); + { + int coreAlts = list + 2; + { + int HList = coreAlts + 1; + quantifierPropertyNode(nodes, HList + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_L, '*'); + { + int HAlt2 = HList + 2; + quantifierPropertyNode(nodes, HAlt2 + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_V, '+'); + { + int HList2 = HAlt2 + 2; + createPropertyNode(nodes, HList2 + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_LV); + quantifierPropertyNode(nodes, HList2 + 1, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_V, '*'); + createNodeFromArray(true, nodes, HAlt2 + 1, HList2); + } + createPropertyNode(nodes, HAlt2 + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_LVT); + createNodeFromArray(false, nodes, HList + 1, HAlt2); + } + quantifierPropertyNode(nodes, HList + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_T, '*'); + createNodeFromArray(true, nodes, coreAlts + 0, HList); + } + quantifierPropertyNode(nodes, coreAlts + 1, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_L, '+'); + quantifierPropertyNode(nodes, coreAlts + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_T, '+'); + quantifierPropertyNode(nodes, coreAlts + 3, UnicodeCodeRange.REGIONALINDICATOR, '2'); + { + int XPList = coreAlts + 5; + createPropertyNode(nodes, XPList + 0, UnicodeCodeRange.EXTENDEDPICTOGRAPHIC); + { + int ExList = XPList + 2; + quantifierPropertyNode(nodes, ExList + 0, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_EXTEND, '*'); + strNode = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN); + strNode.setRaw(); + strNode.catCode(0x200D, enc); + nodes[ExList + 1] = strNode; + createPropertyNode(nodes, ExList + 2, UnicodeCodeRange.EXTENDEDPICTOGRAPHIC); + createNodeFromArray(true, nodes, XPList + 1, ExList); + } + quantifierNode(nodes, XPList + 1, 0, QuantifierNode.REPEAT_INFINITE); + createNodeFromArray(true, nodes, coreAlts + 4, XPList); + } + cc = new CClassNode(); + nodes[coreAlts + 5] = cc; + if (enc.minLength() > 1) { + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, false); + cc.addCodeRange(env, 0x000A, 0x000A); + cc.addCodeRange(env, 0x000D, 0x000D); + cc.mbuf = CodeRangeBuffer.notCodeRangeBuff(env, cc.mbuf); + } else { + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_CONTROL, true); + cc.bs.clear(0x0A); + cc.bs.clear(0x0D); + } + createNodeFromArray(false, nodes, list + 1, coreAlts); + } + createPropertyNode(nodes, list + 2, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_EXTEND); + cc = (CClassNode)nodes[list + 2]; + addPropertyToCC(cc, UnicodeCodeRange.GRAPHEMECLUSTERBREAK_SPACINGMARK, false); + cc.addCodeRange(env, 0x200D, 0x200D); + quantifierNode(nodes, list + 2, 0, QuantifierNode.REPEAT_INFINITE); + createNodeFromArray(true, nodes, alts + 2, list); + + } + anyTargetPosition = 3; + } else { // enc.isUnicode() + anyTargetPosition = 1; + } + + Node any = new AnyCharNode(); + EncloseNode option = EncloseNode.newOption(bsOnOff(env.option, Option.MULTILINE, false)); + option.setTarget(any); + nodes[anyTargetPosition] = option; + + Node topAlt = createNodeFromArray(nodes, alts); + EncloseNode enclose = new EncloseNode(EncloseType.STOP_BACKTRACK); + enclose.setTarget(topAlt); + + if (Config.USE_UNICODE_PROPERTIES && enc.isUnicode()) { + option = EncloseNode.newOption(bsOnOff(env.option, Option.IGNORECASE, true)); + option.setTarget(enclose); + return option; + } else { + return enclose; + } + } + + private Node parseExpTkByte(boolean group) { + StringNode node = new StringNode(bytes, token.backP, p); // tk_byte: + return parseStringLoop(node, group); + } + + private Node parseStringLoop(StringNode node, boolean group) { + while (true) { + fetchToken(); + if (token.type == TokenType.STRING) { + if (token.backP == node.end) { + node.end = p; // non escaped character, remain shared, just increase shared range + } else { + node.catBytes(bytes, token.backP, p); // non continuous string stream, need to COW + } + } else if (token.type == TokenType.CODE_POINT) { + node.catCode(token.getCode(), enc); + } else { + break; + } + } + // targetp = node; + return parseExpRepeat(node, group); // string_end:, goto repeat + } + + private Node parseExpTkRawByte(boolean group) { + // tk_raw_byte: + StringNode node = new StringNode(); + node.setRaw(); + node.catByte((byte)token.getC()); + + int len = 1; + while (true) { + if (len >= enc.minLength()) { + if (len == enc.length(node.bytes, node.p, node.end)) { + fetchToken(); + node.clearRaw(); + // !goto string_end;! + return parseExpRepeat(node, group); + } + } + + fetchToken(); + if (token.type != TokenType.RAW_BYTE) { + /* Don't use this, it is wrong for little endian encodings. */ + // USE_PAD_TO_SHORT_BYTE_CHAR ... + newValueException(TOO_SHORT_MULTI_BYTE_STRING); + } + node.catByte((byte)token.getC()); + len++; + } // while + } + + private Node parseExpRepeat(Node target, boolean group) { + while (token.type == TokenType.OP_REPEAT || token.type == TokenType.INTERVAL) { // repeat: + if (isInvalidQuantifier(target)) newSyntaxException(TARGET_OF_REPEAT_OPERATOR_INVALID); + + if (!group && syntax.op3OptionECMAScript() && target.getType() == NodeType.QTFR) { + newSyntaxException(NESTED_REPEAT_NOT_ALLOWED); + } + QuantifierNode qtfr = new QuantifierNode(token.getRepeatLower(), + token.getRepeatUpper(), + token.type == TokenType.INTERVAL); + + qtfr.greedy = token.getRepeatGreedy(); + int ret = qtfr.setQuantifier(target, group, env, bytes, getBegin(), getEnd()); + Node qn = qtfr; + + if (token.getRepeatPossessive()) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + en.setTarget(qn); + qn = en; + } + + if (ret == 0 || (syntax.op3OptionECMAScript() && ret == 1)) { + target = qn; + } else if (ret == 2) { /* split case: /abc+/ */ + target = ListNode.newList(target, null); + ListNode tmp = ListNode.newList(qn, null); + ((ListNode)target).setTail(tmp); + + fetchToken(); + return parseExpRepeatForCar(target, tmp, group); + } + fetchToken(); // goto re_entry + } + return target; + } + + private Node parseExpRepeatForCar(Node top, ListNode target, boolean group) { + while (token.type == TokenType.OP_REPEAT || token.type == TokenType.INTERVAL) { // repeat: + if (isInvalidQuantifier(target.value)) newSyntaxException(TARGET_OF_REPEAT_OPERATOR_INVALID); + + QuantifierNode qtfr = new QuantifierNode(token.getRepeatLower(), + token.getRepeatUpper(), + token.type == TokenType.INTERVAL); + + qtfr.greedy = token.getRepeatGreedy(); + int ret = qtfr.setQuantifier(target.value, group, env, bytes, getBegin(), getEnd()); + Node qn = qtfr; + + if (token.getRepeatPossessive()) { + EncloseNode en = new EncloseNode(EncloseType.STOP_BACKTRACK); // node_new_enclose + en.setTarget(qn); + qn = en; + } + + if (ret == 0) { + target.setValue(qn); + } else if (ret == 2) { /* split case: /abc+/ */ + assert false; + } + fetchToken(); // goto re_entry + } + return top; + } + + private boolean isInvalidQuantifier(Node node) { + if (Config.USE_NO_INVALID_QUANTIFIER) return false; + + ListNode consAlt; + switch(node.getType()) { + case NodeType.ANCHOR: + return true; + + case NodeType.ENCLOSE: + /* allow enclosed elements */ + /* return is_invalid_quantifier_target(NENCLOSE(node)->target); */ + break; + + case NodeType.LIST: + consAlt = (ListNode)node; + do { + if (!isInvalidQuantifier(consAlt.value)) return false; + } while ((consAlt = consAlt.tail) != null); + return false; + + case NodeType.ALT: + consAlt = (ListNode)node; + do { + if (isInvalidQuantifier(consAlt.value)) return true; + } while ((consAlt = consAlt.tail) != null); + break; + + default: + break; + } + return false; + } + + private Node parseQuoteOpen() { + int[]endOp = new int[]{syntax.metaCharTable.esc, 'E'}; + int qstart = p; + Ptr nextChar = new Ptr(); + int qend = findStrPosition(endOp, endOp.length, qstart, stop, nextChar); + if (qend == -1) nextChar.p = qend = stop; + Node node = new StringNode(bytes, qstart, qend); + p = nextChar.p; + return node; + } + + private Node parseCharType(Node node) { + switch(token.getPropCType()) { + case CharacterType.WORD: + node = new CTypeNode(token.getPropCType(), token.getPropNot(), isAsciiRange(env.option)); + break; + + case CharacterType.SPACE: + case CharacterType.DIGIT: + case CharacterType.XDIGIT: + CClassNode ccn = new CClassNode(); + ccn.addCType(token.getPropCType(), false, isAsciiRange(env.option), env, this); + if (token.getPropNot()) ccn.setNot(); + node = ccn; + break; + + default: + newInternalException(PARSER_BUG); + } // inner switch + return node; + } + + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc); + enc.applyAllCaseFold(env.caseFoldFlag, ApplyCaseFold.INSTANCE, arg); + if (arg.altRoot != null) { + node = ListNode.newAlt(node, arg.altRoot); + } + return node; + } + + private Node parseCharProperty() { + int ctype = fetchCharPropertyToCType(); + CClassNode cc = new CClassNode(); + Node node = cc; + cc.addCType(ctype, false, false, env, this); + if (token.getPropNot()) cc.setNot(); + + if (isIgnoreCase(env.option)) { + if (ctype != CharacterType.ASCII) { + node = cClassCaseFold(node, cc, cc); + } + } + return node; + } + + private Node parseAnycharAnytime() { + Node node = new AnyCharNode(); + QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false); + qn.setTarget(node); + return qn; + } + + private Node parseBackref() { + final Node node; + if (syntax.op3OptionECMAScript() && token.getBackrefNum() == 1 && env.memNodes != null) { + EncloseNode encloseNode = env.memNodes[token.getBackrefRef1()]; + boolean shouldIgnore = false; + if (encloseNode != null && encloseNode.containingAnchor != null) { + shouldIgnore = true; + for (Node anchorNode : env.precReadNotNodes) { + if (anchorNode == encloseNode.containingAnchor) { + shouldIgnore = false; + break; + } + } + } + if (shouldIgnore) { + node = StringNode.EMPTY; + } else { + node = newBackRef(new int[]{token.getBackrefRef1()}); + } + } else { + node = newBackRef(token.getBackrefNum() > 1 ? token.getBackrefRefs() : new int[]{token.getBackrefRef1()}); + } + return node; + } + + private BackRefNode newBackRef(int[]backRefs) { + return new BackRefNode(token.getBackrefNum(), + backRefs, + token.getBackrefByName(), + token.getBackrefExistLevel(), + token.getBackrefLevel(), + env); + } + + private Node parseCall() { + int gNum = token.getCallGNum(); + if (gNum < 0 || token.getCallRel()) { + if (gNum > 0) gNum--; + gNum = backrefRelToAbs(gNum); + if (gNum <= 0) newValueException(INVALID_BACKREF); + } + Node node = new CallNode(bytes, token.getCallNameP(), token.getCallNameEnd(), gNum); + env.numCall++; + return node; + } + + private Node parseBranch(TokenType term) { + Node node = parseExp(term); + + if (token.type == TokenType.EOT || token.type == term || token.type == TokenType.ALT) { + return node; + } else { + ListNode top = ListNode.newList(node, null); + ListNode t = top; + + while (token.type != TokenType.EOT && token.type != term && token.type != TokenType.ALT) { + node = parseExp(term); + if (node.getType() == NodeType.LIST) { + t.setTail((ListNode)node); + while (((ListNode)node).tail != null ) node = ((ListNode)node).tail; + + t = ((ListNode)node); + } else { + t.setTail(ListNode.newList(node, null)); + t = t.tail; + } + } + return top; + } + } + + /* term_tok: TK_EOT or TK_SUBEXP_CLOSE */ + private Node parseSubExp(TokenType term) { + Node node = parseBranch(term); + + if (token.type == term) { + return node; + } else if (token.type == TokenType.ALT) { + ListNode top = ListNode.newAlt(node, null); + ListNode t = top; + while (token.type == TokenType.ALT) { + fetchToken(); + node = parseBranch(term); + + t.setTail(ListNode.newAlt(node, null)); + t = t.tail; + } + + if (token.type != term) parseSubExpError(term); + return top; + } else { + parseSubExpError(term); + return null; //not reached + } + } + + private void parseSubExpError(TokenType term) { + if (term == TokenType.SUBEXP_CLOSE) { + newSyntaxException(END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + } else { + newInternalException(PARSER_BUG); + } + } + + protected final Node parseRegexp() { + fetchToken(); + Node top = parseSubExp(TokenType.EOT); + if (Config.USE_SUBEXP_CALL) { + if (env.numCall > 0) { + /* Capture the pattern itself. It is used for (?R), (?0) and \g<0>. */ + EncloseNode np = EncloseNode.newMemory(env.option, false); + np.regNum = 0; + np.setTarget(top); + if (env.memNodes == null) env.memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + env.memNodes[0] = np; + top = np; + } + } + return top; + } +} diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java new file mode 100644 index 000000000..46db17e51 --- /dev/null +++ b/third_party/joni/src/org/joni/Regex.java @@ -0,0 +1,521 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAt; +import static org.joni.Config.USE_SUNDAY_QUICK_SEARCH; +import static org.joni.Option.isCaptureGroup; +import static org.joni.Option.isDontCaptureGroup; + +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.Iterator; + +import org.jcodings.CaseFoldCodeItem; +import org.jcodings.Encoding; +import org.jcodings.EncodingDB; +import org.jcodings.specific.ASCIIEncoding; +import org.jcodings.specific.UTF8Encoding; +import org.jcodings.util.BytesHash; +import org.joni.constants.internal.AnchorType; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.ValueException; + +public final class Regex { + int[] code; /* compiled pattern */ + int codeLength; + boolean requireStack; + + int numMem; /* used memory(...) num counted from 1 */ + int numRepeat; /* OP_REPEAT/OP_REPEAT_NG id-counter */ + int numNullCheck; /* OP_NULL_CHECK_START/END id counter */ + int numCombExpCheck; /* combination explosion check */ + int numCall; /* number of subexp call */ + int captureHistory; /* (?@...) flag (1-31) */ + int btMemStart; /* need backtrack flag */ + int btMemEnd; /* need backtrack flag */ + + int stackPopLevel; + + int[]repeatRangeLo; + int[]repeatRangeHi; + + MatcherFactory factory; + + final Encoding enc; + int options; + int userOptions; + Object userObject; + final int caseFoldFlag; + + private BytesHash nameTable; // named entries + + /* optimization info (string search, char-map and anchors) */ + Search.Forward forward; /* optimize flag */ + Search.Backward backward; + int thresholdLength; /* search str-length for apply optimize */ + int anchor; /* BEGIN_BUF, BEGIN_POS, (SEMI_)END_BUF */ + int anchorDmin; /* (SEMI_)END_BUF anchor distance */ + int anchorDmax; /* (SEMI_)END_BUF anchor distance */ + int subAnchor; /* start-anchor for exact or map */ + + byte[]exact; + int exactP; + int exactEnd; + + byte[]map; /* used as BM skip or char-map */ + int[]intMap; /* BM skip for exact_len > 255 */ + int[]intMapBackward; /* BM skip for backward search */ + int dMin; /* min-distance of exact or map */ + int dMax; /* max-distance of exact or map */ + + byte[][]templates; /* fixed pattern strings not embedded in bytecode */ + int templateNum; + + private static final Encoding DEFAULT_ENCODING; + static { + Encoding defaultEncoding; + EncodingDB.Entry entry = EncodingDB.getEncodings().get(Charset.defaultCharset().name().getBytes()); + if (entry == null) { + defaultEncoding = UTF8Encoding.INSTANCE; + } else { + defaultEncoding = entry.getEncoding(); + } + DEFAULT_ENCODING = defaultEncoding; + } + + public Regex(CharSequence cs) { + this(cs.toString()); + } + + public Regex(CharSequence cs, Encoding enc) { + this(cs.toString(), enc); + } + + public Regex(String str) { + this(str.getBytes(), 0, str.length(), 0, UTF8Encoding.INSTANCE); + } + + public Regex(String str, Encoding enc) { + this(str.getBytes(), 0, str.length(), 0, enc); + } + + public Regex(byte[] bytes) { + this(bytes, 0, bytes.length, 0, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[] bytes, int p, int end) { + this(bytes, p, end, 0, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[] bytes, int p, int end, int option) { + this(bytes, p, end, option, ASCIIEncoding.INSTANCE); + } + + public Regex(byte[]bytes, int p, int end, int option, Encoding enc) { + this(bytes, p, end, option, enc, Syntax.RUBY, WarnCallback.DEFAULT); + } + + // onig_new + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, Syntax syntax) { + this(bytes, p, end, option, Config.ENC_CASE_FOLD_DEFAULT, enc, syntax, WarnCallback.DEFAULT); + } + + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, WarnCallback warnings) { + this(bytes, p, end, option, enc, Syntax.RUBY, warnings); + } + + // onig_new + public Regex(byte[]bytes, int p, int end, int option, Encoding enc, Syntax syntax, WarnCallback warnings) { + this(bytes, p, end, option, Config.ENC_CASE_FOLD_DEFAULT, enc, syntax, warnings); + } + + // onig_alloc_init + public Regex(byte[]bytes, int p, int end, int option, int caseFoldFlag, Encoding enc, Syntax syntax, WarnCallback warnings) { + if (Config.REGEX_MAX_LENGTH > 0 && (end - p) > Config.REGEX_MAX_LENGTH) { + throw new ValueException(ErrorMessages.REGEX_TOO_LONG); + } + + if ((option & (Option.DONT_CAPTURE_GROUP | Option.CAPTURE_GROUP)) == + (Option.DONT_CAPTURE_GROUP | Option.CAPTURE_GROUP)) { + throw new ValueException(ErrorMessages.INVALID_COMBINATION_OF_OPTIONS); + } + + if ((option & Option.NEGATE_SINGLELINE) != 0) { + option |= syntax.options; + option &= ~Option.SINGLELINE; + } else { + option |= syntax.options; + } + + this.enc = enc; + this.options = option; + this.caseFoldFlag = caseFoldFlag; + new Analyser(this, syntax, bytes, p, end, warnings).compile(); + } + + public Matcher matcher(byte[]bytes) { + return matcher(bytes, 0, bytes.length); + } + + public Matcher matcherNoRegion(byte[]bytes) { + return matcherNoRegion(bytes, 0, bytes.length); + } + + public Matcher matcher(byte[]bytes, int p, int end) { + return factory.create(this, numMem == 0 ? null : Region.newRegion(numMem + 1), bytes, p, end); + } + + public Matcher matcher(byte[]bytes, int p, int end, long timeout) { + return factory.create(this, numMem == 0 ? null : Region.newRegion(numMem + 1), bytes, p, end, timeout); + } + + public Matcher matcherNoRegion(byte[]bytes, int p, int end) { + return factory.create(this, null, bytes, p, end); + } + + public Matcher matcherNoRegion(byte[]bytes, int p, int end, long timeout) { + return factory.create(this, null, bytes, p, end, timeout); + } + + public int numberOfCaptures() { + return numMem; + } + + public int numberOfCaptureHistories() { + if (Config.USE_CAPTURE_HISTORY) { + int n = 0; + for (int i=0; i<=Config.MAX_CAPTURE_HISTORY_GROUP; i++) { + if (bsAt(captureHistory, i)) n++; + } + return n; + } else { + return 0; + } + } + + private NameEntry nameFind(byte[]name, int nameP, int nameEnd) { + if (nameTable != null) return nameTable.get(name, nameP, nameEnd); + return null; + } + + void renumberNameTable(int[]map) { + if (nameTable != null) { + for (NameEntry e : nameTable) { + if (e.backNum > 1) { + for (int i=0; i(); // 13, oni defaults to 5 + } else { + e = nameFind(name, nameP, nameEnd); + } + + if (e == null) { + // dup the name here as oni does ?, what for ? (it has to manage it, we don't) + e = new NameEntry(name, nameP, nameEnd); + nameTable.putDirect(name, nameP, nameEnd, e); + } else if (e.backNum >= 1 && !syntax.allowMultiplexDefinitionName()) { + throw new ValueException(ErrorMessages.MULTIPLEX_DEFINED_NAME, new String(name, nameP, nameEnd - nameP)); + } + + e.addBackref(backRef); + } + + NameEntry nameToGroupNumbers(byte[]name, int nameP, int nameEnd) { + return nameFind(name, nameP, nameEnd); + } + + public int nameToBackrefNumber(byte[]name, int nameP, int nameEnd, Region region) { + return nameToBackrefNumber(name, nameP, nameEnd, DEFAULT_ENCODING, region); + } + + public int nameToBackrefNumber(byte[]name, int nameP, int nameEnd, Encoding nameEncoding, Region region) { + NameEntry e = nameToGroupNumbers(name, nameP, nameEnd); + if (e == null) throw new ValueException(ErrorMessages.UNDEFINED_NAME_REFERENCE, + new String(name, nameP, nameEnd - nameP, nameEncoding.getCharset())); + + switch(e.backNum) { + case 0: + throw new InternalException(ErrorMessages.PARSER_BUG); + case 1: + return e.backRef1; + default: + if (region != null) { + for (int i = e.backNum - 1; i >= 0; i--) { + if (region.getBeg(e.backRefs[i]) != Region.REGION_NOTPOS) return e.backRefs[i]; + } + } + return e.backRefs[e.backNum - 1]; + } + } + + String nameTableToString() { + StringBuilder sb = new StringBuilder(); + + if (nameTable != null) { + sb.append("name table\n"); + for (NameEntry ne : nameTable) { + sb.append(" ").append(ne).append("\n"); + } + sb.append("\n"); + } + return sb.toString(); + } + + public Iterator namedBackrefIterator() { + return nameTable == null ? Collections.emptyIterator() : nameTable.iterator(); + } + + public int numberOfNames() { + return nameTable == null ? 0 : nameTable.size(); + } + + public boolean noNameGroupIsActive(Syntax syntax) { + if (isDontCaptureGroup(options)) return false; + + if (Config.USE_NAMED_GROUP) { + if (numberOfNames() > 0 && syntax.captureOnlyNamedGroup() && !isCaptureGroup(options)) return false; + } + return true; + } + + /* set skip map for Boyer-Moor search */ + boolean setupBMSkipMap(boolean ignoreCase) { + byte[]bytes = exact; + int s = exactP; + int end = exactEnd; + int len = end - s; + int clen; + CaseFoldCodeItem[]items = CaseFoldCodeItem.EMPTY_FOLD_CODES; + byte[]buf = new byte[Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM * Config.ENC_MBC_CASE_FOLD_MAXLEN]; + + final int ilen = USE_SUNDAY_QUICK_SEARCH ? len : len - 1; + if (Config.USE_BYTE_MAP || len < Config.CHAR_TABLE_SIZE) { + if (map == null) map = new byte[Config.CHAR_TABLE_SIZE]; // map/skip + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) map[i] = (byte)(USE_SUNDAY_QUICK_SEARCH ? len + 1 : len); + + for (int i = 0; i < ilen; i += clen) { + if (ignoreCase) items = enc.caseFoldCodesByString(caseFoldFlag, bytes, s + i, end); + clen = setupBMSkipMapCheck(bytes, s + i, end, items, buf); + if (clen == 0) return true; + + for (int j = 0; j < clen; j++) { + map[bytes[s + i + j] & 0xff] = (byte)(ilen - i - j); + for (int k = 0; k < items.length; k++) { + map[buf[k * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM + j] & 0xff] = (byte)(ilen - i - j); + } + } + } + } else { + if (intMap == null) intMap = new int[Config.CHAR_TABLE_SIZE]; + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) intMap[i] = (USE_SUNDAY_QUICK_SEARCH ? len + 1 : len); + + for (int i = 0; i < ilen; i += clen) { + if (ignoreCase) items = enc.caseFoldCodesByString(caseFoldFlag, bytes, s + i, end); + clen = setupBMSkipMapCheck(bytes, s + i, end, items, buf); + if (clen == 0) return true; + + for (int j = 0; j < clen; j++) { + intMap[bytes[s + i + j] & 0xff] = ilen - i - j; + for (int k = 0; k < items.length; k++) { + intMap[buf[k * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM + j] & 0xff] = ilen - i - j; + } + } + } + } + return false; + } + + private int setupBMSkipMapCheck(byte[]bytes, int p, int end, CaseFoldCodeItem[]items, byte[]buf) { + int clen = enc.length(bytes, p, end); + if (p + clen > end) clen = end - p; + for (int j = 0; j < items.length; j++) { + if (items[j].code.length != 1 || items[j].byteLen != clen) return 0; + int flen = enc.codeToMbc(items[j].code[0], buf, j * Config.ENC_GET_CASE_FOLD_CODES_MAX_NUM); + if (flen != clen) return 0; + } + return clen; + } + + void setOptimizeExactInfo(OptExactInfo e) { + if (e.length == 0) return; + + // shall we copy that ? + exact = e.bytes; + exactP = 0; + exactEnd = e.length; + boolean allowReverse = enc.isReverseMatchAllowed(exact, exactP, exactEnd); + + if (e.ignoreCase > 0) { + if (e.length >= 3 || (e.length >= 2 && allowReverse)) { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + if (!setupBMSkipMap(true)) { + forward = allowReverse ? (enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD) : Search.BM_NOT_REV_IC_FORWARD; + // FIXME: put above line in place to work around some failures. Either BM_IC_FORWARD is broken here or we are choosing it when we shouldn't. + //forward = allowReverse ? Search.BM_IC_FORWARD : Search.BM_NOT_REV_IC_FORWARD; + } else { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + } + } else { + forward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_FORWARD : Search.SLOW_IC_FORWARD; + } + backward = enc.toLowerCaseTable() != null ? Search.SLOW_IC_SB_BACKWARD : Search.SLOW_IC_BACKWARD; + } else { + if (e.length >= 3 || (e.length >= 2 && allowReverse)) { + if (!setupBMSkipMap(false)) { + forward = allowReverse ? Search.BM_FORWARD : Search.BM_NOT_REV_FORWARD; + } else { + forward = enc.isSingleByte() ? Search.SLOW_SB_FORWARD : Search.SLOW_FORWARD; + } + } else { + forward = enc.isSingleByte() ? Search.SLOW_SB_FORWARD : Search.SLOW_FORWARD; + } + backward = enc.isSingleByte() ? Search.SLOW_SB_BACKWARD : Search.SLOW_BACKWARD; + } + + dMin = e.mmd.min; + dMax = e.mmd.max; + + if (dMin != MinMaxLen.INFINITE_DISTANCE) { + thresholdLength = dMin + (exactEnd - exactP); + } + } + + void setOptimizeMapInfo(OptMapInfo m) { + map = m.map; + + if (enc.isSingleByte()) { + forward = Search.MAP_SB_FORWARD; + backward = Search.MAP_SB_BACKWARD; + } else { + forward = Search.MAP_FORWARD; + backward = Search.MAP_BACKWARD; + } + + dMin = m.mmd.min; + dMax = m.mmd.max; + + if (dMin != MinMaxLen.INFINITE_DISTANCE) { + thresholdLength = dMin + 1; + } + } + + void setSubAnchor(OptAnchorInfo anc) { + subAnchor |= anc.leftAnchor & AnchorType.BEGIN_LINE; + subAnchor |= anc.rightAnchor & AnchorType.END_LINE; + } + + void clearOptimizeInfo() { + forward = null; + backward = null; + anchor = 0; + anchorDmax = 0; + anchorDmin = 0; + subAnchor = 0; + + exact = null; + exactP = exactEnd = 0; + } + + public String optimizeInfoToString() { + String s = ""; + s += "optimize: " + (forward != null ? forward.getName() : "NONE") + "\n"; + s += " anchor: " + OptAnchorInfo.anchorToString(anchor); + + if ((anchor & AnchorType.END_BUF_MASK) != 0) { + s += MinMaxLen.distanceRangeToString(anchorDmin, anchorDmax); + } + + s += "\n"; + + if (forward != null) { + s += " sub anchor: " + OptAnchorInfo.anchorToString(subAnchor) + "\n"; + } + + s += "dmin: " + dMin + " dmax: " + dMax + "\n"; + s += "threshold length: " + thresholdLength + "\n"; + + if (exact != null) { + s += "exact: [" + new String(exact, exactP, exactEnd - exactP) + "]: length: " + (exactEnd - exactP) + "\n"; + } else if (forward == Search.MAP_FORWARD || forward == Search.MAP_SB_FORWARD) { + int n=0; + for (int i=0; i 0) { + int c=0; + s += "["; + for (int i=0; i 0) s += ", "; + c++; + if (enc.maxLength() == 1 && enc.isPrint(i)) s += ((char)i); + else s += i; + } + } + s += "]\n"; + } + } + + return s; + } + + public Encoding getEncoding() { + return enc; + } + + public int getOptions() { + return options; + } + + public void setUserOptions(int options) { + this.userOptions = options; + } + + public int getUserOptions() { + return userOptions; + } + + public void setUserObject(Object object) { + this.userObject = object; + } + + public Object getUserObject() { + return userObject; + } + + public boolean isLinear() { + return !requireStack; + } +} diff --git a/third_party/joni/src/org/joni/Region.java b/third_party/joni/src/org/joni/Region.java new file mode 100644 index 000000000..4507c4517 --- /dev/null +++ b/third_party/joni/src/org/joni/Region.java @@ -0,0 +1,66 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public abstract class Region { + static final int REGION_NOTPOS = -1; + + protected CaptureTreeNode historyRoot; + + public static Region newRegion(int num) { + if (num == 1) return new SingleRegion(num); + return new MultiRegion(num); + } + + public static Region newRegion(int begin, int end) { + return new SingleRegion(begin, end); + } + + @Override + public abstract Region clone(); + + public abstract int getNumRegs(); + + public abstract int getBeg(int index); + + public abstract int setBeg(int index, int value); + + public abstract int getEnd(int index); + + public abstract int setEnd(int index, int value); + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Region: \n"); + for (int i=0; i= Config.MAX_CAPTURE_GROUP_NUM) throw new InternalException(ErrorMessages.TOO_MANY_CAPTURE_GROUPS); + if (numMem++ == 0) { + memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + } else if (numMem >= memNodes.length) { + EncloseNode[]tmp = new EncloseNode[memNodes.length << 1]; + System.arraycopy(memNodes, 0, tmp, 0, memNodes.length); + memNodes = tmp; + } + + return numMem; + } + + void setMemNode(int num, EncloseNode node) { + if (numMem >= num) { + memNodes[num] = node; + } else { + throw new InternalException(ErrorMessages.PARSER_BUG); + } + } + + + void pushPrecReadNotNode(Node node) { + numPrecReadNotNodes++; + if (precReadNotNodes == null) { + precReadNotNodes = new Node[Config.SCANENV_MEMNODES_SIZE]; + } else if (numPrecReadNotNodes >= precReadNotNodes.length) { + Node[]tmp = new Node[precReadNotNodes.length << 1]; + System.arraycopy(precReadNotNodes, 0, tmp, 0, precReadNotNodes.length); + precReadNotNodes = tmp; + } + precReadNotNodes[numPrecReadNotNodes - 1] = node; + } + + void popPrecReadNotNode(Node node) { + if (precReadNotNodes != null && precReadNotNodes[numPrecReadNotNodes - 1] == node) { + precReadNotNodes[numPrecReadNotNodes - 1] = null; + numPrecReadNotNodes--; + } + } + + Node currentPrecReadNotNode() { + if (numPrecReadNotNodes > 0) { + return precReadNotNodes[numPrecReadNotNodes - 1]; + } + return null; + } + + int convertBackslashValue(int c) { + if (syntax.opEscControlChars()) { + switch (c) { + case 'n': return '\n'; + case 't': return '\t'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'a': return '\007'; + case 'b': return '\010'; + case 'e': return '\033'; + case 'v': + if (syntax.op2EscVVtab()) return 11; // '\v' + break; + default: + if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')) unknownEscWarn(String.valueOf((char)c)); + } + } + return c; + } + + void ccEscWarn(String s) { + if (warnings != WarnCallback.NONE) { + if (syntax.warnCCOpNotEscaped() && syntax.backSlashEscapeInCC()) { + warnings.warn("character class has '" + s + "' without escape"); + } + } + } + + void unknownEscWarn(String s) { + if (warnings != WarnCallback.NONE) { + warnings.warn("Unknown escape \\" + s + " is ignored"); + } + } + + void closeBracketWithoutEscapeWarn(String s) { + if (warnings != WarnCallback.NONE) { + if (syntax.warnCCOpNotEscaped()) { + warnings.warn("regular expression has '" + s + "' without escape"); + } + } + } + + void ccDuplicateWarn() { + if (syntax.warnCCDup() && (warningsFlag & SyntaxProperties.WARN_CC_DUP) == 0) { + warningsFlag |= SyntaxProperties.WARN_CC_DUP; + // FIXME: #34 points out problem and what it will take to uncomment this (we were getting erroneous versions of this) + // warnings.warn("character class has duplicated range"); + } + } +} diff --git a/third_party/joni/src/org/joni/ScannerSupport.java b/third_party/joni/src/org/joni/ScannerSupport.java new file mode 100644 index 000000000..9b9857992 --- /dev/null +++ b/third_party/joni/src/org/joni/ScannerSupport.java @@ -0,0 +1,182 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; +import org.joni.exception.SyntaxException; +import org.joni.exception.ValueException; + +abstract class ScannerSupport extends IntHolder implements ErrorMessages { + protected final Encoding enc; // fast access to encoding + protected final byte[]bytes; // pattern + protected int p; // current scanner position + protected int stop; // pattern end (mutable) + private int lastFetched; // last fetched value for unfetch support + protected int c; // current code point + + private final int begin; // pattern begin position for reset() support + private final int end; // pattern end position for reset() support + protected int _p; // used by mark()/restore() to mark positions + + protected ScannerSupport(Encoding enc, byte[]bytes, int p, int end) { + this.enc = enc; + this.bytes = bytes; + this.begin = p; + this.end = end; + } + + protected final int getBegin() { + return begin; + } + + protected final int getEnd() { + return end; + } + + private static final int INT_SIGN_BIT = 1 << 31; + protected final int scanUnsignedNumber() { + int last = c; + int num = 0; // long ??? + while(left()) { + fetch(); + if (enc.isDigit(c)) { + int onum = num; + num = num * 10 + Encoding.digitVal(c); + if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; + } else { + unfetch(); + break; + } + } + c = last; + return num; + } + + protected final int scanUnsignedHexadecimalNumber(int minLength, int maxLength) { + int last = c; + int num = 0; + int restLen = maxLength - minLength; + while(left() && maxLength-- != 0) { + fetch(); + if (enc.isXDigit(c)) { + int val = enc.xdigitVal(c); + if ((Integer.MAX_VALUE - val) / 16 < num) return -1; + num = (num << 4) + val; + } else { + unfetch(); + maxLength++; + break; + } + } + if (maxLength > restLen) return -2; + c = last; + return num; + } + + protected final int scanUnsignedOctalNumber(int maxLength) { + int last = c; + int num = 0; + while(left() && maxLength-- != 0) { + fetch(); + if (enc.isDigit(c) && c < '8') { + int onum = num; + int val = Encoding.odigitVal(c); + num = (num << 3) + val; + if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; + } else { + unfetch(); + break; + } + } + c = last; + return num; + } + + protected final void reset() { + p = begin; + stop = end; + } + + protected final void mark() { + _p = p; + } + + protected final void restore() { + p = _p; + } + + protected final void inc() { + lastFetched = p; + p += enc.length(bytes, p, stop); + } + + protected final void fetch() { + c = enc.mbcToCode(bytes, p, stop); + lastFetched = p; + p += enc.length(bytes, p, stop); + } + + protected int fetchTo() { + int to = enc.mbcToCode(bytes, p, stop); + lastFetched = p; + p += enc.length(bytes, p, stop); + return to; + } + + protected final void unfetch() { + p = lastFetched; + } + + protected final int peek() { + return p < stop ? enc.mbcToCode(bytes, p, stop) : 0; + } + + protected final boolean peekIs(int c) { + return peek() == c; + } + + protected final boolean left() { + return p < stop; + } + + protected void newSyntaxException(String message) { + throw new SyntaxException(message); + } + + protected void newValueException(String message) { + throw new ValueException(message); + } + + protected void newValueException(String message, String str) { + throw new ValueException(message, str); + } + + protected void newValueException(String message, int p, int end) { + throw new ValueException(message, new String(bytes, p, end - p)); + } + + protected void newInternalException(String message) { + throw new InternalException(message); + } + +} diff --git a/third_party/joni/src/org/joni/Search.java b/third_party/joni/src/org/joni/Search.java new file mode 100644 index 000000000..9616d5a80 --- /dev/null +++ b/third_party/joni/src/org/joni/Search.java @@ -0,0 +1,633 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.Config.USE_SUNDAY_QUICK_SEARCH; + +import org.jcodings.Encoding; +import org.jcodings.IntHolder; + +final class Search { + + static abstract class Forward { + abstract String getName(); + abstract int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange); + } + + static abstract class Backward { + abstract int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_); + } + + private static boolean lowerCaseMatch(byte[] t, int tP, int tEnd, byte[] bytes, int p, int end, Encoding enc, byte[] buf, int caseFoldFlag) { + final IntHolder holder = new IntHolder(); + holder.value = p; + while (tP < tEnd) { + int lowlen = enc.mbcCaseFold(caseFoldFlag, bytes, holder, end, buf); + if (lowlen == 1) { + if (t[tP++] != buf[0]) + return false; + } else { + int q = 0; + while (lowlen > 0) { + if (t[tP++] != buf[q++]) + return false; + lowlen--; + } + } + } + return true; + } + + static final Forward SLOW_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + + if (t == targetEnd) return s; + } + s += enc.length(text, s, textEnd); + } + return -1; + } + }; + + static final Backward SLOW_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + s = (s > textStart) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + + while (s >= textP) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + if (t == targetEnd) return s; + } + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward SLOW_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + + if (t == targetEnd) return s; + } + s++; + } + return -1; + } + }; + + static final Backward SLOW_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + if (s > textStart) s = textStart; + + while (s >= textP) { + if (text[s] == target[targetP]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != text[p++]) break; + t++; + } + if (t == targetEnd) return s; + } + //s = s <= adjustText ? -1 : s - 1; + s--; + } + return -1; + } + }; + + static final Forward SLOW_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + if (end > textRange) end = textRange; + int s = textP; + byte[]buf = matcher.icbuf(); + + while (s < end) { + if (lowerCaseMatch(target, targetP, targetEnd, text, s, textEnd, enc, buf, regex.caseFoldFlag)) return s; + s += enc.length(text, s, textEnd); + } + return -1; + } + + }; + + static final Backward SLOW_IC_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + s = (s > textStart) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + byte[]buf = matcher.icbuf(); + + while (s >= textP) { + if (lowerCaseMatch(target, targetP, targetEnd, text, s, textEnd, enc, buf, regex.caseFoldFlag)) return s; + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward SLOW_IC_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_IC_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + final byte[]toLowerTable = regex.enc.toLowerCaseTable(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end = textEnd; + end -= targetEnd - targetP - 1; + + if (end > textRange) end = textRange; + int s = textP; + + while (s < end) { + if (target[targetP] == toLowerTable[text[s] & 0xff]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != toLowerTable[text[p++] & 0xff]) break; + t++; + } + + if (t == targetEnd) return s; + } + s++; + } + return -1; + } + + }; + + static final Backward SLOW_IC_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + final byte[]toLowerTable = regex.enc.toLowerCaseTable(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int s = textEnd; + s -= targetEnd - targetP; + + if (s > textStart) s = textStart; + + while (s >= textP) { + if (target[targetP] == toLowerTable[text[s] & 0xff]) { + int p = s + 1; + int t = targetP + 1; + while (t < targetEnd) { + if (target[t] != toLowerTable[text[p++] & 0xff]) break; + t++; + } + if (t == targetEnd) return s; + } + //s = s <= adjustText ? -1 : s - 1; + s--; + } + return -1; + } + }; + + static final Forward BM_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end, s; + int tail = targetEnd - 1; + if (USE_SUNDAY_QUICK_SEARCH) { + int tlen1 = tail - targetP; + end = textRange + tlen1; + s = textP + tlen1; + } else { + end = textRange + (targetEnd - targetP) - 1; + s = textP + (targetEnd - targetP) - 1; + } + if (end > textEnd) end = textEnd; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int p = s; + int t = tail; + + while (text[p] == target[t]) { + if (t == targetP) return p; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.map[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } else { /* see int_map[] */ + while (s < end) { + int p = s; + int t = tail; + + while (text[p] == target[t]) { + if (t == targetP) return p; + p--; t--; + } + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } + return -1; + } + }; + + static final Backward BM_BACKWARD = new Backward() { + private static final int BM_BACKWARD_SEARCH_LENGTH_THRESHOLD = 100; + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + if (Config.USE_INT_MAP_BACKWARD) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + if (regex.intMapBackward == null) { + if (s_ - range_ < BM_BACKWARD_SEARCH_LENGTH_THRESHOLD) { + return SLOW_BACKWARD.search(matcher, text, textP, adjustText, textEnd, textStart, s_, range_); // goto exact_method; + } + setBmBackwardSkip(regex, target, targetP, targetEnd); + } + + int s = textEnd - (targetEnd - targetP); + s = (textStart < s) ? textStart : enc.leftAdjustCharHead(text, adjustText, s, textEnd); + + while (s >= textP) { + int p = s; + int t = targetP; + while (t < targetEnd && text[p] == target[t]) { + p++; t++; + } + if (t == targetEnd) return s; + + s -= regex.intMapBackward[text[s] & 0xff]; + s = enc.leftAdjustCharHead(text, adjustText, s, textEnd); + } + return -1; + } else { + return SLOW_BACKWARD.search(matcher, text, textP, adjustText, textEnd, textStart, s_, range_); // goto exact_method; + } + } + + private void setBmBackwardSkip(Regex regex, byte[]bytes, int p, int end) { + final int[] skip; + if (regex.intMapBackward == null) { + regex.intMapBackward = skip = new int[Config.CHAR_TABLE_SIZE]; + } else { + skip = regex.intMapBackward; + } + + int len = end - p; + + for (int i = 0; i < Config.CHAR_TABLE_SIZE; i++) skip[i] = len; + for (int i = len - 1; i > 0; i--) skip[bytes[i] & 0xff] = i; + } + }; + + static final Forward BM_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]buf = matcher.icbuf(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int end, s; + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + if (USE_SUNDAY_QUICK_SEARCH) { + end = textRange + tlen1; + s = textP + tlen1; + } else { + end = textRange + (targetEnd - targetP) - 1; + s = textP + (targetEnd - targetP) - 1; + } + if (end > textEnd) end = textEnd; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int p = USE_SUNDAY_QUICK_SEARCH ? s - tlen1 : s - (targetEnd - targetP) + 1; + if (lowerCaseMatch(target, targetP, targetEnd, text, p, s + 1, enc, buf, regex.caseFoldFlag)) return p; + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.map[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } else { /* see int_map[] */ + while (s < end) { + int p = USE_SUNDAY_QUICK_SEARCH ? s - tlen1 : s - (targetEnd - targetP) + 1; + if (lowerCaseMatch(target, targetP, targetEnd, text, p, s + 1, enc, buf, regex.caseFoldFlag)) return p; + + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + s += regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? s + 1 : s] & 0xff]; + } + } + return -1; + } + }; + + static final Forward BM_NOT_REV_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_NOT_REV_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + int end = textRange; + + if (end + tlen1 > textEnd) end = textEnd - tlen1; + int s = textP, p, se; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + p = se = s + tlen1; + int t = tail; + while (text[p] == target[t]) { + if (t == targetP) return s; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.map[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } else { + while (s < end) { + p = se = s + tlen1; + int t = tail; + while (text[p] == target[t]) { + if (t == targetP) return s; + p--; t--; + } + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + + } + } + return -1; + } + }; + + static final Forward BM_NOT_REV_IC_FORWARD = new Forward() { + @Override + final String getName() { + return "EXACT_BM_NOT_REV_IC_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]buf = matcher.icbuf(); + byte[]target = regex.exact; + int targetP = regex.exactP; + int targetEnd = regex.exactEnd; + + int tail = targetEnd - 1; + int tlen1 = tail - targetP; + int end = textRange; + + if (end + tlen1 > textEnd) end = textEnd - tlen1; + int s = textP; + + if (Config.USE_BYTE_MAP || regex.intMap == null) { + while (s < end) { + int se = s + tlen1; + if (lowerCaseMatch(target, targetP, targetEnd, text, s, se + 1, enc, buf, regex.caseFoldFlag)) return s; + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.map[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + int t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } else { + while (s < end) { + int se = s + tlen1; + if (lowerCaseMatch(target, targetP, targetEnd, text, s, se + 1, enc, buf, regex.caseFoldFlag)) return s; + if (USE_SUNDAY_QUICK_SEARCH && (s + 1 >= end)) break; + int skip = regex.intMap[text[USE_SUNDAY_QUICK_SEARCH ? se + 1 : se] & 0xff]; + int t = s; + do { + s += enc.length(text, s, textEnd); + } while ((s - t) < skip && s < end); + } + } + return -1; + } + }; + + static final Forward MAP_FORWARD = new Forward() { + @Override + final String getName() { + return "MAP_FORWARD"; + } + + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]map = regex.map; + int s = textP; + + while (s < textRange) { + if (map[text[s] & 0xff] != 0) return s; + s += enc.length(text, s, textEnd); + } + return -1; + } + }; + + static final Backward MAP_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + Encoding enc = regex.enc; + byte[]map = regex.map; + int s = textStart; + + if (s >= textEnd) s = textEnd - 1; // multibyte safe ? + while (s >= textP) { + if (map[text[s] & 0xff] != 0) return s; + s = enc.prevCharHead(text, adjustText, s, textEnd); + } + return -1; + } + }; + + static final Forward MAP_SB_FORWARD = new Forward() { + @Override + final String getName() { + return "MAP_SB_FORWARD"; + } + @Override + final int search(Matcher matcher, byte[]text, int textP, int textEnd, int textRange) { + Regex regex = matcher.regex; + byte[]map = regex.map; + int s = textP; + + while (s < textRange) { + if (map[text[s] & 0xff] != 0) return s; + s++; + } + return -1; + } + }; + + static final Backward MAP_SB_BACKWARD = new Backward() { + @Override + final int search(Matcher matcher, byte[]text, int textP, int adjustText, int textEnd, int textStart, int s_, int range_) { + Regex regex = matcher.regex; + byte[]map = regex.map; + int s = textStart; + + if (s >= textEnd) s = textEnd - 1; + while (s >= textP) { + if (map[text[s] & 0xff] != 0) return s; + s--; + } + return -1; + } + }; +} diff --git a/third_party/joni/src/org/joni/SingleRegion.java b/third_party/joni/src/org/joni/SingleRegion.java new file mode 100644 index 000000000..3064264cc --- /dev/null +++ b/third_party/joni/src/org/joni/SingleRegion.java @@ -0,0 +1,75 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +public class SingleRegion extends Region { + int beg; + int end; + + public SingleRegion(int num) { + if (num != 1) throw new IndexOutOfBoundsException(""+num); + } + + public SingleRegion(int begin, int end) { + this.beg = begin; + this.end = end; + } + + @Override + public int getNumRegs() { + return 1; + } + + @Override + public SingleRegion clone() { + SingleRegion region = new SingleRegion(beg, end); + if (getCaptureTree() != null) region.setCaptureTree(getCaptureTree().cloneTree()); + return region; + } + + @Override + public int getBeg(int index) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return beg; + } + + @Override + public int setBeg(int index, int value) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return beg = value; + } + + @Override + public int getEnd(int index) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return end; + } + + @Override + public int setEnd(int index, int value) { + if (index != 0) throw new IndexOutOfBoundsException(""+index); + return end = value; + } + + @Override + void clear() { + beg = end = REGION_NOTPOS; + } +} diff --git a/third_party/joni/src/org/joni/StackEntry.java b/third_party/joni/src/org/joni/StackEntry.java new file mode 100644 index 000000000..2239e3260 --- /dev/null +++ b/third_party/joni/src/org/joni/StackEntry.java @@ -0,0 +1,202 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +class StackEntry { + int type; + private int E1, E2, E3, E4; + private Object calloutToken; + + // first union member + /* byte code position */ + void setStatePCode(int pcode) { + E1 = pcode; + } + int getStatePCode() { + return E1; + } + /* string position */ + void setStatePStr(int pstr) { + E2 = pstr; + } + int getStatePStr() { + return E2; + } + /* previous char position of pstr */ + void setStatePStrPrev(int pstrPrev) { + E3 = pstrPrev; + } + int getStatePStrPrev() { + return E3; + } + + void setPKeep(int pkeep) { + E4 = pkeep; + } + int getPKeep() { + return E4; + } + + // second union member + /* for OP_REPEAT_INC, OP_REPEAT_INC_NG */ + void setRepeatCount(int count) { + E1 = count; + } + int getRepeatCount() { + return E1; + } + void decreaseRepeatCount() { + E1--; + } + void increaseRepeatCount() { + E1++; + } + /* byte code position (head of repeated target) */ + void setRepeatPCode(int pcode) { + E2 = pcode; + } + int getRepeatPCode() { + return E2; + } + /* repeat id */ + void setRepeatNum(int num) { + E3 = num; + } + int getRepeatNum() { + return E3; + } + + // third union member + /* index of stack */ /*int repeat_inc struct*/ + void setSi(int si) { + E1 = si; + } + int getSi() { + return E1; + } + + // fourth union member + /* memory num */ + void setMemNum(int num) { + E1 = num; + } + int getMemNum() { + return E1; + } + /* start/end position */ + void setMemPstr(int pstr) { + E2 = pstr; + } + int getMemPStr() { + return E2; + } + + /* Following information is set, if this stack type is MEM-START */ + /* prev. info (for backtrack "(...)*" ) */ + void setMemStart(int start) { + E3 = start; + } + int getMemStart() { + return E3; + } + /* prev. info (for backtrack "(...)*" ) */ + void setMemEnd(int end) { + E4 = end; + } + int getMemEnd() { + return E4; + } + + // fifth union member + /* null check id */ + void setNullCheckNum(int num) { + E1 = num; + } + int getNullCheckNum() { + return E1; + } + /* start position */ + void setNullCheckPStr(int pstr) { + E2 = pstr; + } + int getNullCheckPStr() { + return E2; + } + + // sixth union member + /* byte code position */ + void setCallFrameRetAddr(int addr) { + E1 = addr; + } + int getCallFrameRetAddr() { + return E1; + } + /* null check id */ + void setCallFrameNum(int num) { + E2 = num; + } + int getCallFrameNum() { + return E2; + } + /* string position */ + void setCallFramePStr(int pstr) { + E3 = pstr; + } + int getCallFramePStr() { + return E3; + } + + /* absent position */ + void setAbsentStr(int pos) { + E1 = pos; + } + int getAbsentStr() { + return E1; + } + + void setAbsentEndStr(int pos) { + E2 = pos; + } + int getAbsentEndStr() { + return E2; + } + + void setCalloutToken(Object token) { + calloutToken = token; + } + + Object takeCalloutToken() { + Object token = calloutToken; + calloutToken = null; + return token; + } +} + +final class SCStackEntry extends StackEntry { + private int E5; + + void setStateCheck(int check) { + E5 = check; + } + + int getStateCheck() { + return E5; + } +} diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java new file mode 100644 index 000000000..d3fb30433 --- /dev/null +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -0,0 +1,647 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.BitStatus.bsAt; +import static org.joni.Config.USE_CEC; + +import java.lang.ref.WeakReference; +import java.util.Arrays; + +import org.joni.constants.internal.StackPopLevel; +import org.joni.constants.internal.StackType; + +abstract class StackMachine extends Matcher implements StackType { + protected static final int INVALID_INDEX = -1; + + protected StackEntry[]stack; + protected int stk; // stkEnd + protected final int[]repeatStk; + protected final int memStartStk, memEndStk; + protected byte[] stateCheckBuff; // CEC, move to int[] ? + protected int stateCheckBuffSize; + + protected StackMachine(Regex regex, Region region, byte[]bytes, int p , int end) { + super(regex, region, bytes, p, end); + stack = regex.requireStack ? fetchStack() : null; + final int n; + if (Config.USE_SUBEXP_CALL) { + n = regex.numRepeat + ((regex.numMem + 1) << 1); + memStartStk = regex.numRepeat; + memEndStk = memStartStk + regex.numMem + 1; + } else { + n = regex.numRepeat + (regex.numMem << 1); + memStartStk = regex.numRepeat - 1; + memEndStk = memStartStk + regex.numMem; + /* for index start from 1, mem_start_stk[1]..mem_start_stk[num_mem] */ + /* for index start from 1, mem_end_stk[1]..mem_end_stk[num_mem] */ + } + repeatStk = n > 0 ? new int[n] : null; + } + + protected final void stackInit() { + if (stack != null) pushEnsured(ALT, regex.codeLength - 1); /* bottom stack */ + if (repeatStk != null) { + for (int i = (Config.USE_SUBEXP_CALL ? 0 : 1); i <= regex.numMem; i++) { + repeatStk[i + memStartStk] = repeatStk[i + memEndStk] = INVALID_INDEX; + } + } + } + + private static StackEntry[] allocateStack() { + StackEntry[]stack = new StackEntry[Config.INIT_MATCH_STACK_SIZE]; + stack[0] = USE_CEC ? new SCStackEntry() : new StackEntry(); + return stack; + } + + private void doubleStack() { + StackEntry[] newStack = new StackEntry[stack.length << 1]; + System.arraycopy(stack, 0, newStack, 0, stack.length); + stack = newStack; + } + + static final ThreadLocal> stacks + = new ThreadLocal<>(); + + private static StackEntry[] fetchStack() { + WeakReference ref = stacks.get(); + StackEntry[] stack; + if (ref == null) { + stacks.set( new WeakReference<>(stack = allocateStack()) ); + } + else { + stack = ref.get(); + if (stack == null) { + stacks.set( new WeakReference<>(stack = allocateStack()) ); + } + } + return stack; + } + + private final StackEntry ensure1() { + if (stk >= stack.length) doubleStack(); + StackEntry e = stack[stk]; + if (e == null) stack[stk] = e = USE_CEC ? new SCStackEntry() : new StackEntry(); + return e; + } + + private final void pushType(int type) { + ensure1().type = type; + stk++; + } + + // CEC + + // STATE_CHECK_POS + private int stateCheckPos(int s, int snum) { + return (s - str) * regex.numCombExpCheck + (snum - 1); + } + + // STATE_CHECK_VAL + protected final boolean stateCheckVal(int s, int snum) { + if (stateCheckBuff != null) { + int x = stateCheckPos(s, snum); + return (stateCheckBuff[x / 8] & (1 << (x % 8))) != 0; + } + return false; + } + + // ELSE_IF_STATE_CHECK_MARK + private void stateCheckMark() { + StackEntry e = stack[stk]; + int x = stateCheckPos(e.getStatePStr(), ((SCStackEntry)e).getStateCheck()); + stateCheckBuff[x / 8] |= (1 << (x % 8)); + } + + // STATE_CHECK_BUFF_INIT + private static final int STATE_CHECK_BUFF_MALLOC_THRESHOLD_SIZE = 16; + @Override + protected final void stateCheckBuffInit(int strLength, int offset, int stateNum) { + if (stateNum > 0 && strLength >= Config.CHECK_STRING_THRESHOLD_LEN) { + int size = ((strLength + 1) * stateNum + 7) >>> 3; + offset = (offset * stateNum) >>> 3; + + if (size > 0 && offset < size && size < Config.CHECK_BUFF_MAX_SIZE) { + if (size >= STATE_CHECK_BUFF_MALLOC_THRESHOLD_SIZE) { + stateCheckBuff = new byte[size]; + } else { + // same impl, reduce... + stateCheckBuff = new byte[size]; + } + Arrays.fill(stateCheckBuff, offset, (size - offset), (byte)0); + stateCheckBuffSize = size; + } else { + stateCheckBuff = null; // reduce + stateCheckBuffSize = 0; + } + } else { + stateCheckBuff = null; // reduce + stateCheckBuffSize = 0; + } + } + + @Override + protected final void stateCheckBuffClear() { + stateCheckBuff = null; + stateCheckBuffSize = 0; + } + + private void push(int type, int pat, int s, int prev, int pkeep) { + StackEntry e = ensure1(); + e.type = type; + e.setStatePCode(pat); + e.setStatePStr(s); + e.setStatePStrPrev(prev); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(0); + e.setPKeep(pkeep); + stk++; + } + + private final void pushEnsured(int type, int pat) { + StackEntry e = stack[stk]; + e.type = type; + e.setStatePCode(pat); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(0); + stk++; + } + + protected final void pushAltWithStateCheck(int pat, int s, int sprev, int snum, int pkeep) { + StackEntry e = ensure1(); + e.type = ALT; + e.setStatePCode(pat); + e.setStatePStr(s); + e.setStatePStrPrev(sprev); + if (USE_CEC) ((SCStackEntry)e).setStateCheck(stateCheckBuff != null ? snum : 0); + e.setPKeep(pkeep); + stk++; + } + + protected final void pushStateCheck(int s, int snum) { + if (stateCheckBuff != null) { + StackEntry e = ensure1(); + e.type = STATE_CHECK_MARK; + e.setStatePStr(s); + ((SCStackEntry)e).setStateCheck(snum); + stk++; + } + } + + protected final void pushAlt(int pat, int s, int prev, int pkeep) { + push(ALT, pat, s, prev, pkeep); + } + + protected final void pushPos(int s, int prev, int pkeep) { + push(POS, -1 /*NULL_UCHARP*/, s, prev, pkeep); + } + + protected final void pushPosNot(int pat, int s, int prev, int pkeep) { + push(POS_NOT, pat, s, prev, pkeep); + } + + protected final void pushStopBT() { + pushType(STOP_BT); + } + + protected final void pushLookBehindNot(int pat, int s, int sprev, int pkeep) { + push(LOOK_BEHIND_NOT, pat, s, sprev, pkeep); + } + + protected final void pushRepeat(int id, int pat) { + StackEntry e = ensure1(); + e.type = REPEAT; + e.setRepeatNum(id); + e.setRepeatPCode(pat); + e.setRepeatCount(0); + stk++; + } + + protected final void pushRepeatInc(int sindex) { + StackEntry e = ensure1(); + e.type = REPEAT_INC; + e.setSi(sindex); + stk++; + } + + protected final void pushMemStart(int mnum, int s) { + StackEntry e = ensure1(); + e.type = MEM_START; + e.setMemNum(mnum); + e.setMemPstr(s); + e.setMemStart(repeatStk[memStartStk + mnum]); + e.setMemEnd(repeatStk[memEndStk + mnum]); + repeatStk[memStartStk + mnum] = stk; + repeatStk[memEndStk + mnum] = INVALID_INDEX; + stk++; + } + + protected final void pushMemEnd(int mnum, int s) { + StackEntry e = ensure1(); + e.type = MEM_END; + e.setMemNum(mnum); + e.setMemPstr(s); + e.setMemStart(repeatStk[memStartStk + mnum]); + e.setMemEnd(repeatStk[memEndStk + mnum]); + repeatStk[memEndStk + mnum] = stk; + stk++; + } + + protected final void pushMemEndMark(int mnum) { + StackEntry e = ensure1(); + e.type = MEM_END_MARK; + e.setMemNum(mnum); + stk++; + } + + protected final int getMemStart(int mnum) { + int level = 0; + int stkp = stk; + + while (stkp > 0) { + stkp--; + StackEntry e = stack[stkp]; + if ((e.type & MASK_MEM_END_OR_MARK) != 0 && e.getMemNum() == mnum) { + level++; + } else if (e.type == MEM_START && e.getMemNum() == mnum) { + if (level == 0) break; + level--; + } + } + return stkp; + } + + protected final void pushNullCheckStart(int cnum, int s) { + StackEntry e = ensure1(); + e.type = NULL_CHECK_START; + e.setNullCheckNum(cnum); + e.setNullCheckPStr(s); + stk++; + } + + protected final void pushNullCheckEnd(int cnum) { + StackEntry e = ensure1(); + e.type = NULL_CHECK_END; + e.setNullCheckNum(cnum); + stk++; + } + + protected final void pushCallFrame(int pat) { + StackEntry e = ensure1(); + e.type = CALL_FRAME; + e.setCallFrameRetAddr(pat); + stk++; + } + + protected final void pushReturn() { + StackEntry e = ensure1(); + e.type = RETURN; + stk++; + } + + protected final void pushAbsent() { + StackEntry e = ensure1(); + e.type = ABSENT; + stk++; + } + + protected final void pushAbsentPos(int start, int end) { + StackEntry e = ensure1(); + e.type = ABSENT_POS; + e.setAbsentStr(start); + e.setAbsentEndStr(end); + stk++; + } + + protected final void pushCallout(Object token) { + StackEntry e = ensure1(); + e.type = CALLOUT; + e.setCalloutToken(token); + stk++; + } + + protected final void popOne() { + stk--; + } + + protected final StackEntry pop() { + switch (regex.stackPopLevel) { + case StackPopLevel.FREE: + return popFree(); + case StackPopLevel.MEM_START: + return popMemStart(); + default: + return popDefault(); + } + } + + private StackEntry popFree() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) { + return e; + } else if (e.type == CALLOUT) { + unwindCallout(e); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + } + + private StackEntry popMemStart() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) { + return e; + } else if (e.type == CALLOUT) { + unwindCallout(e); + } else if (e.type == MEM_START) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + } + + private void popRewrite(StackEntry e) { + if (e.type == CALLOUT) { + unwindCallout(e); + } else if (e.type == MEM_START) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (e.type == REPEAT_INC) { + stack[e.getSi()].decreaseRepeatCount(); + } else if (e.type == MEM_END) { + repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); + repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); + } else if (USE_CEC) { + if (e.type == STATE_CHECK_MARK) stateCheckMark(); + } + } + + protected final void unwindActiveCallouts() { + if (stack == null) return; + for (int i = stk - 1; i >= 0; i--) { + if (stack[i].type == CALLOUT) unwindCallout(stack[i]); + } + } + + protected final void completeActiveCallouts() { + if (stack == null) return; + for (int i = stk - 1; i >= 0; i--) { + if (stack[i].type == CALLOUT) completeCallout(stack[i]); + } + } + + private void completeCallout(StackEntry entry) { + Object token = entry.takeCalloutToken(); + if (token != null) getCalloutHandler().complete(token); + } + + private void unwindCallout(StackEntry entry) { + Object token = entry.takeCalloutToken(); + if (token != null) getCalloutHandler().unwind(token); + } + + private StackEntry popDefault() { + while (true) { + StackEntry e = stack[--stk]; + if ((e.type & MASK_POP_USED) != 0) return e; else popRewrite(e); + } + } + + protected final void popTilPosNot() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == POS_NOT) break; else popRewrite(e); + } + } + + protected final void popTilLookBehindNot() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == LOOK_BEHIND_NOT) break; else popRewrite(e); + } + } + + protected final void popTilAbsent() { + while (true) { + StackEntry e = stack[--stk]; + if (e.type == ABSENT) break; else popRewrite(e); + } + } + + protected final int posEnd() { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + if ((e.type & MASK_TO_VOID_TARGET) != 0) { + e.type = VOID; + } else if (e.type == POS) { + e.type = VOID; + break; + } + } + return k; + } + + protected final void stopBtEnd() { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if ((e.type & MASK_TO_VOID_TARGET) != 0) { + e.type = VOID; + } else if (e.type == STOP_BT) { + e.type = VOID; + break; + } + } + } + + // int for consistency with other null check routines + protected final int nullCheck(int id, int s) { + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + return e.getNullCheckPStr() == s ? 1 : 0; + } + } + } + } + + protected final int nullCheckRec(int id, int s) { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (level == 0) { + return e.getNullCheckPStr() == s ? 1 : 0; + } else { + level--; + } + } + } else if (e.type == NULL_CHECK_END) { + level++; + } + } + } + + protected final int nullCheckMemSt(int id, int s) { + int k = stk; + int isNull; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (e.getNullCheckPStr() != s) { + isNull = 0; + break; + } else { + int endp; + isNull = 1; + while (k < stk) { + e = stack[k++]; + if (e.type == MEM_START) { + if (e.getMemEnd() == INVALID_INDEX) { + isNull = 0; + break; + } + if (bsAt(regex.btMemEnd, e.getMemNum())) { + endp = stack[e.getMemEnd()].getMemPStr(); + } else { + endp = e.getMemEnd(); + } + if (stack[e.getMemStart()].getMemPStr() != endp) { + isNull = 0; + break; + } else if (endp != s) { + isNull = -1; /* empty, but position changed */ + } + } + } + break; + } + } + } + } + return isNull; + } + + protected final int nullCheckMemStRec(int id, int s) { + int level = 0; + int k = stk; + int isNull; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == NULL_CHECK_START) { + if (e.getNullCheckNum() == id) { + if (level == 0) { + if (e.getNullCheckPStr() != s) { + isNull = 0; + break; + } else { + int endp; + isNull = 1; + while (k < stk) { + if (e.type == MEM_START) { + if (e.getMemEnd() == INVALID_INDEX) { + isNull = 0; + break; + } + if (bsAt(regex.btMemEnd, e.getMemNum())) { + endp = stack[e.getMemEnd()].getMemPStr(); + } else { + endp = e.getMemEnd(); + } + if (stack[e.getMemStart()].getMemPStr() != endp) { + isNull = 0; + break; + } else if (endp != s) { + isNull = -1; /* empty, but position changed */ + } + } + k++; + e = stack[k]; + } + break; + } + } else { + level--; + } + } + } else if (e.type == NULL_CHECK_END) { + if (e.getNullCheckNum() == id) level++; + } + } + return isNull; + } + + protected final int getRepeat(int id) { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == REPEAT) { + if (level == 0) { + if (e.getRepeatNum() == id) return k; + } + } else if (e.type == CALL_FRAME) { + level--; + } else if (e.type == RETURN) { + level++; + } + } + } + + protected final int sreturn() { + int level = 0; + int k = stk; + while (true) { + k--; + StackEntry e = stack[k]; + + if (e.type == CALL_FRAME) { + if (level == 0) { + return e.getCallFrameRetAddr(); + } else { + level--; + } + } else if (e.type == RETURN) { + level++; + } + } + } +} diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java new file mode 100644 index 000000000..c24db21a6 --- /dev/null +++ b/third_party/joni/src/org/joni/Syntax.java @@ -0,0 +1,753 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.joni.constants.MetaChar.INEFFECTIVE_META_CHAR; + +import org.joni.constants.SyntaxProperties; + +public final class Syntax implements SyntaxProperties { + public final String name; + public final int op; + public final int op2; + public final int op3; + public final int behavior; + public final int options; + public final MetaCharTable metaCharTable; + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable) { + this.name = name; + this.op = op; + this.op2 = op2; + this.op3 = op3; + this.behavior = behavior; + this.options = options; + this.metaCharTable = metaCharTable; + } + + public static class MetaCharTable { + public final int esc; + public final int anyChar; + public final int anyTime; + public final int zeroOrOneTime; + public final int oneOrMoreTime; + public final int anyCharAnyTime; + + public MetaCharTable(int esc, int anyChar, int anyTime, + int zeroOrOneTime, int oneOrMoreTime, int anyCharAnyTime) { + this.esc = esc; + this.anyChar = anyChar; + this.anyTime = anyTime; + this.zeroOrOneTime = zeroOrOneTime; + this.oneOrMoreTime = oneOrMoreTime; + this.anyCharAnyTime = anyCharAnyTime; + } + } + + /** + * OP + * + */ + protected boolean isOp(int opm) { + return (op & opm) != 0; + } + + public boolean opVariableMetaCharacters() { + return isOp(OP_VARIABLE_META_CHARACTERS); + } + + public boolean opDotAnyChar() { + return isOp(OP_DOT_ANYCHAR); + } + + public boolean opAsteriskZeroInf() { + return isOp(OP_ASTERISK_ZERO_INF); + } + + public boolean opEscAsteriskZeroInf() { + return isOp(OP_ESC_ASTERISK_ZERO_INF); + } + + public boolean opPlusOneInf() { + return isOp(OP_PLUS_ONE_INF); + } + + public boolean opEscPlusOneInf() { + return isOp(OP_ESC_PLUS_ONE_INF); + } + + public boolean opQMarkZeroOne() { + return isOp(OP_QMARK_ZERO_ONE); + } + + public boolean opEscQMarkZeroOne() { + return isOp(OP_ESC_QMARK_ZERO_ONE); + } + + public boolean opBraceInterval() { + return isOp(OP_BRACE_INTERVAL); + } + + public boolean opEscBraceInterval() { + return isOp(OP_ESC_BRACE_INTERVAL); + } + + public boolean opVBarAlt() { + return isOp(OP_VBAR_ALT); + } + + public boolean opEscVBarAlt() { + return isOp(OP_ESC_VBAR_ALT); + } + + public boolean opLParenSubexp() { + return isOp(OP_LPAREN_SUBEXP); + } + + public boolean opEscLParenSubexp() { + return isOp(OP_ESC_LPAREN_SUBEXP); + } + + public boolean opEscAZBufAnchor() { + return isOp(OP_ESC_AZ_BUF_ANCHOR); + } + + public boolean opEscCapitalGBeginAnchor() { + return isOp(OP_ESC_CAPITAL_G_BEGIN_ANCHOR); + } + + public boolean opDecimalBackref() { + return isOp(OP_DECIMAL_BACKREF); + } + + public boolean opBracketCC() { + return isOp(OP_BRACKET_CC); + } + + public boolean opEscWWord() { + return isOp(OP_ESC_W_WORD); + } + + public boolean opEscLtGtWordBeginEnd() { + return isOp(OP_ESC_LTGT_WORD_BEGIN_END); + } + + public boolean opEscBWordBound() { + return isOp(OP_ESC_B_WORD_BOUND); + } + + public boolean opEscSWhiteSpace() { + return isOp(OP_ESC_S_WHITE_SPACE); + } + + public boolean opEscDDigit() { + return isOp(OP_ESC_D_DIGIT); + } + + public boolean opLineAnchor() { + return isOp(OP_LINE_ANCHOR); + } + + public boolean opPosixBracket() { + return isOp(OP_POSIX_BRACKET); + } + + public boolean opQMarkNonGreedy() { + return isOp(OP_QMARK_NON_GREEDY); + } + + public boolean opEscControlChars() { + return isOp(OP_ESC_CONTROL_CHARS); + } + + public boolean opEscCControl() { + return isOp(OP_ESC_C_CONTROL); + } + + public boolean opEscOctal3() { + return isOp(OP_ESC_OCTAL3); + } + + public boolean opEscXHex2() { + return isOp(OP_ESC_X_HEX2); + } + + public boolean opEscXBraceHex8() { + return isOp(OP_ESC_X_BRACE_HEX8); + } + + public boolean opEscOBraceOctal() { + return isOp(OP_ESC_O_BRACE_OCTAL); + } + + /** + * OP + * + */ + protected boolean isOp2(int opm) { + return (op2 & opm) != 0; + } + + public boolean op2EscCapitalQQuote() { + return isOp2(OP2_ESC_CAPITAL_Q_QUOTE); + } + + public boolean op2QMarkGroupEffect() { + return isOp2(OP2_QMARK_GROUP_EFFECT); + } + + public boolean op2OptionPerl() { + return isOp2(OP2_OPTION_PERL); + } + + public boolean op2OptionRuby() { + return isOp2(OP2_OPTION_RUBY); + } + + public boolean op2PlusPossessiveRepeat() { + return isOp2(OP2_PLUS_POSSESSIVE_REPEAT); + } + + public boolean op2PlusPossessiveInterval() { + return isOp2(OP2_PLUS_POSSESSIVE_INTERVAL); + } + + public boolean op2CClassSetOp() { + return isOp2(OP2_CCLASS_SET_OP); + } + + public boolean op2QMarkLtNamedGroup() { + return isOp2(OP2_QMARK_LT_NAMED_GROUP); + } + + public boolean op2EscKNamedBackref() { + return isOp2(OP2_ESC_K_NAMED_BACKREF); + } + + public boolean op2EscGSubexpCall() { + return isOp2(OP2_ESC_G_SUBEXP_CALL); + } + + public boolean op2AtMarkCaptureHistory() { + return isOp2(OP2_ATMARK_CAPTURE_HISTORY); + } + + public boolean op2EscCapitalCBarControl() { + return isOp2(OP2_ESC_CAPITAL_C_BAR_CONTROL); + } + + public boolean op2EscCapitalMBarMeta() { + return isOp2(OP2_ESC_CAPITAL_M_BAR_META); + } + + public boolean op2EscVVtab() { + return isOp2(OP2_ESC_V_VTAB); + } + + public boolean op2EscUHex4() { + return isOp2(OP2_ESC_U_HEX4); + } + + public boolean op2EscGnuBufAnchor() { + return isOp2(OP2_ESC_GNU_BUF_ANCHOR); + } + + public boolean op2EscPBraceCharProperty() { + return isOp2(OP2_ESC_P_BRACE_CHAR_PROPERTY); + } + + public boolean op2EscPBraceCircumflexNot() { + return isOp2(OP2_ESC_P_BRACE_CIRCUMFLEX_NOT); + } + + public boolean op2EscHXDigit() { + return isOp2(OP2_ESC_H_XDIGIT); + } + + public boolean op2IneffectiveEscape() { + return isOp2(OP2_INEFFECTIVE_ESCAPE); + } + + public boolean op2EscCapitalRLinebreak() { + return isOp2(OP2_ESC_CAPITAL_R_LINEBREAK); + } + + public boolean op2EscCapitalXExtendedGraphemeCluster() { + return isOp2(OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER); + } + + public boolean op2EscVVerticalWhiteSpace() { + return isOp2(OP2_ESC_V_VERTICAL_WHITESPACE); + } + + public boolean op2EscHHorizontalWhiteSpace() { + return isOp2(OP2_ESC_H_HORIZONTAL_WHITESPACE); + } + + public boolean op2EscCapitalKKeep() { + return isOp2(OP2_ESC_CAPITAL_K_KEEP); + } + + public boolean op2QMarkTildeAbsent() { + return isOp2(OP2_QMARK_TILDE_ABSENT); + } + + public boolean op2EscGBraceBackref() { + return isOp2(OP2_ESC_G_BRACE_BACKREF); + } + + public boolean op2QMarkSubexpCall() { + return isOp2(OP2_QMARK_SUBEXP_CALL); + } + + public boolean op2QMarkBarBranchReset() { + return isOp2(OP2_QMARK_BAR_BRANCH_RESET); + } + + public boolean op2QMarkLParenCondition() { + return isOp2(OP2_QMARK_LPAREN_CONDITION); + } + + public boolean op2QMarkCapitalPNamedGroup() { + return isOp2(OP2_QMARK_CAPITAL_P_NAMED_GROUP); + } + + protected boolean isOp3(int opm) { + return (op3 & opm) != 0; + } + + public boolean op3OptionJava() { + return isOp3(OP3_OPTION_JAVA); + } + + public boolean op3OptionECMAScript() { + return isOp3(OP3_OPTION_ECMASCRIPT); + } + + + /** + * BEHAVIOR + * + */ + protected boolean isBehavior(int bvm) { + return (behavior & bvm) != 0; + } + + public boolean contextIndepRepeatOps() { + return isBehavior(CONTEXT_INDEP_REPEAT_OPS); + } + + public boolean contextInvalidRepeatOps() { + return isBehavior(CONTEXT_INVALID_REPEAT_OPS); + } + + public boolean allowUnmatchedCloseSubexp() { + return isBehavior(ALLOW_UNMATCHED_CLOSE_SUBEXP); + } + + public boolean allowInvalidInterval() { + return isBehavior(ALLOW_INVALID_INTERVAL); + } + + public boolean allowIntervalLowAbbrev() { + return isBehavior(ALLOW_INTERVAL_LOW_ABBREV); + } + + public boolean strictCheckBackref() { + return isBehavior(STRICT_CHECK_BACKREF); + } + + public boolean differentLengthAltLookBehind() { + return isBehavior(DIFFERENT_LEN_ALT_LOOK_BEHIND); + } + + public boolean captureOnlyNamedGroup() { + return isBehavior(CAPTURE_ONLY_NAMED_GROUP); + } + + public boolean allowMultiplexDefinitionName() { + return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME); + } + + public boolean fixedIntervalIsGreedyOnly() { + return isBehavior(FIXED_INTERVAL_IS_GREEDY_ONLY); + } + + + public boolean notNewlineInNegativeCC() { + return isBehavior(NOT_NEWLINE_IN_NEGATIVE_CC); + } + + public boolean backSlashEscapeInCC() { + return isBehavior(BACKSLASH_ESCAPE_IN_CC); + } + + public boolean allowEmptyRangeInCC() { + return isBehavior(ALLOW_EMPTY_RANGE_IN_CC); + } + + public boolean allowDoubleRangeOpInCC() { + return isBehavior(ALLOW_DOUBLE_RANGE_OP_IN_CC); + } + + public boolean warnCCOpNotEscaped() { + return isBehavior(WARN_CC_OP_NOT_ESCAPED); + } + + public boolean warnCCDup() { + return isBehavior(WARN_CC_DUP); + } + + public boolean warnReduntantNestedRepeat() { + return isBehavior(WARN_REDUNDANT_NESTED_REPEAT); + } + + public static final Syntax RUBY = new Syntax( + "RUBY", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_QMARK_GROUP_EFFECT | + OP2_OPTION_RUBY | + OP2_QMARK_LT_NAMED_GROUP | OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_G_SUBEXP_CALL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_PLUS_POSSESSIVE_REPEAT | + OP2_CCLASS_SET_OP | OP2_ESC_CAPITAL_C_BAR_CONTROL | + OP2_ESC_CAPITAL_M_BAR_META | OP2_ESC_V_VTAB | + OP2_ESC_H_XDIGIT | + OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER | + OP2_QMARK_LPAREN_CONDITION | + OP2_ESC_CAPITAL_R_LINEBREAK | + OP2_ESC_CAPITAL_K_KEEP | + OP2_QMARK_TILDE_ABSENT + ), + + 0, + + ( GNU_REGEX_BV | + ALLOW_INTERVAL_LOW_ABBREV | + DIFFERENT_LEN_ALT_LOOK_BEHIND | + CAPTURE_ONLY_NAMED_GROUP | + ALLOW_MULTIPLEX_DEFINITION_NAME | + FIXED_INTERVAL_IS_GREEDY_ONLY | + WARN_CC_OP_NOT_ESCAPED | + WARN_CC_DUP | + WARN_REDUNDANT_NESTED_REPEAT ), + + (Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax DEFAULT = RUBY; + + public static final Syntax TEST = new Syntax("TEST", RUBY.op, RUBY.op2 | OP2_ESC_U_HEX4, RUBY.op3, RUBY.behavior, RUBY.options & ~ Option.ASCII_RANGE, RUBY.metaCharTable); + + public static final Syntax ASIS = new Syntax( + "ASIS", + 0, + + OP2_INEFFECTIVE_ESCAPE, + + 0, + + 0, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PosixBasic = new Syntax( + "PosixBasic", + (POSIX_COMMON_OP | OP_ESC_LPAREN_SUBEXP | + OP_ESC_BRACE_INTERVAL ), + + 0, + + 0, + + 0, + + ( Option.SINGLELINE | Option.MULTILINE ), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PosixExtended = new Syntax( + "PosixExtended", + ( POSIX_COMMON_OP | OP_LPAREN_SUBEXP | + OP_BRACE_INTERVAL | + OP_PLUS_ONE_INF | OP_QMARK_ZERO_ONE |OP_VBAR_ALT ), + + 0, + + 0, + + ( CONTEXT_INDEP_ANCHORS | + CONTEXT_INDEP_REPEAT_OPS | CONTEXT_INVALID_REPEAT_OPS | + ALLOW_UNMATCHED_CLOSE_SUBEXP | + ALLOW_DOUBLE_RANGE_OP_IN_CC ), + + ( Option.SINGLELINE | Option.MULTILINE ), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Emacs = new Syntax( + "Emacs", + ( OP_DOT_ANYCHAR | OP_BRACKET_CC | + OP_ESC_BRACE_INTERVAL | + OP_ESC_LPAREN_SUBEXP | OP_ESC_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_PLUS_ONE_INF | + OP_QMARK_ZERO_ONE | OP_DECIMAL_BACKREF | + OP_LINE_ANCHOR | OP_ESC_CONTROL_CHARS ), + + OP2_ESC_GNU_BUF_ANCHOR, + + 0, + + ALLOW_EMPTY_RANGE_IN_CC, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Grep = new Syntax( + "Grep", + ( OP_DOT_ANYCHAR | OP_BRACKET_CC | OP_POSIX_BRACKET | + OP_ESC_BRACE_INTERVAL | OP_ESC_LPAREN_SUBEXP | + OP_ESC_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_ESC_PLUS_ONE_INF | + OP_ESC_QMARK_ZERO_ONE | OP_LINE_ANCHOR | + OP_ESC_W_WORD | OP_ESC_B_WORD_BOUND | + OP_ESC_LTGT_WORD_BEGIN_END | OP_DECIMAL_BACKREF ), + + 0, + + 0, + + ( ALLOW_EMPTY_RANGE_IN_CC | NOT_NEWLINE_IN_NEGATIVE_CC ), + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax GnuRegex = new Syntax( + "GnuRegex", + GNU_REGEX_OP, + + 0, + + 0, + + GNU_REGEX_BV, + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Java = new Syntax( + "Java", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_CONTROL_CHARS | OP_ESC_C_CONTROL | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | OP2_QMARK_GROUP_EFFECT | + OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_REPEAT | + OP2_PLUS_POSSESSIVE_INTERVAL | OP2_CCLASS_SET_OP | + OP2_QMARK_LT_NAMED_GROUP | OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_V_VTAB | OP2_ESC_U_HEX4 | + OP2_ESC_P_BRACE_CHAR_PROPERTY ), + + 0, + + ( GNU_REGEX_BV | DIFFERENT_LEN_ALT_LOOK_BEHIND ), + + (Option.SINGLELINE | Option.WORD_BOUND_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax Perl = new Syntax( + "Perl", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT ), + + 0, + + GNU_REGEX_BV, + + Option.SINGLELINE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax PerlNG = new Syntax( + "PerlNG", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_X_BRACE_HEX8 | OP_ESC_CONTROL_CHARS | + OP_ESC_C_CONTROL ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_QMARK_LT_NAMED_GROUP | + OP2_ESC_K_NAMED_BACKREF | + OP2_ESC_G_SUBEXP_CALL ), + + 0, + + ( GNU_REGEX_BV | + CAPTURE_ONLY_NAMED_GROUP | + ALLOW_MULTIPLEX_DEFINITION_NAME ), + + Option.SINGLELINE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); + + public static final Syntax ECMAScript = new Syntax( + "ECMAScript", + (( GNU_REGEX_OP | OP_QMARK_NON_GREEDY | + OP_ESC_OCTAL3 | OP_ESC_X_HEX2 | + OP_ESC_CONTROL_CHARS | OP_ESC_C_CONTROL | + OP_DECIMAL_BACKREF | OP_ESC_D_DIGIT | + OP_ESC_S_WHITE_SPACE | OP_ESC_W_WORD ) + & ~OP_ESC_LTGT_WORD_BEGIN_END ), + + ( OP2_ESC_CAPITAL_Q_QUOTE | + OP2_QMARK_GROUP_EFFECT | OP2_OPTION_PERL | + OP2_ESC_P_BRACE_CHAR_PROPERTY | + OP2_ESC_P_BRACE_CIRCUMFLEX_NOT | + OP2_ESC_U_HEX4 | OP2_ESC_V_VTAB), + + OP3_OPTION_ECMASCRIPT, + + ( CONTEXT_INDEP_ANCHORS | + CONTEXT_INDEP_REPEAT_OPS | + CONTEXT_INVALID_REPEAT_OPS | + ALLOW_INVALID_INTERVAL | + BACKSLASH_ESCAPE_IN_CC | + ALLOW_DOUBLE_RANGE_OP_IN_CC | + DIFFERENT_LEN_ALT_LOOK_BEHIND ), + + Option.NONE, + + new MetaCharTable( + '\\', /* esc */ + INEFFECTIVE_META_CHAR, /* anychar '.' */ + INEFFECTIVE_META_CHAR, /* anytime '*' */ + INEFFECTIVE_META_CHAR, /* zero or one time '?' */ + INEFFECTIVE_META_CHAR, /* one or more time '+' */ + INEFFECTIVE_META_CHAR /* anychar anytime */ + ) + ); +} diff --git a/third_party/joni/src/org/joni/Token.java b/third_party/joni/src/org/joni/Token.java new file mode 100644 index 000000000..425c6f407 --- /dev/null +++ b/third_party/joni/src/org/joni/Token.java @@ -0,0 +1,180 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.constants.internal.TokenType; + +final class Token { + TokenType type; + boolean escaped; + int base; /* is number: 8, 16 (used in [....]) */ + int backP; + + // union fields + private int INT1, INT2, INT3, INT4, INT5; + private int []INTA1; + + // union accessors + int getC() { + return INT1; + } + void setC(int c) { + INT1 = c; + } + + int getCode() { + return INT1; + } + void setCode(int code) { + INT1 = code; + } + + int getAnchorSubtype() { + return INT1; + } + void setAnchorSubtype(int anchor) { + INT1 = anchor; + } + + boolean getAnchorASCIIRange() { + return INT2 == 1; + } + + void setAnchorASCIIRange(boolean ascii) { + INT2 = ascii ? 1 : 0; + } + + // repeat union member + int getRepeatLower() { + return INT1; + } + void setRepeatLower(int lower) { + INT1 = lower; + } + + int getRepeatUpper() { + return INT2; + } + void setRepeatUpper(int upper) { + INT2 = upper; + } + + boolean getRepeatGreedy() { + return INT3 != 0; + } + void setRepeatGreedy(boolean greedy) { + INT3 = greedy ? 1 : 0; + } + + boolean getRepeatPossessive() { + return INT4 != 0; + } + void setRepeatPossessive(boolean possessive) { + INT4 = possessive ? 1 : 0; + } + + // backref union member + int getBackrefNum() { + return INT1; + } + void setBackrefNum(int num) { + INT1 = num; + } + + int getBackrefRef1() { + return INT2; + } + void setBackrefRef1(int ref1) { + INT2 = ref1; + } + + int[]getBackrefRefs() { + return INTA1; + } + void setBackrefRefs(int[]refs) { + INTA1 = refs; + } + + boolean getBackrefByName() { + return INT3 != 0; + } + void setBackrefByName(boolean byName) { + INT3 = byName ? 1 : 0; + } + + // USE_BACKREF_AT_LEVEL + boolean getBackrefExistLevel() { + return INT4 != 0; + } + void setBackrefExistLevel(boolean existLevel) { + INT4 = existLevel ? 1 : 0; + } + + int getBackrefLevel() { + return INT5; + } + void setBackrefLevel(int level) { + INT5 = level; + } + + // call union member + int getCallNameP() { + return INT1; + } + void setCallNameP(int nameP) { + INT1 = nameP; + } + + int getCallNameEnd() { + return INT2; + } + void setCallNameEnd(int nameEnd) { + INT2 = nameEnd; + } + + int getCallGNum() { + return INT3; + } + void setCallGNum(int gnum) { + INT3 = gnum; + } + + boolean getCallRel() { + return INT4 != 0; + } + void setCallRel(boolean rel) { + INT4 = rel ? 1 : 0; + } + + // prop union member + int getPropCType() { + return INT1; + } + void setPropCType(int ctype) { + INT1 = ctype; + } + + boolean getPropNot() { + return INT2 != 0; + } + void setPropNot(boolean not) { + INT2 = not ? 1 : 0; + } +} diff --git a/third_party/joni/src/org/joni/UnsetAddrList.java b/third_party/joni/src/org/joni/UnsetAddrList.java new file mode 100644 index 000000000..c0b64f73e --- /dev/null +++ b/third_party/joni/src/org/joni/UnsetAddrList.java @@ -0,0 +1,68 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.ast.EncloseNode; +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; + +public final class UnsetAddrList { + EncloseNode[]targets; + int[]offsets; + int num; + + public UnsetAddrList(int size) { + targets = new EncloseNode[size]; + offsets = new int[size]; + } + + public void add(int offset, EncloseNode node) { + if (num >= offsets.length) { + EncloseNode []ttmp = new EncloseNode[targets.length << 1]; + System.arraycopy(targets, 0, ttmp, 0, num); + targets = ttmp; + int[]otmp = new int[offsets.length << 1]; + System.arraycopy(offsets, 0, otmp, 0, num); + offsets = otmp; + } + targets[num] = node; + offsets[num] = offset; + num++; + } + + public void fix(Regex regex) { + for (int i=0; i 0) { + for (int i = 0; i < num; i++) { + value.append("offset + ").append(offsets[i]).append(" target: ").append(targets[i].getAddressName()); + } + } + return value.toString(); + } +} diff --git a/third_party/joni/src/org/joni/WarnCallback.java b/third_party/joni/src/org/joni/WarnCallback.java new file mode 100644 index 000000000..d79d6a1bb --- /dev/null +++ b/third_party/joni/src/org/joni/WarnCallback.java @@ -0,0 +1,40 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +/** + * @author Ola Bini + */ +public interface WarnCallback { + WarnCallback DEFAULT = new WarnCallback() { + @Override + public void warn(String message) { + System.err.println(message); + } + }; + + WarnCallback NONE = new WarnCallback() { + @Override + public void warn(String message) { + } + }; + + void warn(String message); +} diff --git a/third_party/joni/src/org/joni/ast/AnchorNode.java b/third_party/joni/src/org/joni/ast/AnchorNode.java new file mode 100644 index 000000000..f69a965b2 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/AnchorNode.java @@ -0,0 +1,95 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.AnchorType; + +public final class AnchorNode extends Node { + public final int type; + public final boolean asciiRange; + public Node target; + public int charLength; + + public AnchorNode(int type, boolean asciiRange) { + super(ANCHOR); + this.type = type; + charLength = -1; + this.asciiRange = asciiRange; + } + + public AnchorNode(int type) { + this(type, false); + } + + @Override + protected void setChild(Node child) { + target = child; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(Node tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Anchor"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(); + value.append("\n type: " + typeToString()); + value.append(", ascii: " + asciiRange); + value.append("\n target: " + pad(target, level + 1)); + return value.toString(); + } + + public String typeToString() { + StringBuilder type = new StringBuilder(); + if (isType(AnchorType.BEGIN_BUF)) type.append("BEGIN_BUF "); + if (isType(AnchorType.BEGIN_LINE)) type.append("BEGIN_LINE "); + if (isType(AnchorType.BEGIN_POSITION)) type.append("BEGIN_POSITION "); + if (isType(AnchorType.END_BUF)) type.append("END_BUF "); + if (isType(AnchorType.SEMI_END_BUF)) type.append("SEMI_END_BUF "); + if (isType(AnchorType.END_LINE)) type.append("END_LINE "); + if (isType(AnchorType.WORD_BOUND)) type.append("WORD_BOUND "); + if (isType(AnchorType.NOT_WORD_BOUND)) type.append("NOT_WORD_BOUND "); + if (isType(AnchorType.WORD_BEGIN)) type.append("WORD_BEGIN "); + if (isType(AnchorType.WORD_END)) type.append("WORD_END "); + if (isType(AnchorType.PREC_READ)) type.append("PREC_READ "); + if (isType(AnchorType.PREC_READ_NOT)) type.append("PREC_READ_NOT "); + if (isType(AnchorType.LOOK_BEHIND)) type.append("LOOK_BEHIND "); + if (isType(AnchorType.LOOK_BEHIND_NOT)) type.append("LOOK_BEHIND_NOT "); + if (isType(AnchorType.ANYCHAR_STAR)) type.append("ANYCHAR_STAR "); + if (isType(AnchorType.ANYCHAR_STAR_ML)) type.append("ANYCHAR_STAR_ML "); + return type.toString(); + } + + private boolean isType(int type) { + return (this.type & type) != 0; + } + +} diff --git a/third_party/joni/src/org/joni/ast/AnyCharNode.java b/third_party/joni/src/org/joni/ast/AnyCharNode.java new file mode 100644 index 000000000..33f43c857 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/AnyCharNode.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +public final class AnyCharNode extends Node { + public AnyCharNode(){ + super(CANY); + } + + @Override + public String getName() { + return "Any Char"; + } + + @Override + public String toString(int level) { + String value = ""; + return value; + } +} diff --git a/third_party/joni/src/org/joni/ast/BackRefNode.java b/third_party/joni/src/org/joni/ast/BackRefNode.java new file mode 100644 index 000000000..fd50d021d --- /dev/null +++ b/third_party/joni/src/org/joni/ast/BackRefNode.java @@ -0,0 +1,86 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.Config; +import org.joni.ScanEnvironment; +import org.joni.exception.ErrorMessages; +import org.joni.exception.ValueException; + +public final class BackRefNode extends StateNode { + public final int[] back; + public int backNum; + public int nestLevel; + + private BackRefNode(int backNum, int[]backRefs, boolean byName, ScanEnvironment env) { + super(BREF); + this.backNum = backNum; + if (byName) setNameRef(); + + for (int i=0; i 0) { + back[pos] = n; + pos++; + } + } + backNum = pos; + } + + @Override + public String getName() { + return "Back Ref"; + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(super.toString(level)); + sb.append("\n backNum: " + backNum); + String backs = ""; + for (int i=0; i= sbOut) { + if (j > CR_FROM(mbr, i)) { + addCodeRangeToBuf(env, j, CR_TO(mbr, i)); + i++; + } + // !goto sb_end!, remove duplication! + for (; i= sbOut) { + // !goto sb_end2!, remove duplication + prev = sbOut; + for (i=0; i 1) { + ccAscii.addCodeRangeToBuf(env, 0x00, 0x7F); + } else { + ccAscii.bs.setRange(env, 0x00, 0x7F); + } + ccWork.and(ccAscii, env); + } + or(ccWork, env); + } else { + addCTypeByRange(ctype, not, env, sbOut.value, ranges); + } + return; + } + + int maxCode = asciiRange ? 0x80 : BitSet.SINGLE_BYTE_SIZE; + switch(ctype) { + case CharacterType.ALPHA: + case CharacterType.BLANK: + case CharacterType.CNTRL: + case CharacterType.DIGIT: + case CharacterType.LOWER: + case CharacterType.PUNCT: + case CharacterType.SPACE: + case CharacterType.UPPER: + case CharacterType.XDIGIT: + case CharacterType.ASCII: + case CharacterType.ALNUM: + if (not) { + for (int c=0; c= maxCode) bs.set(env, c); + } + if (asciiRange) addAllMultiByteRange(env); + } else { + for (int c=0; c 0 && /* check invalid code point */ + !(enc.isWord(c) || c >= maxCode)) bs.set(env, c); + } + if (asciiRange) addAllMultiByteRange(env); + } + break; + + default: + throw new InternalException(ErrorMessages.PARSER_BUG); + } // switch + } + + public enum CCVALTYPE { + SB, + CODE_POINT, + CLASS + } + + public enum CCSTATE { + VALUE, + RANGE, + COMPLETE, + START + } + + public static final class CCStateArg { + public int from; + public int to; + public boolean fromIsRaw; + public boolean toIsRaw; + public CCVALTYPE inType; + public CCVALTYPE type; + public CCSTATE state; + } + + public void nextStateClass(CCStateArg arg, CClassNode ascCC, ScanEnvironment env) { + if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + + if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { + if (arg.type == CCVALTYPE.SB) { + bs.set(env, arg.from); + if (ascCC != null) ascCC.bs.set(arg.from); + } else if (arg.type == CCVALTYPE.CODE_POINT) { + addCodeRange(env, arg.from, arg.from); + if (ascCC != null) ascCC.addCodeRange(env, arg.from, arg.from, false); + } + } + arg.state = CCSTATE.VALUE; + arg.type = CCVALTYPE.CLASS; + } + + public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env) { + switch(arg.state) { + case VALUE: + if (arg.type == CCVALTYPE.SB) { + bs.set(env, arg.from); + if (ascCc != null) ascCc.bs.set(arg.from); + } else if (arg.type == CCVALTYPE.CODE_POINT) { + addCodeRange(env, arg.from, arg.from); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + } + break; + + case RANGE: + if (arg.inType == arg.type) { + if (arg.inType == CCVALTYPE.SB) { + if (arg.from > 0xff || arg.to > 0xff) throw new ValueException(ErrorMessages.ERR_INVALID_CODE_POINT_VALUE); + + if (arg.from > arg.to) { + if (env.syntax.allowEmptyRangeInCC()) { + // goto ccs_range_end + arg.state = CCSTATE.COMPLETE; + break; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + bs.setRange(env, arg.from, arg.to); + if (ascCc != null) ascCc.bs.setRange(null, arg.from, arg.to); + } else { + addCodeRange(env, arg.from, arg.to); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.to, false); + } + } else { + if (arg.from > arg.to) { + if (env.syntax.allowEmptyRangeInCC()) { + // goto ccs_range_end + arg.state = CCSTATE.COMPLETE; + break; + } else { + throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + } + } + bs.setRange(env, arg.from, arg.to < 0xff ? arg.to : 0xff); + addCodeRange(env, arg.from, arg.to); + if (ascCc != null) { + ascCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); + ascCc.addCodeRange(env, arg.from, arg.to, false); + } + } + // ccs_range_end: + arg.state = CCSTATE.COMPLETE; + break; + + case COMPLETE: + case START: + arg.state = CCSTATE.VALUE; + break; + + default: + break; + + } // switch + + arg.fromIsRaw = arg.toIsRaw; + arg.from = arg.to; + arg.type = arg.inType; + } + + // onig_is_code_in_cc_len + boolean isCodeInCCLength(int encLength, int code) { + boolean found; + + if (encLength > 1 || code >= BitSet.SINGLE_BYTE_SIZE) { + if (mbuf == null) { + found = false; + } else { + found = CodeRange.isInCodeRange(mbuf.getCodeRange(), code); + } + } else { + found = bs.at(code); + } + + if (isNot()) { + return !found; + } else { + return found; + } + } + + // onig_is_code_in_cc + public boolean isCodeInCC(Encoding enc, int code) { + int len; + if (enc.minLength() > 1) { + len = 2; + } else { + len = enc.codeToMbcLength(code); + } + return isCodeInCCLength(len, code); + } + + public void setNot() { + flags |= FLAG_NCCLASS_NOT; + } + + public void clearNot() { + flags &= ~FLAG_NCCLASS_NOT; + } + + public boolean isNot() { + return (flags & FLAG_NCCLASS_NOT) != 0; + } +} diff --git a/third_party/joni/src/org/joni/ast/CTypeNode.java b/third_party/joni/src/org/joni/ast/CTypeNode.java new file mode 100644 index 000000000..fd14de4e6 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CTypeNode.java @@ -0,0 +1,48 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +public final class CTypeNode extends Node { + public final int ctype; + public final boolean not; + public final boolean asciiRange; + + public CTypeNode(int type, boolean not, boolean asciiRange) { + super(CTYPE); + this.ctype= type; + this.not = not; + this.asciiRange = asciiRange; + } + + @Override + public String getName() { + return "Character Type"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(); + value.append("\n ctype: " + ctype); + value.append(", not: " + not); + value.append(", ascii: " + asciiRange); + return value.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/ast/CallNode.java b/third_party/joni/src/org/joni/ast/CallNode.java new file mode 100644 index 000000000..1bc7f9e92 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CallNode.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.UnsetAddrList; + +public final class CallNode extends StateNode { + public final byte[]name; + public final int nameP; + public final int nameEnd; + + public int groupNum; + public EncloseNode target; + public UnsetAddrList unsetAddrList; + + public CallNode(byte[]name, int nameP, int nameEnd, int gnum) { + super(CALL); + this.name = name; + this.nameP = nameP; + this.nameEnd = nameEnd; + this.groupNum = gnum; /* call by number if gnum != 0 */ + } + + @Override + protected void setChild(Node newChild) { + target = (EncloseNode)newChild; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(EncloseNode tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Call"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(super.toString(level)); + value.append("\n name: " + new String(name, nameP, nameEnd - nameP)); + value.append(", groupNum: " + groupNum); + value.append("\n unsetAddrList: " + pad(unsetAddrList, level + 1)); + value.append("\n target: " + pad(target.getAddressName(), level + 1)); + return value.toString(); + } + +} diff --git a/third_party/joni/src/org/joni/ast/CalloutNode.java b/third_party/joni/src/org/joni/ast/CalloutNode.java new file mode 100644 index 000000000..fa6f2c365 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/CalloutNode.java @@ -0,0 +1,42 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +/** Zero-width internal node carrying an engine callout-table index. */ +public final class CalloutNode extends StringNode { + public final int calloutId; + + public CalloutNode(int calloutId) { + super(0); + this.calloutId = calloutId; + setRaw(); + setDontGetOptInfo(); + } + + @Override + public String getName() { + return "Callout"; + } + + @Override + public String toString(int level) { + return "\n id: " + calloutId; + } +} diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java new file mode 100644 index 000000000..4e791d164 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -0,0 +1,132 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.Config; +import org.joni.Option; +import org.joni.constants.internal.EncloseType; + +public final class EncloseNode extends StateNode implements EncloseType { + public final int type; // enclose type + public int regNum; + public int option; + public Node target; /* EncloseNode : ENCLOSE_MEMORY */ + public int callAddr; // AbsAddrType + public int minLength; // OnigDistance + public int maxLength; // OnigDistance + public int charLength; + public int optCount; // referenced count in optimize_node_left() + public int calloutConditionId = -1; + public Node containingAnchor; // + + // node_new_enclose / onig_node_new_enclose + public EncloseNode(int type) { + super(ENCLOSE); + this.type = type; + callAddr = -1; + } + + public static EncloseNode newMemory(int option, boolean isNamed) { + EncloseNode en = new EncloseNode(MEMORY); + if (Config.USE_SUBEXP_CALL) en.option = option; + if (isNamed) en.setNamedGroup(); + return en; + } + + public static EncloseNode newOption(int option) { + EncloseNode en = new EncloseNode(OPTION); + en.option = option; + return en; + } + + @Override + protected void setChild(Node child) { + target = child; + } + + @Override + protected Node getChild() { + return target; + } + + public void setTarget(Node tgt) { + target = tgt; + tgt.parent = this; + } + + @Override + public String getName() { + return "Enclose"; + } + + @Override + public String toString(int level) { + StringBuilder value = new StringBuilder(super.toString(level)); + value.append("\n type: " + typeToString()); + value.append("\n regNum: " + regNum); + value.append(", option: " + Option.toString(option)); + value.append(", callAddr: " + callAddr); + value.append(", minLength: " + minLength); + value.append(", maxLength: " + maxLength); + value.append(", charLength: " + charLength); + value.append(", optCount: " + optCount); + value.append(", calloutConditionId: " + calloutConditionId); + value.append("\n target: " + pad(target, level + 1)); + return value.toString(); + } + + public String typeToString() { + StringBuilder types = new StringBuilder(); + if (isStopBacktrack()) types.append("STOP_BACKTRACK "); + if (isMemory()) types.append("MEMORY "); + if (isOption()) types.append("OPTION "); + if (isCondition()) types.append("CONDITION "); + if (isAbsent()) types.append("ABSENT "); + return types.toString(); + } + + public void setEncloseStatus(int flag) { + state |= flag; + } + + public void clearEncloseStatus(int flag) { + state &= ~flag; + } + + public boolean isMemory() { + return (type & MEMORY) != 0; + } + + public boolean isOption() { + return (type & OPTION) != 0; + } + + public boolean isCondition() { + return (type & CONDITION) != 0; + } + + public boolean isStopBacktrack() { + return (type & STOP_BACKTRACK) != 0; + } + + public boolean isAbsent() { + return (type & ABSENT) != 0; + } +} diff --git a/third_party/joni/src/org/joni/ast/ListNode.java b/third_party/joni/src/org/joni/ast/ListNode.java new file mode 100644 index 000000000..7684f4c51 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/ListNode.java @@ -0,0 +1,96 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.exception.ErrorMessages; +import org.joni.exception.InternalException; + +public final class ListNode extends Node { + public Node value; + public ListNode tail; + + private ListNode(Node value, ListNode tail, int type) { + super(type); + this.value = value; + if (value != null) value.parent = this; + this.tail = tail; + if (tail != null) tail.parent = this; + } + + public static ListNode newAlt(Node value, ListNode tail) { + return new ListNode(value, tail, ALT); + } + + public static ListNode newList(Node value, ListNode tail) { + return new ListNode(value, tail, LIST); + } + + public static ListNode listAdd(ListNode list, Node value) { + ListNode n = newList(value, null); + if (list != null) { + while (list.tail != null) list = list.tail; + list.setTail(n); + } + return n; + } + + public void toListNode() { + type = LIST; + } + + @Override + protected void setChild(Node child) { + value = child; + } + + @Override + protected Node getChild() { + return value; + } + + public void setValue(Node value) { + this.value = value; + value.parent = this; + } + + public void setTail(ListNode tail) { + this.tail = tail; + } + + @Override + public String getName() { + switch (type) { + case ALT: + return "Alt"; + case LIST: + return "List"; + default: + throw new InternalException(ErrorMessages.PARSER_BUG); + } + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(); + sb.append("\n value: " + pad(value, level + 1)); + sb.append("\n tail: " + (tail == null ? "NULL" : tail.toString())); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/ast/Node.java b/third_party/joni/src/org/joni/ast/Node.java new file mode 100644 index 000000000..759f0917b --- /dev/null +++ b/third_party/joni/src/org/joni/ast/Node.java @@ -0,0 +1,120 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.NodeType; + +public abstract class Node implements NodeType { + public Node parent; + protected int type; + + Node(int type) { + this.type = type; + } + + public final int getType() { + return type; + } + + public final int getType2Bit() { + return 1 << getType(); + } + + protected void setChild(Node tgt){ + // default definition + } + + protected Node getChild(){ + // default definition + return null; + } + + public void replaceWith(Node with) { + with.parent = parent; + parent.setChild(with); + parent = null; + } + + public abstract String getName(); + protected abstract String toString(int level); + + public String getAddressName() { + return getName() + ":0x" + Integer.toHexString(System.identityHashCode(this)); + } + + @Override + public final String toString() { + StringBuilder s = new StringBuilder(); + s.append("<" + getAddressName() + " (" + (parent == null ? "NULL" : parent.getAddressName()) + ")>"); + return s + toString(0); + } + + protected static String pad(Object value, int level) { + if (value == null) return "NULL"; + + StringBuilder pad = new StringBuilder(" "); + for (int i=0; i= 0 && targetQNum >= 0 && env.syntax.warnReduntantNestedRepeat()) { + switch(REDUCE_TABLE[targetQNum][nestQNum]) { + case ASIS: + break; + case DEL: + env.warnings.warn("regular expression has redundant nested repeat operator " + PopularQStr[targetQNum] + " /" + new String(bytes, p, end - p) + "/"); + break; + default: + env.warnings.warn("nested repeat operator '" + PopularQStr[targetQNum] + "' and '" + PopularQStr[nestQNum] + + "' was replaced with '" + ReduceQStr[REDUCE_TABLE[targetQNum][nestQNum].ordinal()] + "' in regular expression " + "/" + new String(bytes, p, end - p) + "/"); + } + } + } // USE_WARNING_REDUNDANT_NESTED_REPEAT_OPERATOR + + if (targetQNum >= 0) { + if (nestQNum >= 0) { + reduceNestedQuantifier(qnt); + return 0; + } else if (targetQNum == 1 || targetQNum == 2) { /* * or + */ + /* (?:a*){n,m}, (?:a+){n,m} => (?:a*){n,n}, (?:a+){n,n} */ + if (!isRepeatInfinite(upper) && upper > 1 && greedy) { + upper = lower == 0 ? 1 : lower; + } + } + } + + default: + break; + } + + setTarget(tgt); + return 0; + } + + public static boolean isRepeatInfinite(int n) { + return n == REPEAT_INFINITE; + } +} diff --git a/third_party/joni/src/org/joni/ast/StateNode.java b/third_party/joni/src/org/joni/ast/StateNode.java new file mode 100644 index 000000000..4feb1c864 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/StateNode.java @@ -0,0 +1,235 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.joni.constants.internal.NodeStatus; + +abstract class StateNode extends Node implements NodeStatus { + protected int state; + + StateNode(int type) { + super(type); + } + + public boolean isMinFixed() { + return (state & NST_MIN_FIXED) != 0; + } + + public void setMinFixed() { + state |= NST_MIN_FIXED; + } + + public void clearMinFixed() { + state &= ~NST_MIN_FIXED; + } + + public boolean isMaxFixed() { + return (state & NST_MAX_FIXED) != 0; + } + + public void setMaxFixed() { + state |= NST_MAX_FIXED; + } + + public void clearMaxFixed() { + state &= ~NST_MAX_FIXED; + } + + public boolean isCLenFixed() { + return (state & NST_CLEN_FIXED) != 0; + } + + public void setCLenFixed() { + state |= NST_CLEN_FIXED; + } + + public void clearCLenFixed() { + state &= ~NST_CLEN_FIXED; + } + + public boolean isMark1() { + return (state & NST_MARK1) != 0; + } + + public void setMark1() { + state |= NST_MARK1; + } + + public void clearMark1() { + state &= ~NST_MARK1; + } + + public boolean isMark2() { + return (state & NST_MARK2) != 0; + } + + public void setMark2() { + state |= NST_MARK2; + } + + public void clearMark2() { + state &= ~NST_MARK2; + } + + public boolean isMemBackrefed() { + return (state & NST_MEM_BACKREFED) != 0; + } + + public void setMemBackrefed() { + state |= NST_MEM_BACKREFED; + } + + public void clearMemBackrefed() { + state &= ~NST_MEM_BACKREFED; + } + + public boolean isStopBtSimpleRepeat() { + return (state & NST_STOP_BT_SIMPLE_REPEAT) != 0; + } + + public void setStopBtSimpleRepeat() { + state |= NST_STOP_BT_SIMPLE_REPEAT; + } + + public void clearStopBtSimpleRepeat() { + state &= ~NST_STOP_BT_SIMPLE_REPEAT; + } + + public boolean isRecursion() { + return (state & NST_RECURSION) != 0; + } + + public void setRecursion() { + state |= NST_RECURSION; + } + + public void clearRecursion() { + state &= ~NST_RECURSION; + } + + public boolean isCalled() { + return (state & NST_CALLED) != 0; + } + + public void setCalled() { + state |= NST_CALLED; + } + + public void clearCAlled() { + state &= ~NST_CALLED; + } + + public boolean isAddrFixed() { + return (state & NST_ADDR_FIXED) != 0; + } + + public void setAddrFixed() { + state |= NST_ADDR_FIXED; + } + + public void clearAddrFixed() { + state &= ~NST_ADDR_FIXED; + } + + public boolean isNamedGroup() { + return (state & NST_NAMED_GROUP) != 0; + } + + public void setNamedGroup() { + state |= NST_NAMED_GROUP; + } + + public void clearNamedGroup() { + state &= ~NST_NAMED_GROUP; + } + + public boolean isNameRef() { + return (state & NST_NAME_REF) != 0; + } + + public void setNameRef() { + state |= NST_NAME_REF; + } + + public void clearNameRef() { + state &= ~NST_NAME_REF; + } + + public boolean isInRepeat() { + return (state & NST_IN_REPEAT) != 0; + } + + public void setInRepeat() { + state |= NST_IN_REPEAT; + } + + public void clearInRepeat() { + state &= ~NST_IN_REPEAT; + } + + public boolean isNestLevel() { + return (state & NST_NEST_LEVEL) != 0; + } + + public void setNestLevel() { + state |= NST_NEST_LEVEL; + } + + public void clearNestLevel() { + state &= ~NST_NEST_LEVEL; + } + + public boolean isByNumber() { + return (state & NST_BY_NUMBER) != 0; + } + + public void setByNumber() { + state |= NST_BY_NUMBER; + } + + public void clearByNumber() { + state &= ~NST_BY_NUMBER; + } + + @Override + public String toString(int level) { + return "\n state: " + stateToString(); + } + + public String stateToString() { + StringBuilder states = new StringBuilder(); + if (isMinFixed()) states.append("MIN_FIXED "); + if (isMaxFixed()) states.append("MAX_FIXED "); + if (isMark1()) states.append("MARK1 "); + if (isMark2()) states.append("MARK2 "); + if (isMemBackrefed()) states.append("MEM_BACKREFED "); + if (isStopBtSimpleRepeat()) states.append("STOP_BT_SIMPLE_REPEAT "); + if (isRecursion()) states.append("RECURSION "); + if (isCalled()) states.append("CALLED "); + if (isAddrFixed()) states.append("ADDR_FIXED "); + if (isNamedGroup()) states.append("NAMED_GROUP "); + if (isNameRef()) states.append("NAME_REF "); + if (isInRepeat()) states.append("IN_REPEAT "); + if (isNestLevel()) states.append("NEST_LEVEL "); + if (isByNumber()) states.append("BY_NUMBER "); + + return states.toString(); + } +} diff --git a/third_party/joni/src/org/joni/ast/StringNode.java b/third_party/joni/src/org/joni/ast/StringNode.java new file mode 100644 index 000000000..2e3e32d62 --- /dev/null +++ b/third_party/joni/src/org/joni/ast/StringNode.java @@ -0,0 +1,216 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.ast; + +import org.jcodings.Encoding; +import org.joni.Config; +import org.joni.constants.internal.StringType; + +public class StringNode extends Node implements StringType { + private static final int NODE_STR_MARGIN = 16; + private static final int NODE_STR_BUF_SIZE = 24; + public static final StringNode EMPTY = new StringNode(null, Integer.MAX_VALUE, Integer.MAX_VALUE); + + public byte[]bytes; + public int p; + public int end; + public int flag; + + public StringNode(int size) { + super(STR); + this.bytes = new byte[size]; + } + + public StringNode() { + this(NODE_STR_BUF_SIZE); + } + + public static StringNode fromCodePoint(int code, Encoding enc) { + StringNode str = new StringNode(Config.ENC_CODE_TO_MBC_MAXLEN); + str.end = enc.codeToMbc(code, str.bytes, str.p); + return str; + } + + public StringNode(byte[]bytes, int p, int end) { + super(STR); + this.bytes = bytes; + this.p = p; + this.end = end; + setShared(); + } + + /* Ensure there is ahead bytes available in node's buffer + * (assumes that the node is not shared) + */ + private void ensure(int ahead) { + int len = (end - p) + ahead; + if (len >= bytes.length) { + byte[]tmp = new byte[len + NODE_STR_MARGIN]; + System.arraycopy(bytes, p, tmp, 0, end - p); + bytes = tmp; + } + } + + /* COW and/or ensure there is ahead bytes available in node's buffer + */ + private void modifyEnsure(int ahead) { + if (isShared()) { + int len = (end - p) + ahead; + byte[]tmp = new byte[len + NODE_STR_MARGIN]; + System.arraycopy(bytes, p, tmp, 0, end - p); + bytes = tmp; + end = end - p; + p = 0; + clearShared(); + } else { + ensure(ahead); + } + } + + @Override + public String getName() { + return "String"; + } + + public int length() { + return end - p; + } + + public int length(Encoding enc) { + return enc.strLength(bytes, p, end); + } + + public StringNode splitLastChar(Encoding enc) { + StringNode n = null; + if (end > p) { + int prev = enc.prevCharHead(bytes, p, end, end); + if (prev != -1 && prev > p) { /* can be split */ + n = new StringNode(bytes, prev, end); + if (isRaw()) n.setRaw(); + end = prev; + } + } + return n; + } + + public boolean canBeSplit(Encoding enc) { + if (end > p) { + return enc.length(bytes, p, end) < (end - p); + } + return false; + } + + public void set(byte[]bytes, int p, int end) { + this.bytes = bytes; + this.p = p; + this.end = end; + setShared(); + } + + public void catBytes(byte[]cat, int catP, int catEnd) { + int len = catEnd - catP; + modifyEnsure(len); + System.arraycopy(cat, catP, bytes, end, len); + end += len; + } + + public void catByte(byte c) { + modifyEnsure(1); + bytes[end++] = c; + } + + public void catCode(int code, Encoding enc) { + modifyEnsure(Config.ENC_CODE_TO_MBC_MAXLEN); + end += enc.codeToMbc(code, bytes, end); + } + + public void setRaw() { + flag |= NSTR_RAW; + } + + public void clearRaw() { + flag &= ~NSTR_RAW; + } + + public boolean isRaw() { + return (flag & NSTR_RAW) != 0; + } + + public void setAmbig() { + flag |= NSTR_AMBIG; + } + + public void clearAmbig() { + flag &= ~NSTR_AMBIG; + } + + public boolean isAmbig() { + return (flag & NSTR_AMBIG) != 0; + } + + public void setDontGetOptInfo() { + flag |= NSTR_DONT_GET_OPT_INFO; + } + + public void clearDontGetOptInfo() { + flag &= ~NSTR_DONT_GET_OPT_INFO; + } + + public boolean isDontGetOptInfo() { + return (flag & NSTR_DONT_GET_OPT_INFO) != 0; + } + + public void setShared() { + flag |= NSTR_SHARED; + } + + public void clearShared() { + flag &= ~NSTR_SHARED; + } + + public boolean isShared() { + return (flag & NSTR_SHARED) != 0; + } + + public String flagsToString() { + StringBuilder flags = new StringBuilder(); + if (isRaw()) flags.append("RAW "); + if (isAmbig()) flags.append("AMBIG "); + if (isDontGetOptInfo()) flags.append("DONT_GET_OPT_INFO "); + if (isShared()) flags.append("SHARED "); + return flags.toString(); + } + + @Override + public String toString(int level) { + StringBuilder sb = new StringBuilder(); + sb.append("\n flags: " + flagsToString()); + sb.append("\n bytes: '"); + for (int i=p; i= 0x20 && (bytes[i] & 0xff) < 0x7f) { + sb.append((char)bytes[i]); + } else { + sb.append(String.format("[0x%02x]", bytes[i])); + } + } + sb.append("'"); + return sb.toString(); + } +} diff --git a/third_party/joni/src/org/joni/bench/AbstractBench.java b/third_party/joni/src/org/joni/bench/AbstractBench.java new file mode 100644 index 000000000..7abcb51b8 --- /dev/null +++ b/third_party/joni/src/org/joni/bench/AbstractBench.java @@ -0,0 +1,50 @@ +package org.joni.bench; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; + +public abstract class AbstractBench { + protected void bench(String _reg, String _str, int warmup, int times) throws Exception { + byte[] reg = _reg.getBytes(); + byte[] str = _str.getBytes(); + + Regex p = new Regex(reg,0,reg.length,Option.DEFAULT,ASCIIEncoding.INSTANCE,Syntax.DEFAULT); + + System.err.println("::: /" + _reg + "/ =~ \"" + _str + "\", " + warmup + " * " + times + " times"); + + for(int j=0;j */ + int OP_ESC_B_WORD_BOUND = (1<<20); /* \b, \B */ + int OP_ESC_S_WHITE_SPACE = (1<<21); /* \s, \S */ + int OP_ESC_D_DIGIT = (1<<22); /* \d, \D */ + int OP_LINE_ANCHOR = (1<<23); /* ^, $ */ + int OP_POSIX_BRACKET = (1<<24); /* [:xxxx:] */ + int OP_QMARK_NON_GREEDY = (1<<25); /* ??,*?,+?,{n,m}? */ + int OP_ESC_CONTROL_CHARS = (1<<26); /* \n,\r,\t,\a ... */ + int OP_ESC_C_CONTROL = (1<<27); /* \cx */ + int OP_ESC_OCTAL3 = (1<<28); /* \OOO */ + int OP_ESC_X_HEX2 = (1<<29); /* \xHH */ + int OP_ESC_X_BRACE_HEX8 = (1<<30); /* \x{7HHHHHHH} */ + int OP_ESC_O_BRACE_OCTAL = (1<<31); /* \o{OOO} */ + + int OP2_ESC_CAPITAL_Q_QUOTE = (1<<0); /* \Q...\E */ + int OP2_QMARK_GROUP_EFFECT = (1<<1); /* (?...); */ + int OP2_OPTION_PERL = (1<<2); /* (?imsxadlu), (?-imsx), (?^imsxalu) */ + int OP2_OPTION_RUBY = (1<<3); /* (?imxadu);, (?-imx); */ + int OP2_PLUS_POSSESSIVE_REPEAT = (1<<4); /* ?+,*+,++ */ + int OP2_PLUS_POSSESSIVE_INTERVAL = (1<<5); /* {n,m}+ */ + int OP2_CCLASS_SET_OP = (1<<6); /* [...&&..[..]..] */ + int OP2_QMARK_LT_NAMED_GROUP = (1<<7); /* (?...); */ + int OP2_ESC_K_NAMED_BACKREF = (1<<8); /* \k */ + int OP2_ESC_G_SUBEXP_CALL = (1<<9); /* \g, \g */ + int OP2_ATMARK_CAPTURE_HISTORY = (1<<10); /* (?@..);,(?@..); */ + int OP2_ESC_CAPITAL_C_BAR_CONTROL = (1<<11); /* \C-x */ + int OP2_ESC_CAPITAL_M_BAR_META = (1<<12); /* \M-x */ + int OP2_ESC_V_VTAB = (1<<13); /* \v as VTAB */ + int OP2_ESC_U_HEX4 = (1<<14); /* \\uHHHH */ + int OP2_ESC_GNU_BUF_ANCHOR = (1<<15); /* \`, \' */ + int OP2_ESC_P_BRACE_CHAR_PROPERTY = (1<<16); /* \p{...}, \P{...} */ + int OP2_ESC_P_BRACE_CIRCUMFLEX_NOT = (1<<17); /* \p{^..}, \P{^..} */ + /* final int OP2_CHAR_PROPERTY_PREFIX_IS = (1<<18); */ + int OP2_ESC_H_XDIGIT = (1<<19); /* \h, \H */ + int OP2_INEFFECTIVE_ESCAPE = (1<<20); /* \ */ + int OP2_ESC_CAPITAL_R_LINEBREAK = (1<<21); /* \R as (?>\x0D\x0A|[\x0A-\x0D\x{85}\x{2028}\x{2029}]) */ + int OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER = (1<<22); /* \X as (?:\P{M}\p{M}*) */ + int OP2_ESC_V_VERTICAL_WHITESPACE = (1<<23); /* \v, \V -- Perl */ + int OP2_ESC_H_HORIZONTAL_WHITESPACE= (1<<24); /* \h, \H -- Perl */ + int OP2_ESC_CAPITAL_K_KEEP = (1<<25); /* \K */ + int OP2_ESC_G_BRACE_BACKREF = (1<<26); /* \g{name}, \g{n} */ + int OP2_QMARK_SUBEXP_CALL = (1<<27); /* (?&name), (?n), (?R), (?0) */ + int OP2_QMARK_BAR_BRANCH_RESET = (1<<28); /* (?|...) */ + int OP2_QMARK_LPAREN_CONDITION = (1<<29); /* (?(cond)yes...|no...) */ + int OP2_QMARK_CAPITAL_P_NAMED_GROUP= (1<<30); /* (?P...), (?P=name), (?P>name) -- Python/PCRE */ + int OP2_QMARK_TILDE_ABSENT = (1<<31); /* (?~...) */ + + int OP3_OPTION_JAVA = (1<<0); /* (?idmsux), (?-idmsux) */ + int OP3_OPTION_ECMASCRIPT = (1<<1); /* EcmaScript quirks */ + + /* syntax (behavior); */ + int CONTEXT_INDEP_ANCHORS = (1<<31); /* not implemented */ + int CONTEXT_INDEP_REPEAT_OPS = (1<<0); /* ?, *, +, {n,m} */ + int CONTEXT_INVALID_REPEAT_OPS = (1<<1); /* error or ignore */ + int ALLOW_UNMATCHED_CLOSE_SUBEXP = (1<<2); /* ...);... */ + int ALLOW_INVALID_INTERVAL = (1<<3); /* {??? */ + int ALLOW_INTERVAL_LOW_ABBREV = (1<<4); /* {,n} => {0,n} */ + int STRICT_CHECK_BACKREF = (1<<5); /* /(\1);/,/\1();/ ..*/ + int DIFFERENT_LEN_ALT_LOOK_BEHIND = (1<<6); /* (?<=a|bc); */ + int CAPTURE_ONLY_NAMED_GROUP = (1<<7); /* see doc/RE */ + int ALLOW_MULTIPLEX_DEFINITION_NAME = (1<<8); /* (?);(?); */ + int FIXED_INTERVAL_IS_GREEDY_ONLY = (1<<9); /* a{n}?=(?:a{n});? */ + int ALLOW_MULTIPLEX_DEFINITION_NAME_CALL = (1<<10); /* (?)(?)(?&x) */ + + /* syntax (behavior); in char class [...] */ + int NOT_NEWLINE_IN_NEGATIVE_CC = (1<<20); /* [^...] */ + int BACKSLASH_ESCAPE_IN_CC = (1<<21); /* [..\w..] etc.. */ + int ALLOW_EMPTY_RANGE_IN_CC = (1<<22); + int ALLOW_DOUBLE_RANGE_OP_IN_CC = (1<<23); /* [0-9-a]=[0-9\-a] */ + /* syntax (behavior); warning */ + int WARN_CC_OP_NOT_ESCAPED = (1<<24); /* [,-,] */ + int WARN_REDUNDANT_NESTED_REPEAT = (1<<25); /* (?:a*);+ */ + int WARN_CC_DUP = (1<<26); /* [aa] */ + + int POSIX_COMMON_OP = + OP_DOT_ANYCHAR | OP_POSIX_BRACKET | + OP_DECIMAL_BACKREF | + OP_BRACKET_CC | OP_ASTERISK_ZERO_INF | + OP_LINE_ANCHOR | + OP_ESC_CONTROL_CHARS; + + int GNU_REGEX_OP = + OP_DOT_ANYCHAR | OP_BRACKET_CC | + OP_POSIX_BRACKET | OP_DECIMAL_BACKREF | + OP_BRACE_INTERVAL | OP_LPAREN_SUBEXP | + OP_VBAR_ALT | + OP_ASTERISK_ZERO_INF | OP_PLUS_ONE_INF | + OP_QMARK_ZERO_ONE | + OP_ESC_AZ_BUF_ANCHOR | OP_ESC_CAPITAL_G_BEGIN_ANCHOR | + OP_ESC_W_WORD | + OP_ESC_B_WORD_BOUND | OP_ESC_LTGT_WORD_BEGIN_END | + OP_ESC_S_WHITE_SPACE | OP_ESC_D_DIGIT | + OP_LINE_ANCHOR; + + int GNU_REGEX_BV = + CONTEXT_INDEP_ANCHORS | CONTEXT_INDEP_REPEAT_OPS | + CONTEXT_INVALID_REPEAT_OPS | ALLOW_INVALID_INTERVAL | + BACKSLASH_ESCAPE_IN_CC | ALLOW_DOUBLE_RANGE_OP_IN_CC; +} diff --git a/third_party/joni/src/org/joni/constants/internal/AnchorType.java b/third_party/joni/src/org/joni/constants/internal/AnchorType.java new file mode 100644 index 000000000..6851c995f --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/AnchorType.java @@ -0,0 +1,71 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface AnchorType { + int BEGIN_BUF = (1<<0); + int BEGIN_LINE = (1<<1); + int BEGIN_POSITION = (1<<2); + int END_BUF = (1<<3); + int SEMI_END_BUF = (1<<4); + int END_LINE = (1<<5); + + int WORD_BOUND = (1<<6); + int NOT_WORD_BOUND = (1<<7); + int WORD_BEGIN = (1<<8); + int WORD_END = (1<<9); + int PREC_READ = (1<<10); + int PREC_READ_NOT = (1<<11); + int LOOK_BEHIND = (1<<12); + int LOOK_BEHIND_NOT = (1<<13); + + int ANYCHAR_STAR = (1<<14); /* ".*" optimize info */ + int ANYCHAR_STAR_ML = (1<<15); /* ".*" optimize info (multi-line) */ + + int ANYCHAR_STAR_MASK = (ANYCHAR_STAR | ANYCHAR_STAR_ML); + int END_BUF_MASK = (END_BUF | SEMI_END_BUF); + + int KEEP = (1<<16); + + int ALLOWED_IN_LB = ( LOOK_BEHIND | + LOOK_BEHIND_NOT | + BEGIN_LINE | + END_LINE | + BEGIN_BUF | + BEGIN_POSITION | + KEEP | + WORD_BOUND | + NOT_WORD_BOUND | + WORD_BEGIN | + WORD_END ); + + + int ALLOWED_IN_LB_NOT = ( LOOK_BEHIND | + LOOK_BEHIND_NOT | + BEGIN_LINE | + END_LINE | + BEGIN_BUF | + BEGIN_POSITION | + KEEP | + WORD_BOUND | + NOT_WORD_BOUND | + WORD_BEGIN | + WORD_END ); +} diff --git a/third_party/joni/src/org/joni/constants/internal/Arguments.java b/third_party/joni/src/org/joni/constants/internal/Arguments.java new file mode 100644 index 000000000..02a1896d0 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/Arguments.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface Arguments { + int SPECIAL = -1; + int NON = 0; + int RELADDR = 1; + int ABSADDR = 2; + int LENGTH = 3; + int MEMNUM = 4; + int OPTION = 5; + int STATE_CHECK = 6; +} diff --git a/third_party/joni/src/org/joni/constants/internal/EncloseType.java b/third_party/joni/src/org/joni/constants/internal/EncloseType.java new file mode 100644 index 000000000..67e0b82df --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/EncloseType.java @@ -0,0 +1,31 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface EncloseType { + int MEMORY = 1<<0; + int OPTION = 1<<1; + int STOP_BACKTRACK = 1<<2; + int CONDITION = 1<<3; + int ABSENT = 1<<4; + + int ALLOWED_IN_LB = MEMORY | OPTION; + int ALLOWED_IN_LB_NOT = OPTION; +} diff --git a/third_party/joni/src/org/joni/constants/internal/NodeStatus.java b/third_party/joni/src/org/joni/constants/internal/NodeStatus.java new file mode 100644 index 000000000..335acd72e --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/NodeStatus.java @@ -0,0 +1,39 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface NodeStatus { + /* status bits */ + int NST_MIN_FIXED = (1<<0); + int NST_MAX_FIXED = (1<<1); + int NST_CLEN_FIXED = (1<<2); + int NST_MARK1 = (1<<3); + int NST_MARK2 = (1<<4); + int NST_MEM_BACKREFED = (1<<5); + int NST_STOP_BT_SIMPLE_REPEAT= (1<<6); + int NST_RECURSION = (1<<7); + int NST_CALLED = (1<<8); + int NST_ADDR_FIXED = (1<<9); + int NST_NAMED_GROUP = (1<<10); + int NST_NAME_REF = (1<<11); + int NST_IN_REPEAT = (1<<12); /* STK_REPEAT is nested in stack. */ + int NST_NEST_LEVEL = (1<<13); + int NST_BY_NUMBER = (1<<14); /* {n,m} */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/NodeType.java b/third_party/joni/src/org/joni/constants/internal/NodeType.java new file mode 100644 index 000000000..dd1acda70 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/NodeType.java @@ -0,0 +1,66 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface NodeType { + /* node type */ + int STR = 0; + int CCLASS = 1; + int CTYPE = 2; + int CANY = 3; + int BREF = 4; + int QTFR = 5; + int ENCLOSE = 6; + int ANCHOR = 7; + int LIST = 8; + int ALT = 9; + int CALL = 10; + + int BIT_STR = 1 << STR; + int BIT_CCLASS = 1 << CCLASS; + int BIT_CTYPE = 1 << CTYPE; + int BIT_CANY = 1 << CANY; + int BIT_BREF = 1 << BREF; + int BIT_QTFR = 1 << QTFR; + int BIT_ENCLOSE = 1 << ENCLOSE; + int BIT_ANCHOR = 1 << ANCHOR; + int BIT_LIST = 1 << LIST; + int BIT_ALT = 1 << ALT; + int BIT_CALL = 1 << CALL; + + /* allowed node types in look-behind */ + int ALLOWED_IN_LB = ( BIT_LIST | + BIT_ALT | + BIT_STR | + BIT_CCLASS | + BIT_CTYPE | + BIT_CANY | + BIT_ANCHOR | + BIT_ENCLOSE | + BIT_QTFR | + BIT_CALL ); + + int SIMPLE = ( BIT_STR | + BIT_CCLASS | + BIT_CTYPE | + BIT_CANY | + BIT_BREF); + +} diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java new file mode 100644 index 000000000..9a382b6a9 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -0,0 +1,358 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +import org.joni.Config; + +public interface OPCode { + int FINISH = 0; /* matching process terminator (no more alternative) */ + int END = 1; /* pattern code terminator (success end) */ + + int EXACT1 = 2; /* single byte, N = 1 */ + int EXACT2 = 3; /* single byte, N = 2 */ + int EXACT3 = 4; /* single byte, N = 3 */ + int EXACT4 = 5; /* single byte, N = 4 */ + int EXACT5 = 6; /* single byte, N = 5 */ + int EXACTN = 7; /* single byte */ + int EXACTMB2N1 = 8; /* mb-length = 2 N = 1 */ + int EXACTMB2N2 = 9; /* mb-length = 2 N = 2 */ + int EXACTMB2N3 = 10; /* mb-length = 2 N = 3 */ + int EXACTMB2N = 11; /* mb-length = 2 */ + int EXACTMB3N = 12; /* mb-length = 3 */ + int EXACTMBN = 13; /* other length */ + + int EXACT1_IC = 14; /* single byte, N = 1, ignore case */ + int EXACTN_IC = 15; /* single byte, ignore case */ + + int CCLASS = 16; + int CCLASS_MB = 17; + int CCLASS_MIX = 18; + int CCLASS_NOT = 19; + int CCLASS_MB_NOT = 20; + int CCLASS_MIX_NOT = 21; + + int ANYCHAR = 22; /* "." */ + int ANYCHAR_ML = 23; /* "." multi-line */ + int ANYCHAR_STAR = 24; /* ".*" */ + int ANYCHAR_ML_STAR = 25; /* ".*" multi-line */ + int ANYCHAR_STAR_PEEK_NEXT = 26; + int ANYCHAR_ML_STAR_PEEK_NEXT = 27; + + int WORD = 28; + int NOT_WORD = 29; + int WORD_BOUND = 30; + int NOT_WORD_BOUND = 31; + int WORD_BEGIN = 32; + int WORD_END = 33; + + int ASCII_WORD = 34; + int ASCII_NOT_WORD = 35; + int ASCII_WORD_BOUND = 36; + int ASCII_NOT_WORD_BOUND = 37; + int ASCII_WORD_BEGIN = 38; + int ASCII_WORD_END = 39; + + int BEGIN_BUF = 40; + int END_BUF = 41; + int BEGIN_LINE = 42; + int END_LINE = 43; + int SEMI_END_BUF = 44; + int BEGIN_POSITION = 45; + + int BACKREF1 = 46; + int BACKREF2 = 47; + int BACKREFN = 48; + int BACKREFN_IC = 49; + int BACKREF_MULTI = 50; + int BACKREF_MULTI_IC = 51; + int BACKREF_WITH_LEVEL = 52; /* \k, \k */ + + int MEMORY_START = 53; + int MEMORY_START_PUSH = 54; /* push back-tracker to stack */ + int MEMORY_END_PUSH = 55; /* push back-tracker to stack */ + int MEMORY_END_PUSH_REC = 56; /* push back-tracker to stack */ + int MEMORY_END = 57; + int MEMORY_END_REC = 58; /* push marker to stack */ + + int KEEP = 59; + int FAIL = 60; /* pop stack and move */ + int JUMP = 61; + int PUSH = 62; + int POP = 63; + int PUSH_OR_JUMP_EXACT1 = 64; /* if match exact then push, else jump. */ + int PUSH_IF_PEEK_NEXT = 65; /* if match exact then push, else none. */ + + int REPEAT = 66; /* {n,m} */ + int REPEAT_NG = 67; /* {n,m}? (non greedy) */ + int REPEAT_INC = 68; + int REPEAT_INC_NG = 69; /* non greedy */ + int REPEAT_INC_SG = 70; /* search and get in stack */ + int REPEAT_INC_NG_SG = 71; /* search and get in stack (non greedy) */ + + int NULL_CHECK_START = 72; /* null loop checker start */ + int NULL_CHECK_END = 73; /* null loop checker end */ + int NULL_CHECK_END_MEMST = 74; /* null loop checker end (with capture status) */ + int NULL_CHECK_END_MEMST_PUSH = 75; /* with capture status and push check-end */ + + int PUSH_POS = 76; /* (?=...) start */ + int POP_POS = 77; /* (?=...) end */ + int PUSH_POS_NOT = 78; /* (?!...) start */ + int FAIL_POS = 79; /* (?!...) end */ + int PUSH_STOP_BT = 80; /* (?>...) start */ + int POP_STOP_BT = 81; /* (?>...) end */ + int LOOK_BEHIND = 82; /* (?<=...) start (no needs end opcode) */ + int PUSH_LOOK_BEHIND_NOT = 83; /* (? */ + int RETURN = 89; + int CONDITION = 90; + + int STATE_CHECK_PUSH = 91; /* combination explosion check and push */ + int STATE_CHECK_PUSH_OR_JUMP = 92; /* check ok -> push, else jump */ + int STATE_CHECK = 93; /* check only */ + int STATE_CHECK_ANYCHAR_STAR = 94; + int STATE_CHECK_ANYCHAR_ML_STAR = 95; + + /* no need: IS_DYNAMIC_OPTION() == 0 */ + int SET_OPTION_PUSH = 96; /* set option and push recover option */ + int SET_OPTION = 97; /* set option */ + + int EXACT1_IC_SB = 98; /* single byte, N = 1, ignore case */ + int EXACTN_IC_SB = 99; /* single byte, ignore case */ + int CALLOUT = 100; /* generic match-time callout */ + int CALLOUT_CONDITION = 101; /* callback-selected conditional */ + + String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { + "finish", /*OP_FINISH*/ + "end", /*OP_END*/ + "exact1", /*OP_EXACT1*/ + "exact2", /*OP_EXACT2*/ + "exact3", /*OP_EXACT3*/ + "exact4", /*OP_EXACT4*/ + "exact5", /*OP_EXACT5*/ + "exactn", /*OP_EXACTN*/ + "exactmb2-n1", /*OP_EXACTMB2N1*/ + "exactmb2-n2", /*OP_EXACTMB2N2*/ + "exactmb2-n3", /*OP_EXACTMB2N3*/ + "exactmb2-n", /*OP_EXACTMB2N*/ + "exactmb3n", /*OP_EXACTMB3N*/ + "exactmbn", /*OP_EXACTMBN*/ + "exact1-ic", /*OP_EXACT1_IC*/ + "exactn-ic", /*OP_EXACTN_IC*/ + "cclass", /*OP_CCLASS*/ + "cclass-mb", /*OP_CCLASS_MB*/ + "cclass-mix", /*OP_CCLASS_MIX*/ + "cclass-not", /*OP_CCLASS_NOT*/ + "cclass-mb-not", /*OP_CCLASS_MB_NOT*/ + "cclass-mix-not", /*OP_CCLASS_MIX_NOT*/ + "anychar", /*OP_ANYCHAR*/ + "anychar-ml", /*OP_ANYCHAR_ML*/ + "anychar*", /*OP_ANYCHAR_STAR*/ + "anychar-ml*", /*OP_ANYCHAR_ML_STAR*/ + "anychar*-peek-next", /*OP_ANYCHAR_STAR_PEEK_NEXT*/ + "anychar-ml*-peek-next", /*OP_ANYCHAR_ML_STAR_PEEK_NEXT*/ + "word", /*OP_WORD*/ + "not-word", /*OP_NOT_WORD*/ + "word-bound", /*OP_WORD_BOUND*/ + "not-word-bound", /*OP_NOT_WORD_BOUND*/ + "word-begin", /*OP_WORD_BEGIN*/ + "word-end", /*OP_WORD_END*/ + "ascii-word", /*OP_ASCII_WORD*/ + "not-ascii-word", /*OP_NOT_ASCII_WORD*/ + "ascii-word-bound", /*OP_ASCII_WORD_BOUND*/ + "not-ascii-word-bound", /*OP_NOT_ASCII_WORD_BOUND*/ + "ascii-word-begin", /*OP_ASCII_WORD_BEGIN*/ + "ascii-word-end", /*OP_ASCII_WORD_END*/ + "begin-buf", /*OP_BEGIN_BUF*/ + "end-buf", /*OP_END_BUF*/ + "begin-line", /*OP_BEGIN_LINE*/ + "end-line", /*OP_END_LINE*/ + "semi-end-buf", /*OP_SEMI_END_BUF*/ + "begin-position", /*OP_BEGIN_POSITION*/ + "backref1", /*OP_BACKREF1*/ + "backref2", /*OP_BACKREF2*/ + "backrefn", /*OP_BACKREFN*/ + "backrefn-ic", /*OP_BACKREFN_IC*/ + "backref_multi", /*OP_BACKREF_MULTI*/ + "backref_multi-ic", /*OP_BACKREF_MULTI_IC*/ + "backref_at_level", /*OP_BACKREF_AT_LEVEL*/ + "mem-start", /*OP_MEMORY_START*/ + "mem-start-push", /*OP_MEMORY_START_PUSH*/ + "mem-end-push", /*OP_MEMORY_END_PUSH*/ + "mem-end-push-rec", /*OP_MEMORY_END_PUSH_REC*/ + "mem-end", /*OP_MEMORY_END*/ + "mem-end-rec", /*OP_MEMORY_END_REC*/ + "keep", /*OP_KEEP*/ + "fail", /*OP_FAIL*/ + "jump", /*OP_JUMP*/ + "push", /*OP_PUSH*/ + "pop", /*OP_POP*/ + "push-or-jump-e1", /*OP_PUSH_OR_JUMP_EXACT1*/ + "push-if-peek-next", /*OP_PUSH_IF_PEEK_NEXT*/ + "repeat", /*OP_REPEAT*/ + "repeat-ng", /*OP_REPEAT_NG*/ + "repeat-inc", /*OP_REPEAT_INC*/ + "repeat-inc-ng", /*OP_REPEAT_INC_NG*/ + "repeat-inc-sg", /*OP_REPEAT_INC_SG*/ + "repeat-inc-ng-sg", /*OP_REPEAT_INC_NG_SG*/ + "null-check-start", /*OP_NULL_CHECK_START*/ + "null-check-end", /*OP_NULL_CHECK_END*/ + "null-check-end-memst", /*OP_NULL_CHECK_END_MEMST*/ + "null-check-end-memst-push", /*OP_NULL_CHECK_END_MEMST_PUSH*/ + "push-pos", /*OP_PUSH_POS*/ + "pop-pos", /*OP_POP_POS*/ + "push-pos-not", /*OP_PUSH_POS_NOT*/ + "fail-pos", /*OP_FAIL_POS*/ + "push-stop-bt", /*OP_PUSH_STOP_BT*/ + "pop-stop-bt", /*OP_POP_STOP_BT*/ + "look-behind", /*OP_LOOK_BEHIND*/ + "push-look-behind-not", /*OP_PUSH_LOOK_BEHIND_NOT*/ + "fail-look-behind-not", /*OP_FAIL_LOOK_BEHIND_NOT*/ + "push-absent-pos", /*OP_PUSH_ABSENT_POS*/ + "absent", /*OP_ABSENT*/ + "absent-end", /*OP_ABSENT_END*/ + "call", /*OP_CALL*/ + "return", /*OP_RETURN*/ + "condition", /*OP_CONDITION*/ + "state-check-push", /*OP_STATE_CHECK_PUSH*/ + "state-check-push-or-jump", /*OP_STATE_CHECK_PUSH_OR_JUMP*/ + "state-check", /*OP_STATE_CHECK*/ + "state-check-anychar*", /*OP_STATE_CHECK_ANYCHAR_STAR*/ + "state-check-anychar-ml*", /*OP_STATE_CHECK_ANYCHAR_ML_STAR*/ + "set-option-push", /*OP_SET_OPTION_PUSH*/ + "set-option", /*OP_SET_OPTION*/ + + "exact1-ic-sb", /*OP_EXACT1_IC*/ + "exactn-ic-sb", /*OP_EXACTN_IC*/ + "callout", /*OP_CALLOUT*/ + "callout-condition", /*OP_CALLOUT_CONDITION*/ + } : null; + + int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { + Arguments.NON, /*OP_FINISH*/ + Arguments.NON, /*OP_END*/ + Arguments.SPECIAL, /*OP_EXACT1*/ + Arguments.SPECIAL, /*OP_EXACT2*/ + Arguments.SPECIAL, /*OP_EXACT3*/ + Arguments.SPECIAL, /*OP_EXACT4*/ + Arguments.SPECIAL, /*OP_EXACT5*/ + Arguments.SPECIAL, /*OP_EXACTN*/ + Arguments.SPECIAL, /*OP_EXACTMB2N1*/ + Arguments.SPECIAL, /*OP_EXACTMB2N2*/ + Arguments.SPECIAL, /*OP_EXACTMB2N3*/ + Arguments.SPECIAL, /*OP_EXACTMB2N*/ + Arguments.SPECIAL, /*OP_EXACTMB3N*/ + Arguments.SPECIAL, /*OP_EXACTMBN*/ + Arguments.SPECIAL, /*OP_EXACT1_IC*/ + Arguments.SPECIAL, /*OP_EXACTN_IC*/ + Arguments.SPECIAL, /*OP_CCLASS*/ + Arguments.SPECIAL, /*OP_CCLASS_MB*/ + Arguments.SPECIAL, /*OP_CCLASS_MIX*/ + Arguments.SPECIAL, /*OP_CCLASS_NOT*/ + Arguments.SPECIAL, /*OP_CCLASS_MB_NOT*/ + Arguments.SPECIAL, /*OP_CCLASS_MIX_NOT*/ + Arguments.NON, /*OP_ANYCHAR*/ + Arguments.NON, /*OP_ANYCHAR_ML*/ + Arguments.NON, /*OP_ANYCHAR_STAR*/ + Arguments.NON, /*OP_ANYCHAR_ML_STAR*/ + Arguments.SPECIAL, /*OP_ANYCHAR_STAR_PEEK_NEXT*/ + Arguments.SPECIAL, /*OP_ANYCHAR_ML_STAR_PEEK_NEXT*/ + Arguments.NON, /*OP_WORD*/ + Arguments.NON, /*OP_NOT_WORD*/ + Arguments.NON, /*OP_WORD_BOUND*/ + Arguments.NON, /*OP_NOT_WORD_BOUND*/ + Arguments.NON, /*OP_WORD_BEGIN*/ + Arguments.NON, /*OP_WORD_END*/ + Arguments.NON, /*OP_ASCII_WORD*/ + Arguments.NON, /*OP_NOT_ASCII_WORD*/ + Arguments.NON, /*OP_ASCII_WORD_BOUND*/ + Arguments.NON, /*OP_NOT_ASCII_WORD_BOUND*/ + Arguments.NON, /*OP_ASCII_WORD_BEGIN*/ + Arguments.NON, /*OP_ASCII_WORD_END*/ + Arguments.NON, /*OP_BEGIN_BUF*/ + Arguments.NON, /*OP_END_BUF*/ + Arguments.NON, /*OP_BEGIN_LINE*/ + Arguments.NON, /*OP_END_LINE*/ + Arguments.NON, /*OP_SEMI_END_BUF*/ + Arguments.NON, /*OP_BEGIN_POSITION*/ + Arguments.NON, /*OP_BACKREF1*/ + Arguments.NON, /*OP_BACKREF2*/ + Arguments.MEMNUM, /*OP_BACKREFN*/ + Arguments.SPECIAL, /*OP_BACKREFN_IC*/ + Arguments.SPECIAL, /*OP_BACKREF_MULTI*/ + Arguments.SPECIAL, /*OP_BACKREF_MULTI_IC*/ + Arguments.SPECIAL, /*OP_BACKREF_AT_LEVEL*/ + Arguments.MEMNUM, /*OP_MEMORY_START*/ + Arguments.MEMNUM, /*OP_MEMORY_START_PUSH*/ + Arguments.MEMNUM, /*OP_MEMORY_END_PUSH*/ + Arguments.MEMNUM, /*OP_MEMORY_END_PUSH_REC*/ + Arguments.MEMNUM, /*OP_MEMORY_END*/ + Arguments.MEMNUM, /*OP_MEMORY_END_REC*/ + Arguments.NON, /*OP_KEEP*/ + Arguments.NON, /*OP_FAIL*/ + Arguments.RELADDR, /*OP_JUMP*/ + Arguments.RELADDR, /*OP_PUSH*/ + Arguments.NON, /*OP_POP*/ + Arguments.SPECIAL, /*OP_PUSH_OR_JUMP_EXACT1*/ + Arguments.SPECIAL, /*OP_PUSH_IF_PEEK_NEXT*/ + Arguments.SPECIAL, /*OP_REPEAT*/ + Arguments.SPECIAL, /*OP_REPEAT_NG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_NG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_SG*/ + Arguments.MEMNUM, /*OP_REPEAT_INC_NG_SG*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_START*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END_MEMST*/ + Arguments.MEMNUM, /*OP_NULL_CHECK_END_MEMST_PUSH*/ + Arguments.NON, /*OP_PUSH_POS*/ + Arguments.NON, /*OP_POP_POS*/ + Arguments.RELADDR, /*OP_PUSH_POS_NOT*/ + Arguments.NON, /*OP_FAIL_POS*/ + Arguments.NON, /*OP_PUSH_STOP_BT*/ + Arguments.NON, /*OP_POP_STOP_BT*/ + Arguments.SPECIAL, /*OP_LOOK_BEHIND*/ + Arguments.SPECIAL, /*OP_PUSH_LOOK_BEHIND_NOT*/ + Arguments.NON, /*OP_FAIL_LOOK_BEHIND_NOT*/ + Arguments.NON, /*OP_PUSH_ABSENT_POS*/ + Arguments.RELADDR, /*OP_ABSENT*/ + Arguments.NON, /*OP_ABSENT_END*/ + Arguments.ABSADDR, /*OP_CALL*/ + Arguments.NON, /*OP_RETURN*/ + Arguments.SPECIAL, /*OP_CONDITION*/ + Arguments.SPECIAL, /*OP_STATE_CHECK_PUSH*/ + Arguments.SPECIAL, /*OP_STATE_CHECK_PUSH_OR_JUMP*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK_ANYCHAR_STAR*/ + Arguments.STATE_CHECK, /*OP_STATE_CHECK_ANYCHAR_ML_STAR*/ + Arguments.OPTION, /*OP_SET_OPTION_PUSH*/ + Arguments.OPTION, /*OP_SET_OPTION*/ + + Arguments.SPECIAL, /*OP_EXACT1_IC*/ + Arguments.SPECIAL, /*OP_EXACTN_IC*/ + Arguments.MEMNUM, /*OP_CALLOUT*/ + Arguments.SPECIAL, /*OP_CALLOUT_CONDITION*/ + } : null; +} diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java new file mode 100644 index 000000000..d5a411b69 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -0,0 +1,82 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface OPSize { + + // this might be helpful for potential byte[] migration + int OPCODE = 1; + int RELADDR = 1; + int ABSADDR = 1; + int LENGTH = 1; + int MEMNUM = 1; + int STATE_CHECK_NUM = 1; + int REPEATNUM = 1; + int OPTION = 1; + int CODE_POINT = 1; + int POINTER = 1; + int INDEX = 1; + + /* op-code + arg size */ + + int ANYCHAR_STAR = OPCODE; + int ANYCHAR_STAR_PEEK_NEXT = (OPCODE + 1); + int JUMP = (OPCODE + RELADDR); + int PUSH = (OPCODE + RELADDR); + int POP = OPCODE; + int PUSH_OR_JUMP_EXACT1 = (OPCODE + RELADDR + 1); + int PUSH_IF_PEEK_NEXT = (OPCODE + RELADDR + 1); + int REPEAT_INC = (OPCODE + MEMNUM); + int REPEAT_INC_NG = (OPCODE + MEMNUM); + int PUSH_POS = OPCODE; + int PUSH_POS_NOT = (OPCODE + RELADDR); + int POP_POS = OPCODE; + int FAIL_POS = OPCODE; + int SET_OPTION = (OPCODE + OPTION); + int SET_OPTION_PUSH = (OPCODE + OPTION); + int FAIL = OPCODE; + int MEMORY_START = (OPCODE + MEMNUM); + int MEMORY_START_PUSH = (OPCODE + MEMNUM); + int MEMORY_END_PUSH = (OPCODE + MEMNUM); + int MEMORY_END_PUSH_REC = (OPCODE + MEMNUM); + int MEMORY_END = (OPCODE + MEMNUM); + int MEMORY_END_REC = (OPCODE + MEMNUM); + int PUSH_STOP_BT = OPCODE; + int POP_STOP_BT = OPCODE; + int NULL_CHECK_START = (OPCODE + MEMNUM); + int NULL_CHECK_END = (OPCODE + MEMNUM); + int LOOK_BEHIND = (OPCODE + LENGTH); + int PUSH_LOOK_BEHIND_NOT = (OPCODE + RELADDR + LENGTH); + int FAIL_LOOK_BEHIND_NOT = OPCODE; + int CALL = (OPCODE + ABSADDR); + int RETURN = OPCODE; + int CONDITION = (OPCODE + MEMNUM + RELADDR); + int PUSH_ABSENT_POS = OPCODE; + int ABSENT = (OPCODE + RELADDR); + int ABSENT_END = OPCODE; + int CALLOUT = (OPCODE + MEMNUM); + int CALLOUT_CONDITION = (OPCODE + MEMNUM + RELADDR); + + // #ifdef USE_COMBINATION_EXPLOSION_CHECK + int STATE_CHECK = (OPCODE + STATE_CHECK_NUM); + int STATE_CHECK_PUSH = (OPCODE + STATE_CHECK_NUM + RELADDR); + int STATE_CHECK_PUSH_OR_JUMP = (OPCODE + STATE_CHECK_NUM + RELADDR); + int STATE_CHECK_ANYCHAR_STAR = (OPCODE + STATE_CHECK_NUM); +} diff --git a/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java b/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java new file mode 100644 index 000000000..c27ad5593 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StackPopLevel.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StackPopLevel { + int FREE = 0; + int MEM_START = 1; + int ALL = 2; + +} diff --git a/third_party/joni/src/org/joni/constants/internal/StackType.java b/third_party/joni/src/org/joni/constants/internal/StackType.java new file mode 100644 index 000000000..8f7224076 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StackType.java @@ -0,0 +1,54 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StackType { + /** stack **/ + int INVALID_STACK_INDEX = -1; + + /* stack type */ + /* used by normal-POP */ + int ALT = 0x0001; + int LOOK_BEHIND_NOT = 0x0002; + int POS_NOT = 0x0003; + /* handled by normal-POP */ + int MEM_START = 0x0100; + int MEM_END = 0x8200; + int REPEAT_INC = 0x0300; + int STATE_CHECK_MARK = 0x1000; + /* avoided by normal-POP */ + int NULL_CHECK_START = 0x3000; + int NULL_CHECK_END = 0x5000; /* for recursive call */ + int MEM_END_MARK = 0x8400; + int POS = 0x0500; /* used when POP-POS */ + int STOP_BT = 0x0600; /* mark for "(?>...)" */ + int REPEAT = 0x0700; + int CALL_FRAME = 0x0800; + int RETURN = 0x0900; + int VOID = 0x0a00; /* for fill a blank */ + int ABSENT_POS = 0x0b00; /* for absent */ + int ABSENT = 0x0c00; /* absent inner loop marker */ + int CALLOUT = 0x0d00; /* match-time callback unwind token */ + + /* stack type check mask */ + int MASK_POP_USED = 0x00ff; + int MASK_TO_VOID_TARGET = 0x10ff; + int MASK_MEM_END_OR_MARK = 0x8000; /* MEM_END or MEM_END_MARK */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/StringType.java b/third_party/joni/src/org/joni/constants/internal/StringType.java new file mode 100644 index 000000000..032e452a2 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/StringType.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface StringType { + int NSTR_RAW = 1<<0; + int NSTR_AMBIG = 1<<1; + int NSTR_DONT_GET_OPT_INFO = 1<<2; + int NSTR_SHARED = 1<<3; +} diff --git a/third_party/joni/src/org/joni/constants/internal/TargetInfo.java b/third_party/joni/src/org/joni/constants/internal/TargetInfo.java new file mode 100644 index 000000000..cad12f2d8 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/TargetInfo.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface TargetInfo { + int ISNOT_EMPTY = 0; + int IS_EMPTY = 1; + int IS_EMPTY_MEM = 2; + int IS_EMPTY_REC = 3; +} diff --git a/third_party/joni/src/org/joni/constants/internal/TokenType.java b/third_party/joni/src/org/joni/constants/internal/TokenType.java new file mode 100644 index 000000000..2a4891765 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/TokenType.java @@ -0,0 +1,51 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public enum TokenType { + EOT, /* end of token */ + RAW_BYTE, + CHAR, + STRING, + CODE_POINT, + ANYCHAR, + CHAR_TYPE, + BACKREF, + CALL, + ANCHOR, + OP_REPEAT, + INTERVAL, + ANYCHAR_ANYTIME, /* SQL '%' == .* */ + ALT, + SUBEXP_OPEN, + SUBEXP_CLOSE, + CC_OPEN, + QUOTE_OPEN, + CHAR_PROPERTY, /* \p{...}, \P{...} */ + LINEBREAK, + EXTENDED_GRAPHEME_CLUSTER, + KEEP, + /* in cc */ + CC_CLOSE, + CC_RANGE, + POSIX_BRACKET_OPEN, + CC_AND, /* && */ + CC_CC_OPEN /* [ */ +} diff --git a/third_party/joni/src/org/joni/constants/internal/Traverse.java b/third_party/joni/src/org/joni/constants/internal/Traverse.java new file mode 100644 index 000000000..5fab1ec16 --- /dev/null +++ b/third_party/joni/src/org/joni/constants/internal/Traverse.java @@ -0,0 +1,26 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.constants.internal; + +public interface Traverse { + int TRAVERSE_CALLBACK_AT_FIRST = 1; + int TRAVERSE_CALLBACK_AT_LAST = 2; + int TRAVERSE_CALLBACK_AT_BOTH = TRAVERSE_CALLBACK_AT_FIRST | TRAVERSE_CALLBACK_AT_LAST; +} diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java new file mode 100644 index 000000000..b80340b72 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -0,0 +1,89 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +import org.joni.Config; + +public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { + /* internal error */ + String PARSER_BUG = "internal parser error (bug)"; + String UNDEFINED_BYTECODE = "undefined bytecode (bug)"; + String UNEXPECTED_BYTECODE = "unexpected bytecode (bug)"; + String TOO_MANY_CAPTURE_GROUPS = "too many capture groups are specified"; + + /* general error */ + String INVALID_ARGUMENT = "invalid argument"; + + /* syntax error */ + String REGEX_TOO_LONG = "regex length too long"; + String END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace"; + String END_PATTERN_AT_LEFT_BRACKET = "end pattern at left bracket"; + String EMPTY_CHAR_CLASS = "empty char-class"; + String PREMATURE_END_OF_CHAR_CLASS = "premature end of char-class"; + String END_PATTERN_AT_ESCAPE = "end pattern at escape"; + String END_PATTERN_AT_META = "end pattern at meta"; + String END_PATTERN_AT_CONTROL = "end pattern at control"; + String META_CODE_SYNTAX = "invalid meta-code syntax"; + String CONTROL_CODE_SYNTAX = "invalid control-code syntax"; + String CHAR_CLASS_VALUE_AT_END_OF_RANGE = "char-class value at end of range"; + String CHAR_CLASS_VALUE_AT_START_OF_RANGE = "char-class value at start of range"; + String UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS = "unmatched range specifier in char-class"; + String TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED = "target of repeat operator is not specified"; + String TARGET_OF_REPEAT_OPERATOR_INVALID = "target of repeat operator is invalid"; + String NESTED_REPEAT_NOT_ALLOWED = "nested repeat is not allowed"; + String NESTED_REPEAT_OPERATOR = "nested repeat operator"; + String UNMATCHED_CLOSE_PARENTHESIS = "unmatched close parenthesis"; + String END_PATTERN_WITH_UNMATCHED_PARENTHESIS = "end pattern with unmatched parenthesis"; + String END_PATTERN_IN_GROUP = "end pattern in group"; + String UNDEFINED_GROUP_OPTION = "undefined group option"; + String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; + String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; + String INVALID_REPEAT_RANGE_PATTERN = "invalid repeat range {lower,upper}"; + String INVALID_CONDITION_PATTERN = "invalid conditional pattern"; + + /* values error (syntax error) */ + String TOO_BIG_NUMBER = "too big number"; + String TOO_BIG_NUMBER_FOR_REPEAT_RANGE = "too big number for repeat range"; + String UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE = "upper is smaller than lower in repeat range"; + String EMPTY_RANGE_IN_CHAR_CLASS = "empty range in char class"; + String MISMATCH_CODE_LENGTH_IN_CLASS_RANGE = "mismatch multibyte code length in char-class range"; + String TOO_MANY_MULTI_BYTE_RANGES = "too many multibyte code ranges are specified"; + String TOO_SHORT_MULTI_BYTE_STRING = "too short multibyte code string"; + String TOO_BIG_BACKREF_NUMBER = "too big backref number"; + String INVALID_BACKREF = Config.USE_NAMED_GROUP ? "invalid backref number/name" : "invalid backref number"; + String NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED = "numbered backref/call is not allowed. (use name)"; + String TOO_SHORT_DIGITS = "too short digits"; + String INVALID_WIDE_CHAR_VALUE = "invalid wide-char value"; + String EMPTY_GROUP_NAME = "group name is empty"; + String INVALID_GROUP_NAME = "invalid group name <%n>"; + String INVALID_CHAR_IN_GROUP_NAME = Config.USE_NAMED_GROUP ? "invalid char in group name <%n>" : "invalid char in group number <%n>"; + String UNDEFINED_NAME_REFERENCE = "undefined name <%n> reference"; + String UNDEFINED_GROUP_REFERENCE = "undefined group <%n> reference"; + String MULTIPLEX_DEFINED_NAME = "multiplex defined name <%n>"; + String MULTIPLEX_DEFINITION_NAME_CALL = "multiplex definition name <%n> call"; + String PROPERTY_NAME_NEVER_TERMINATED = "property name never terminated \\p{%n"; + String NEVER_ENDING_RECURSION = "never ending recursion"; + String GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY = "group number is too big for capture history"; + String NOT_SUPPORTED_ENCODING_COMBINATION = "not supported encoding combination"; + String INVALID_COMBINATION_OF_OPTIONS = "invalid combination of options"; + String OVER_THREAD_PASS_LIMIT_COUNT = "over thread pass limit count"; + String TOO_BIG_SB_CHAR_VALUE = "too big singlebyte char value"; + +} diff --git a/third_party/joni/src/org/joni/exception/InternalException.java b/third_party/joni/src/org/joni/exception/InternalException.java new file mode 100644 index 000000000..b41e0aaf3 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/InternalException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class InternalException extends JOniException{ + private static final long serialVersionUID = -3871816465397927992L; + + public InternalException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/JOniException.java b/third_party/joni/src/org/joni/exception/JOniException.java new file mode 100644 index 000000000..c2c5d4794 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/JOniException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class JOniException extends RuntimeException{ + private static final long serialVersionUID = -6027192180014164667L; + + public JOniException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/SyntaxException.java b/third_party/joni/src/org/joni/exception/SyntaxException.java new file mode 100644 index 000000000..7e50cf5ba --- /dev/null +++ b/third_party/joni/src/org/joni/exception/SyntaxException.java @@ -0,0 +1,28 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class SyntaxException extends JOniException{ + private static final long serialVersionUID = 7862720128961874288L; + + public SyntaxException(String message) { + super(message); + } +} diff --git a/third_party/joni/src/org/joni/exception/TimeoutException.java b/third_party/joni/src/org/joni/exception/TimeoutException.java new file mode 100644 index 000000000..914892540 --- /dev/null +++ b/third_party/joni/src/org/joni/exception/TimeoutException.java @@ -0,0 +1,9 @@ +package org.joni.exception; + +public class TimeoutException extends InterruptedException { + private static final long serialVersionUID = 1L; + + public TimeoutException() { + super(); + } +} diff --git a/third_party/joni/src/org/joni/exception/ValueException.java b/third_party/joni/src/org/joni/exception/ValueException.java new file mode 100644 index 000000000..790f3546c --- /dev/null +++ b/third_party/joni/src/org/joni/exception/ValueException.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.exception; + +public class ValueException extends SyntaxException{ + private static final long serialVersionUID = -196013852479929134L; + + public ValueException(String message) { + super(message); + } + + public ValueException(String message, String str) { + super(message.replaceAll("%n", str)); + } + + public ValueException(String message, byte[]bytes, int p, int end) { + this(message, new String(bytes, p, end - p)); + } + +} diff --git a/third_party/joni/test/org/joni/test/Test.java b/third_party/joni/test/org/joni/test/Test.java new file mode 100644 index 000000000..91c00be66 --- /dev/null +++ b/third_party/joni/test/org/joni/test/Test.java @@ -0,0 +1,303 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.io.UnsupportedEncodingException; + +import org.jcodings.Encoding; +import org.jcodings.exception.CharacterPropertyException; +import org.joni.Config; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Region; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.JOniException; + +public abstract class Test { + static final boolean VERBOSE = false; + + int nsucc; + int nerror; + int nfail; + + public abstract int option(); + public abstract Encoding encoding(); + public abstract String testEncoding(); + public abstract Syntax syntax(); + + protected String repr(byte[] bytes) { + return new String(bytes); + } + + protected int length(byte[] bytes) { + return bytes.length; + } + + protected String reprTest(byte[] pattern, byte[]str, int option) { + StringBuilder sb = new StringBuilder(); + sb.append("Pattern: [/").append(repr(pattern)).append("/]"); + sb.append(" Str: [\"").append(repr(str)).append("\"]"); + sb.append(" Encoding: [" + encoding() + "]"); + sb.append(" Option: [" + Option.toString(option) + "]"); + sb.append(" Syntax: [" + syntax().name + "]"); + return sb.toString(); + } + + protected void assertTrue(boolean expression, String failMessage) { + if (expression) { + nsucc++; + } else { + Config.err.println(failMessage); + nfail++; + } + } + + public void xerrs(String pattern, String msg) throws Exception { + xerr(pattern.getBytes(testEncoding()), msg, option()); + } + + public void xerrs(String patternIsSubstring, int p, int end, String msg, WarnCallback warnCallback) throws Exception { + xerrs(patternIsSubstring, p, end, msg, option(), warnCallback); + } + + public void xerrs(String pattern, String msg, int option) throws Exception { + xerr(pattern.getBytes(testEncoding()), msg, option); + } + + public void xerrs(String patternIsSubstring, int p, int end, String msg, int option, WarnCallback warnCallback) throws Exception { + p = length(patternIsSubstring.substring(0, p).getBytes(testEncoding())); + end = length(patternIsSubstring.substring(0, end).getBytes(testEncoding())); + xerr(patternIsSubstring.getBytes(testEncoding()), p, end, msg, option, warnCallback); + } + + public void xerr(byte[] pattern, String msg, int option) throws Exception { + xerr(pattern, 0, length(pattern), msg, option, WarnCallback.NONE); + } + + public void xerr(byte[] pattern, int p, int end, String msg, int option, WarnCallback warnCallback) throws Exception { + try { + new Regex(pattern, p, end, option, encoding(), syntax(), warnCallback); + nfail++; + } catch (JOniException | CharacterPropertyException cpe) { + nsucc++; + assertEquals(cpe.getMessage(), msg); + } + } + + public void xx(byte[] pattern, byte[] str, int from, int to, int mem, boolean not) throws InterruptedException { + xx(pattern, str, from, to, mem, not, option()); + } + + public void xx(byte[] pattern, byte[] str, int gpos, int from, int to, int mem, boolean not) throws InterruptedException { + xx(pattern, str, gpos, 0, from, to, mem, not, option()); + } + + static boolean is7bit(byte[]bytes, int p, int end) { + for (int i = p; i < end; i++) { + if ((bytes[i] & 0xff) >= 0x80) return false; + } + return true; + } + + public int xx(byte[] pattern, byte[] str, int from, int to, int mem, boolean not, int option) throws InterruptedException { + return xx(pattern, str, 0, 0, from, to, mem, not, option); + } + + public int xx(byte[] pattern, byte[] str, int gpos, int searchStart, int from, int to, int mem, boolean not, int option) throws InterruptedException { + return xx(pattern, str, 0, length(pattern), gpos, searchStart, from, to, mem, not, option, WarnCallback.NONE); + } + + public int xx(byte[] pattern, byte[] str, int p, int end, int gpos, int searchStart, int from, int to, int mem, boolean not, int option, WarnCallback warnCallback) throws InterruptedException { + Regex reg; + + try { + reg = new Regex(pattern, p, end, option, encoding(), syntax(), warnCallback); + } catch (JOniException je) { + Config.err.println(reprTest(pattern, str, option)); + je.printStackTrace(Config.err); + Config.err.println("ERROR: " + je.getMessage()); + nerror++; + return Matcher.FAILED; + } catch (Exception e) { + Config.err.println(reprTest(pattern, str, option)); + e.printStackTrace(Config.err); + Config.err.println("SEVERE ERROR: " + e.getMessage()); + nerror++; + return Matcher.FAILED; + } + + if ((!encoding().isSingleByte()) && encoding().isAsciiCompatible() && is7bit(str, 0, str.length)) { + check(reg, pattern, str, option | Option.CR_7_BIT, gpos, searchStart, from, to, mem, not); + } + + return check(reg, pattern, str, option, gpos, searchStart, from, to, mem, not); + } + + private int check(Regex reg, byte[]pattern, byte[] str, int option, int gpos, int searchStart, int from, int to, int mem, boolean not) throws InterruptedException { + Matcher m = reg.matcher(str, 0, length(str)); + final Region region; + final int result; + try { + result = m.searchInterruptible(gpos, searchStart, length(str), option); + region = m.getEagerRegion(); + } catch (JOniException je) { + Config.err.println("Pattern: " + reprTest(pattern, str, option)); + je.printStackTrace(Config.err); + Config.err.println("ERROR: " + je.getMessage()); + nerror++; + return Matcher.FAILED; + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + Config.err.println("Pattern: " + reprTest(pattern, str, option)); + e.printStackTrace(Config.err); + Config.err.println("SEVERE ERROR: " + e.getMessage()); + nerror++; + return Matcher.FAILED; + } + + if (result == -1) { + if (not) { + if (VERBOSE) Config.log.println("OK(NOT): " + reprTest(pattern, str, option)); + nsucc++; + } else { + Config.log.println("FAIL: " + reprTest(pattern, str, option) + " GPOS: " + + gpos + " Start: " + searchStart); + nfail++; + } + } else { + if (not) { + Config.log.println("FAIL(NOT): " + reprTest(pattern, str, option)); + nfail++; + } else { + if (region.getBeg(mem) == from && region.getEnd(mem) == to) { + if (VERBOSE) Config.log.println("OK: " + reprTest(pattern, str, option)); + nsucc++; + } else { + Config.log.println("FAIL: " + reprTest(pattern, str, option) + " GPOS: " + gpos + " Start: " + + searchStart + " Groups: [Exp " + from + "-" + to + ", Act " + region.getBeg(mem) + "-" + + region.getEnd(mem) + "]"); + nfail++; + } + } + } + return result; + } + + protected void x2(byte[] pattern, byte[] str, int from, int to) throws InterruptedException { + xx(pattern, str, from, to, 0, false); + } + + protected void x2(byte[] pattern, byte[] str, int from, int to, int option) throws InterruptedException { + xx(pattern, str, from, to, 0, false, option); + } + + protected void x3(byte[] pattern, byte[] str, int from, int to, int mem) throws InterruptedException { + xx(pattern, str, from, to, mem, false); + } + + protected void n(byte[] pattern, byte[] str) throws InterruptedException { + xx(pattern, str, 0, 0, 0, true); + } + + protected void n(byte[] pattern, byte[] str, int option) throws InterruptedException { + xx(pattern, str, 0, 0, 0, true, option); + } + + protected void n(byte[] pattern, byte[] str, int gpos, int option) throws InterruptedException { + xx(pattern, str, gpos, 0, 0, 0, 0, true, option); + } + + public void xxs(String pattern, String str, int from, int to, int mem, boolean not) throws InterruptedException { + xxs(pattern, str, from, to, mem, not, option()); + } + + public void xxs(String pattern, String str, int from, int to, int mem, boolean not, int option) + throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, 0, 0, from, to, mem, not, option, WarnCallback.NONE); + } + + public int x2s(String pattern, String str, int from, int to) throws InterruptedException { + return x2s(pattern, str, from, to, option()); + } + + public int x2s(String pattern, String str, int from, int to, int option) throws InterruptedException { + return x2s(pattern, str, 0, 0, from, to, option); + } + + public int x2s(String pattern, String str, int gpos, int searchStart, int from, int to) throws InterruptedException { + return x2s(pattern, str, gpos, searchStart, from, to, option()); + } + + public int x2s(String pattern, String str, int gpos, int searchStart, int from, int to, int option) throws InterruptedException { + return x2s(pattern, 0, pattern.length(), str, gpos, searchStart, from, to, 0, false, option, WarnCallback.NONE); + } + + public int x2s(String patternIsSubstring, int p, int end, String str, int from, int to, boolean not, WarnCallback warnCallback) throws InterruptedException { + return x2s(patternIsSubstring, p, end, str, 0, 0, from, to, 0, not, option(), warnCallback); + } + + public int x2s(String patternIsSubstring, int p, int end, String str, int gpos, int searchStart, int from, int to, int mem, boolean not, int option, WarnCallback warnCallback) throws InterruptedException { + try { + byte[] patternBytes = patternIsSubstring.substring(p, end).getBytes(testEncoding()); + // convert char p and end to byte + byte[] prefixBytes = patternIsSubstring.substring(0, p).getBytes(testEncoding()); + p = prefixBytes.length == 0 ? 0 : length(prefixBytes); + end = p + length(patternBytes); + return xx(patternIsSubstring.getBytes(testEncoding()), str.getBytes(testEncoding()), p, end, gpos, searchStart, from, to, mem, not, option, warnCallback); + } catch (UnsupportedEncodingException uee) { + uee.printStackTrace(); + return Matcher.FAILED; + } + } + + public void x3s(String pattern, String str, int from, int to, int mem) throws InterruptedException { + x3s(pattern, str, from, to, mem, option()); + } + + public void x3s(String pattern, String str, int from, int to, int mem, int option) throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, 0, 0, from, to, mem, false, option, WarnCallback.NONE); + } + + public void ns(String pattern, String str) throws InterruptedException { + ns(pattern, str, option()); + } + + public void ns(String pattern, String str, int option) throws InterruptedException { + ns(pattern, str, 0, option); + } + + public void ns(String pattern, String str, int gpos, int option) throws InterruptedException { + x2s(pattern, 0, pattern.length(), str, gpos, 0, 0, 0, 0, true, option, WarnCallback.NONE); + } + + @org.junit.Test + public void testRegexp() throws Exception { + test(); + Config.log.println("RESULT SUCC: " + nsucc + ", FAIL: " + nfail + ", ERROR: " + nerror + " Test: " + getClass().getSimpleName() + ", Encoding: " + encoding()); + assertEquals(0, nfail + nerror); + } + + public abstract void test() throws Exception; +} diff --git a/third_party/joni/test/org/joni/test/TestA.java b/third_party/joni/test/org/joni/test/TestA.java new file mode 100644 index 000000000..d814e3687 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestA.java @@ -0,0 +1,557 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.ErrorMessages; + +public class TestA extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "iso-8859-2"; + } + + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + x2s("", "", 0, 0); + x2s("^", "", 0, 0); + x2s("$", "", 0, 0); + x2s("\\G", "", 0, 0); + x2s("\\A", "", 0, 0); + x2s("\\Z", "", 0, 0); + x2s("\\z", "", 0, 0); + x2s("^$", "", 0, 0); + x2s("\\ca", "\001", 0, 1); + x2s("\\C-b", "\002", 0, 1); + x2s("\\c\\\\", "\034", 0, 1); + x2s("q[\\c\\\\]", "q\034", 0, 2); + x2s("", "a", 0, 0); + x2s("a", "a", 0, 1); + x2s("\\x61", "a", 0, 1); + x2s("aa", "aa", 0, 2); + x2s("aaa", "aaa", 0, 3); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35); + x2s("ab", "ab", 0, 2); + x2s("b", "ab", 1, 2); + x2s("bc", "abc", 1, 3); + x2s("(?i:#RET#)", "#INS##RET#", 5, 10); + x2s("\\17", "\017", 0, 1); + x2s("\\x1f", "\u001f", 0, 1); + x2s("\\xED\\xF2", "\u00ed\u0148", 0, 2); + x2s("a(?#....\\\\JJJJ)b", "ab", 0, 2); + x2s("(?x) G (o O(?-x)oO) g L", "GoOoOgLe", 0, 7); + x2s(".", "a", 0, 1); + ns(".", ""); + x2s("..", "ab", 0, 2); + x2s("\\w", "e", 0, 1); + ns("\\W", "e"); + x2s("\\s", " ", 0, 1); + x2s("\\S", "b", 0, 1); + x2s("\\d", "4", 0, 1); + ns("\\D", "4"); + x2s("\\b", "z ", 0, 0); + x2s("\\b", " z", 1, 1); + x2s("\\B", "zz ", 1, 1); + x2s("\\B", "z ", 2, 2); + x2s("\\B", " z", 0, 0); + x2s("[ab]", "b", 0, 1); + ns("[ab]", "c"); + x2s("[a-z]", "t", 0, 1); + ns("[^a]", "a"); + x2s("[^a]", "\n", 0, 1); + x2s("[]]", "]", 0, 1); + ns("[^]]", "]"); + x2s("[\\^]+", "0^^1", 1, 3); + x2s("[b-]", "b", 0, 1); + x2s("[b-]", "-", 0, 1); + x2s("[\\w]", "z", 0, 1); + ns("[\\w]", " "); + x2s("[\\W]", "b$", 1, 2); + x2s("[\\d]", "5", 0, 1); + ns("[\\d]", "e"); + x2s("[\\D]", "t", 0, 1); + ns("[\\D]", "3"); + x2s("[\\s]", " ", 0, 1); + ns("[\\s]", "a"); + x2s("[\\S]", "b", 0, 1); + ns("[\\S]", " "); + x2s("[\\w\\d]", "2", 0, 1); + ns("[\\w\\d]", " "); + x2s("[[:upper:]]", "B", 0, 1); + x2s("[*[:xdigit:]+]", "+", 0, 1); + x2s("[*[:xdigit:]+]", "GHIKK-9+*", 6, 7); + x2s("[*[:xdigit:]+]", "-@^+", 3, 4); + ns("[[:upper]]", "A"); + x2s("[[:upper]]", ":", 0, 1); + x2s("[\\044-\\047]", "\046", 0, 1); + x2s("[\\x5a-\\x5c]", "\u005b", 0, 1); + x2s("[\\x6A-\\x6D]", "\u006c", 0, 1); + ns("[\\x6A-\\x6D]", "\u006e"); + ns("^[0-9A-F]+ 0+ UNDEF ", "75F 00000000 SECT14A notype () External | _rb_apply"); + x2s("[\\[]", "[", 0, 1); + x2s("[\\]]", "]", 0, 1); + x2s("[&]", "&", 0, 1); + x2s("[[ab]]", "b", 0, 1); + x2s("[[ab]c]", "c", 0, 1); + ns("[[^a]]", "a"); + ns("[^[a]]", "a"); + x2s("[[ab]&&bc]", "b", 0, 1); + ns("[[ab]&&bc]", "a"); + ns("[[ab]&&bc]", "c"); + x2s("[a-z&&b-y&&c-x]", "w", 0, 1); + ns("[^a-z&&b-y&&c-x]", "w"); + x2s("[[^a&&a]&&a-z]", "b", 0, 1); + ns("[[^a&&a]&&a-z]", "a"); + x2s("[[^a-z&&bcdef]&&[^c-g]]", "h", 0, 1); + ns("[[^a-z&&bcdef]&&[^c-g]]", "c"); + x2s("[^[^abc]&&[^cde]]", "c", 0, 1); + x2s("[^[^abc]&&[^cde]]", "e", 0, 1); + ns("[^[^abc]&&[^cde]]", "f"); + x2s("[a-&&-a]", "-", 0, 1); + ns("[a\\-&&\\-a]", "&"); + ns("\\wabc", " abc"); + x2s("a\\Wbc", "a bc", 0, 4); + x2s("a.b.c", "aabbc", 0, 5); + x2s(".\\wb\\W..c", "abb bcc", 0, 7); + x2s("\\s\\wzzz", " zzzz", 0, 5); + x2s("aa.b", "aabb", 0, 4); + ns(".a", "ab"); + x2s(".a", "aa", 0, 2); + x2s("^a", "a", 0, 1); + x2s("^a$", "a", 0, 1); + x2s("^\\w$", "a", 0, 1); + ns("^\\w$", " "); + x2s("^\\wab$", "zab", 0, 3); + x2s("^\\wabcdef$", "zabcdef", 0, 7); + x2s("^\\w...def$", "zabcdef", 0, 7); + x2s("\\w\\w\\s\\Waaa\\d", "aa aaa4", 0, 8); + x2s("\\A\\Z", "", 0, 0); + x2s("\\Axyz", "xyz", 0, 3); + x2s("xyz\\Z", "xyz", 0, 3); + x2s("xyz\\z", "xyz", 0, 3); + x2s("a\\Z", "a", 0, 1); + x2s("0\\gA", 1, 4, "doesntMatter", 0, 0, true, WarnCallback.DEFAULT); + x2s("\\Gaz", "az", 0, 2); + ns("\\Gz", "bza"); + ns("az\\G", "az"); + ns("az\\A", "az"); + ns("a\\Az", "az"); + x2s("\\^\\$", "^$", 0, 2); + x2s("^x?y", "xy", 0, 2); + x2s("^(x?y)", "xy", 0, 2); + x2s("\\w", "_", 0, 1); + ns("\\W", "_"); + x2s("(?=z)z", "z", 0, 1); + ns("(?=z).", "a"); + x2s("(?!z)a", "a", 0, 1); + ns("(?!z)a", "z"); + x2s("(?i:a)", "a", 0, 1); + x2s("(?i:a)", "A", 0, 1); + x2s("(?i:A)", "a", 0, 1); + ns("(?i:A)", "b"); + x2s("(?i:[A-Z])", "a", 0, 1); + x2s("(?i:[f-m])", "H", 0, 1); + x2s("(?i:[f-m])", "h", 0, 1); + ns("(?i:[f-m])", "e"); + x2s("(?i:[A-c])", "D", 0, 1); + x2s("(?i:[!-k])", "Z", 0, 1); + x2s("(?i:[!-k])", "7", 0, 1); + x2s("(?i:[T-}])", "b", 0, 1); + x2s("(?i:[T-}])", "{", 0, 1); + x2s("(?i:\\?a)", "?A", 0, 2); + x2s("(?i:\\*A)", "*a", 0, 2); + ns(".", "\n"); + x2s("(?m:.)", "\n", 0, 1); + x2s("(?m:a.)", "a\n", 0, 2); + x2s("(?m:.b)", "a\nb", 1, 3); + x2s(".*abc", "dddabdd\nddabc", 8, 13); + x2s("(?m:.*abc)", "dddabddabc", 0, 10); + ns("(?i)(?-i)a", "A"); + ns("(?i)(?-i:a)", "A"); + x2s("a?", "", 0, 0); + x2s("a?", "b", 0, 0); + x2s("a?", "a", 0, 1); + x2s("a*", "", 0, 0); + x2s("a*", "a", 0, 1); + x2s("a*", "aaa", 0, 3); + x2s("a*", "baaaa", 0, 0); + ns("a+", ""); + x2s("a+", "a", 0, 1); + x2s("a+", "aaaa", 0, 4); + x2s("a+", "aabbb", 0, 2); + x2s("a+", "baaaa", 1, 5); + x2s(".?", "", 0, 0); + x2s(".?", "f", 0, 1); + x2s(".?", "\n", 0, 0); + x2s(".*", "", 0, 0); + x2s(".*", "abcde", 0, 5); + x2s(".+", "z", 0, 1); + x2s(".+", "zdswer\n", 0, 6); + x2s("(.*)a\\1f", "babfbac", 0, 4); + x2s("(.*)a\\1f", "bacbabf", 3, 7); + x2s("((.*)a\\2f)", "bacbabf", 3, 7); + x2s("(.*)a\\1f", "baczzzzzz\nbazz\nzzzzbabf", 19, 23); + x2s("a|b", "a", 0, 1); + x2s("a|b", "b", 0, 1); + x2s("|a", "a", 0, 0); + x2s("(|a)", "a", 0, 0); + x2s("ab|bc", "ab", 0, 2); + x2s("ab|bc", "bc", 0, 2); + x2s("z(?:ab|bc)", "zbc", 0, 3); + x2s("a(?:ab|bc)c", "aabc", 0, 4); + x2s("ab|(?:ac|az)", "az", 0, 2); + x2s("a|b|c", "dc", 1, 2); + x2s("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "pqr", 0, 2); + ns("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "mn"); + x2s("a|^z", "ba", 1, 2); + x2s("a|^z", "za", 0, 1); + x2s("a|\\Gz", "bza", 2, 3); + x2s("a|\\Gz", "za", 0, 1); + x2s("a|\\Az", "bza", 2, 3); + x2s("a|\\Az", "za", 0, 1); + x2s("a|b\\Z", "ba", 1, 2); + x2s("a|b\\Z", "b", 0, 1); + x2s("a|b\\z", "ba", 1, 2); + x2s("a|b\\z", "b", 0, 1); + x2s("\\w|\\s", " ", 0, 1); + ns("\\w|\\w", " "); + x2s("\\w|%", "%", 0, 1); + x2s("\\w|[&$]", "&", 0, 1); + x2s("[b-d]|[^e-z]", "a", 0, 1); + x2s("(?:a|[c-f])|bz", "dz", 0, 1); + x2s("(?:a|[c-f])|bz", "bz", 0, 2); + x2s("abc|(?=zz)..f", "zzf", 0, 3); + x2s("abc|(?!zz)..f", "abf", 0, 3); + x2s("(?=za)..a|(?=zz)..a", "zza", 0, 3); + ns("(?>a|abd)c", "abdc"); + x2s("(?>abd|a)c", "abdc", 0, 4); + x2s("a?|b", "a", 0, 1); + x2s("a?|b", "b", 0, 0); + x2s("a?|b", "", 0, 0); + x2s("a*|b", "aa", 0, 2); + x2s("a*|b*", "ba", 0, 0); + x2s("a*|b*", "ab", 0, 1); + x2s("a+|b*", "", 0, 0); + x2s("a+|b*", "bbb", 0, 3); + x2s("a+|b*", "abbb", 0, 1); + ns("a+|b+", ""); + x2s("(a|b)?", "b", 0, 1); + x2s("(a|b)*", "ba", 0, 2); + x2s("(a|b)+", "bab", 0, 3); + x2s("(ab|ca)+", "caabbc", 0, 4); + x2s("(ab|ca)+", "aabca", 1, 5); + x2s("(ab|ca)+", "abzca", 0, 2); + x2s("(a|bab)+", "ababa", 0, 5); + x2s("(a|bab)+", "ba", 1, 2); + x2s("(a|bab)+", "baaaba", 1, 4); + x2s("(?:a|b)(?:a|b)", "ab", 0, 2); + x2s("(?:a*|b*)(?:a*|b*)", "aaabbb", 0, 3); + x2s("(?:a*|b*)(?:a+|b+)", "aaabbb", 0, 6); + x2s("(?:a+|b+){2}", "aaabbb", 0, 6); + x2s("h{0,}", "hhhh", 0, 4); + x2s("(?:a+|b+){1,2}", "aaabbb", 0, 6); + ns("ax{2}*a", "0axxxa1"); + ns("a.{0,2}a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXXa0"); + x2s("^a{2,}?a$", "aaa", 0, 3); + x2s("^[a-z]{2,}?$", "aaa", 0, 3); + x2s("(?:a+|\\Ab*)cc", "cc", 0, 2); + ns("(?:a+|\\Ab*)cc", "abcc"); + x2s("(?:^a+|b+)*c", "aabbbabc", 6, 8); + x2s("(?:^a+|b+)*c", "aabbbbc", 0, 7); + x2s("a|(?i)c", "C", 0, 1); + x2s("(?i)c|a", "C", 0, 1); + x2s("(?i)c|a", "A", 0, 1); + x2s("(?i:c)|a", "C", 0, 1); + ns("(?i:c)|a", "A"); + x2s("[abc]?", "abc", 0, 1); + x2s("[abc]*", "abc", 0, 3); + x2s("[^abc]*", "abc", 0, 0); + ns("[^abc]+", "abc"); + x2s("a??", "aaa", 0, 0); + x2s("ba??b", "bab", 0, 3); + x2s("a*?", "aaa", 0, 0); + x2s("ba*?", "baa", 0, 1); + x2s("ba*?b", "baab", 0, 4); + x2s("a+?", "aaa", 0, 1); + x2s("ba+?", "baa", 0, 2); + x2s("ba+?b", "baab", 0, 4); + x2s("(?:a?)??", "a", 0, 0); + x2s("(?:a??)?", "a", 0, 0); + x2s("(?:a?)+?", "aaa", 0, 1); + x2s("(?:a+)??", "aaa", 0, 0); + x2s("(?:a+)??b", "aaab", 0, 4); + x2s("(?:ab)?{2}", "", 0, 0); + x2s("(?:ab)?{2}", "ababa", 0, 4); + x2s("(?:ab)*{0}", "ababa", 0, 0); + x2s("(?:ab){3,}", "abababab", 0, 8); + ns("(?:ab){3,}", "abab"); + x2s("(?:ab){2,4}", "ababab", 0, 6); + x2s("(?:ab){2,4}", "ababababab", 0, 8); + x2s("(?:ab){2,4}?", "ababababab", 0, 4); + x2s("(?:ab){,}", "ab{,}", 0, 5); + x2s("(?:abc)+?{2}", "abcabcabc", 0, 6); + x2s("(?:X*)(?i:xa)", "XXXa", 0, 4); + x2s("(d+)([^abc]z)", "dddz", 0, 4); + x2s("([^abc]*)([^abc]z)", "dddz", 0, 4); + x2s("(\\w+)(\\wz)", "dddz", 0, 4); + x3s("(a)", "a", 0, 1, 1); + x3s("(ab)", "ab", 0, 2, 1); + x2s("((ab))", "ab", 0, 2); + x3s("((ab))", "ab", 0, 2, 1); + x3s("((ab))", "ab", 0, 2, 2); + x3s("((((((((((((((((((((ab))))))))))))))))))))", "ab", 0, 2, 20); + x3s("(ab)(cd)", "abcd", 0, 2, 1); + x3s("(ab)(cd)", "abcd", 2, 4, 2); + x3s("()(a)bc(def)ghijk", "abcdefghijk", 3, 6, 3); + x3s("(()(a)bc(def)ghijk)", "abcdefghijk", 3, 6, 4); + x2s("(^a)", "a", 0, 1); + x3s("(a)|(a)", "ba", 1, 2, 1); + x3s("(^a)|(a)", "ba", 1, 2, 2); + x3s("(a?)", "aaa", 0, 1, 1); + x3s("(a*)", "aaa", 0, 3, 1); + x3s("(a*)", "", 0, 0, 1); + x3s("(a+)", "aaaaaaa", 0, 7, 1); + x3s("(a+|b*)", "bbbaa", 0, 3, 1); + x3s("(a+|b?)", "bbbaa", 0, 1, 1); + x3s("(abc)?", "abc", 0, 3, 1); + x3s("(abc)*", "abc", 0, 3, 1); + x3s("(abc)+", "abc", 0, 3, 1); + x3s("(xyz|abc)+", "abc", 0, 3, 1); + x3s("([xyz][abc]|abc)+", "abc", 0, 3, 1); + x3s("((?i:abc))", "AbC", 0, 3, 1); + x2s("(abc)(?i:\\1)", "abcABC", 0, 6); + x3s("((?m:a.c))", "a\nc", 0, 3, 1); + x3s("((?=az)a)", "azb", 0, 1, 1); + x3s("abc|(.abd)", "zabd", 0, 4, 1); + x2s("(?:abc)|(ABC)", "abc", 0, 3); + x3s("(?i:(abc))|(zzz)", "ABC", 0, 3, 1); + x3s("a*(.)", "aaaaz", 4, 5, 1); + x3s("a*?(.)", "aaaaz", 0, 1, 1); + x3s("a*?(c)", "aaaac", 4, 5, 1); + x3s("[bcd]a*(.)", "caaaaz", 5, 6, 1); + x3s("(\\Abb)cc", "bbcc", 0, 2, 1); + ns("(\\Abb)cc", "zbbcc"); + x3s("(^bb)cc", "bbcc", 0, 2, 1); + ns("(^bb)cc", "zbbcc"); + x3s("cc(bb$)", "ccbb", 2, 4, 1); + ns("cc(bb$)", "ccbbb"); + ns("(\\1)", ""); + ns("\\1(a)", "aa"); + ns("(a(b)\\1)\\2+", "ababb"); + ns("(?:(?:\\1|z)(a))+$", "zaa"); + x2s("(?:(?:\\1|z)(a))+$", "zaaa", 0, 4); + x2s("(a)(?=\\1)", "aa", 0, 1); + ns("(a)$|\\1", "az"); + x2s("(a)\\1", "aa", 0, 2); + ns("(a)\\1", "ab"); + x2s("(a?)\\1", "aa", 0, 2); + x2s("(a??)\\1", "aa", 0, 0); + x2s("(a*)\\1", "aaaaa", 0, 4); + x3s("(a*)\\1", "aaaaa", 0, 2, 1); + x2s("a(b*)\\1", "abbbb", 0, 5); + x2s("a(b*)\\1", "ab", 0, 1); + x2s("(a*)(b*)\\1\\2", "aaabbaaabb", 0, 10); + x2s("(a*)(b*)\\2", "aaabbbb", 0, 7); + x2s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 8); + x3s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 3, 7); + x2s("(a)(b)(c)\\2\\1\\3", "abcbac", 0, 6); + x2s("([a-d])\\1", "cc", 0, 2); + x2s("(\\w\\d\\s)\\1", "f5 f5 ", 0, 6); + ns("(\\w\\d\\s)\\1", "f5 f5"); + x2s("(who|[a-c]{3})\\1", "whowho", 0, 6); + x2s("...(who|[a-c]{3})\\1", "abcwhowho", 0, 9); + x2s("(who|[a-c]{3})\\1", "cbccbc", 0, 6); + x2s("(^a)\\1", "aa", 0, 2); + ns("(^a)\\1", "baa"); + ns("(a$)\\1", "aa"); + ns("(ab\\Z)\\1", "ab"); + x2s("(a*\\Z)\\1", "a", 1, 1); + x2s(".(a*\\Z)\\1", "ba", 1, 2); + x3s("(.(abc)\\2)", "zabcabc", 0, 7, 1); + x3s("(.(..\\d.)\\2)", "z12341234", 0, 9, 1); + x2s("((?i:az))\\1", "AzAz", 0, 4); + ns("((?i:az))\\1", "Azaz"); + x2s("(?<=a)b", "ab", 1, 2); + ns("(?<=a)b", "bb"); + x2s("(?<=a|b)b", "bb", 1, 2); + x2s("(?<=a|bc)b", "bcb", 2, 3); + x2s("(?<=a|bc)b", "ab", 1, 2); + x2s("(?<=a|bc||defghij|klmnopq|r)z", "rz", 1, 2); + x2s("(a)\\g<1>", "aa", 0, 2); + x2s("(?a)", "a", 0, 1); + x2s("(?ab)\\g", "abab", 0, 4); + x2s("(?.zv.)\\k", "azvbazvb", 0, 8); + x2s("(?<=\\g)|-\\zEND (?XyZ)", "XyZ", 3, 3); + x2s("(?|a\\g)+", "", 0, 0); + x2s("(?|\\(\\g\\))+$", "()(())", 0, 6); + x3s("\\g(?.){0}", "X", 0, 1, 1); + x2s("\\g(abc|df(?.YZ){2,8}){0}", "XYZ", 0, 3); + x2s("\\A(?(a\\g)|)\\z", "aaaa", 0, 4); + x2s("(?|\\g\\g)\\z|\\zEND (?a|(b)\\g)", "bbbbabba", 0, 8); + x2s("(?\\w+\\sx)a+\\k", " fg xaaaaaaaafg x", 2, 18); + x3s("(z)()()(?<_9>a)\\g<_9>", "zaa", 2, 3, 1); + x2s("(.)(((?<_>a)))\\k<_>", "zaa", 0, 3); + x2s("((?\\d)|(?\\w))(\\k|\\k)", "ff", 0, 2); + x2s("(?:(?)|(?efg))\\k", "", 0, 0); + x2s("(?:(?abc)|(?efg))\\k", "abcefgefg", 3, 9); + ns("(?:(?abc)|(?efg))\\k", "abcefg"); + x2s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "a-pyumpyum", 2, 10); + x3s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "xxxxabcdefghijklmnabcdefghijklmn", 4, 18, 14); + x3s("(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?aaa)(?)$", "aaa", 0, 3, 16); + x2s("(?a|\\(\\g\\))", "a", 0, 1); + x2s("(?a|\\(\\g\\))", "((((((a))))))", 0, 13); + x3s("(?a|\\(\\g\\))", "((((((((a))))))))", 0, 17, 1); + x2s("\\g|\\zEND(?.*abc$)", "abcxxxabc", 0, 9); + x2s("\\g<1>|\\zEND(.a.)", "bac", 0, 3); + x3s("\\g<_A>\\g<_A>|\\zEND(.a.)(?<_A>.b.)", "xbxyby", 3, 6, 1); + x2s("\\A(?:\\g|\\g|\\zEND (?a|c\\gc)(?b|d\\gd))$", "cdcbcdc", 0, 7); + x2s("\\A(?|a\\g)\\z|\\zEND (?\\g)", "aaaa", 0, 4); + x2s("(?(a|b\\gc){3,5})", "baaaaca", 1, 5); + x2s("(?(a|b\\gc){3,5})", "baaaacaaaaa", 0, 10); + x2s("(?\\(([^\\(\\)]++|\\g)*+\\))", "((a))", 0, 5); + x2s("()*\\1", "", 0, 0); + x2s("(?:()|())*\\1\\2", "", 0, 0); + x3s("(?:\\1a|())*", "a", 0, 0, 1); + x2s("x((.)*)*x", "0x1x2x3", 1, 6); + x2s("x((.)*)*x(?i:\\1)\\Z", "0x1x2x1X2", 1, 9); + x2s("(?:()|()|()|()|()|())*\\2\\5", "", 0, 0); + x2s("(?:()|()|()|(x)|()|())*\\2b\\5", "b", 0, 1); + + x3s("\\A(?|.|(?:(?.)\\g\\k))\\z", "reer", 0, 4, 1); + x3s("(?-i:\\g)(?i:(?a)){0}", "A", 0, 1, 1); + + String pat = + "(? \\g \\g* \\g ){0}" + + "(? < \\g \\s* > ){0}" + + "(? [a-zA-Z_:]+ ){0}" + + "(? [^<&]+ (\\g | [^<&]+)* ){0}" + + "(? >){0}" + + "\\g"; + + String str = "fbbbf"; + + x3s(pat, str, 0, 27, 0, Option.EXTEND); + x3s(pat, str, 0, 27, 1, Option.EXTEND); + x3s(pat, str, 6, 11, 2, Option.EXTEND); + x3s(pat, str, 7, 10, 3, Option.EXTEND); + x3s(pat, str, 5, 21, 4, Option.EXTEND); + x3s(pat, str, 21, 27, 5, Option.EXTEND); + + x2s("(a)b\\k<1>", "aba", 0, 3); + x2s("^(?>(?=a)(a|))++$", "a", 0, 1); + x2s("\\k", "k", 0, 1); + x2s("\\kx", "kx", 0, 2); + x2s("\\g", "g", 0, 1); + x2s("\\gx", "gx", 0, 2); + x2s("\\k\\g", "kg", 0, 2); + ns("\\00", "00"); + ns("\\70", "70"); + x2s("\\80", "80", 0, 2); + x2s("\\90", "90", 0, 2); + + ns("(?a)|(?b))(?:(?()cd|x)e|fg)", "bxe", 0, 3); + ns("(?:(?a)|(?b))(?:(?()cd|x)e|fg)", "bxe"); + x2s("((?<=a))?(?(1)b|c)", "abc", 1, 2); + x2s("((?<=a))?(?(1)b|c)", "bc", 1, 2); + x2s("((?x)|(?y))(?()y|x)", "xy", 0, 2); + x2s("((?x)|(?y))(?()y|x)", "yx", 0, 2); + ns("((?x)|(?y))(?()y|x)", "xx"); + ns("((?x)|(?y))(?()y|x)", "yy"); + xerrs("(a)?(?b)?(?(1)a)(?()b)", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + xerrs("()(?(2))", ErrorMessages.INVALID_BACKREF); + xerrs("(?(700000))", ErrorMessages.INVALID_BACKREF); + + x2s("\\R", "\n", 0, 1); + x2s("\\R", "\r", 0, 1); + x2s("\\R{3}", "\r\r\n\n", 0, 4); + + x2s("\\X{5}", "あいab\n", 0, 5); + x2s("\\X{5}", "あいab\n", 0, 5); + x2s("(?<=a).*b", "aab", 1, 3); + + x2s("([.])", ".", 0, 1); + x2s("([a])", "a", 0, 1); + x2s("([\\w])", "a", 0, 1); + + // gpos + ns("\\Gabc", "123abcdef", 2, Option.DEFAULT); + x2s("\\Gabc", "123abcdef", 3, 0, 3, 6); + x2s("\\Gabc", "123abcdef", 3, 3, 3, 6); + ns("\\Gabc", "123abcdef", 0, 3); + + x2s("(?!\\G)", "abcd", 2, 3, 3, 3); + x2s("(?!\\G)", "abcd", 3, 3, 4, 4); + } +} diff --git a/third_party/joni/test/org/joni/test/TestC.java b/third_party/joni/test/org/joni/test/TestC.java new file mode 100755 index 000000000..d9d5d16ad --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestC.java @@ -0,0 +1,732 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.EUCJPEncoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestC extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return EUCJPEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "cp1250"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + x2s("", "", 0, 0); + x2s("^", "", 0, 0); + x2s("$", "", 0, 0); + x2s("\\G", "", 0, 0); + x2s("\\A", "", 0, 0); + x2s("\\Z", "", 0, 0); + x2s("\\z", "", 0, 0); + x2s("^$", "", 0, 0); + x2s("\\ca", "\001", 0, 1); + x2s("\\C-b", "\002", 0, 1); + x2s("\\c\\\\", "\034", 0, 1); + x2s("q[\\c\\\\]", "q\034", 0, 2); + x2s("", "a", 0, 0); + x2s("a", "a", 0, 1); + x2s("\\x61", "a", 0, 1); + x2s("aa", "aa", 0, 2); + x2s("aaa", "aaa", 0, 3); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35); + x2s("ab", "ab", 0, 2); + x2s("b", "ab", 1, 2); + x2s("bc", "abc", 1, 3); + x2s("(?i:#RET#)", "#INS##RET#", 5, 10); + x2s("\\17", "\017", 0, 1); + x2s("\\x1f", "\u001f", 0, 1); + x2s("a(?#....\\\\JJJJ)b", "ab", 0, 2); + x2s("(?x) G (o O(?-x)oO) g L", "GoOoOgLe", 0, 7); + x2s(".", "a", 0, 1); + ns(".", ""); + x2s("..", "ab", 0, 2); + x2s("\\w", "e", 0, 1); + ns("\\W", "e"); + x2s("\\s", " ", 0, 1); + x2s("\\S", "b", 0, 1); + x2s("\\d", "4", 0, 1); + ns("\\D", "4"); + x2s("\\b", "z ", 0, 0); + x2s("\\b", " z", 1, 1); + x2s("\\B", "zz ", 1, 1); + x2s("\\B", "z ", 2, 2); + x2s("\\B", " z", 0, 0); + x2s("[ab]", "b", 0, 1); + ns("[ab]", "c"); + x2s("[a-z]", "t", 0, 1); + ns("[^a]", "a"); + x2s("[^a]", "\n", 0, 1); + x2s("[]]", "]", 0, 1); + ns("[^]]", "]"); + x2s("[\\^]+", "0^^1", 1, 3); + x2s("[b-]", "b", 0, 1); + x2s("[b-]", "-", 0, 1); + x2s("[\\w]", "z", 0, 1); + ns("[\\w]", " "); + x2s("[\\W]", "b$", 1, 2); + x2s("[\\d]", "5", 0, 1); + ns("[\\d]", "e"); + x2s("[\\D]", "t", 0, 1); + ns("[\\D]", "3"); + x2s("[\\s]", " ", 0, 1); + ns("[\\s]", "a"); + x2s("[\\S]", "b", 0, 1); + ns("[\\S]", " "); + x2s("[\\w\\d]", "2", 0, 1); + ns("[\\w\\d]", " "); + x2s("[[:upper:]]", "B", 0, 1); + x2s("[*[:xdigit:]+]", "+", 0, 1); + x2s("[*[:xdigit:]+]", "GHIKK-9+*", 6, 7); + x2s("[*[:xdigit:]+]", "-@^+", 3, 4); + ns("[[:upper]]", "A"); + x2s("[[:upper]]", ":", 0, 1); + x2s("[\\044-\\047]", "\046", 0, 1); + x2s("[\\x5a-\\x5c]", "\u005b", 0, 1); + x2s("[\\x6A-\\x6D]", "\u006c", 0, 1); + ns("[\\x6A-\\x6D]", "\u006e"); + ns("^[0-9A-F]+ 0+ UNDEF ", "75F 00000000 SECT14A notype () External | _rb_apply"); + x2s("[\\[]", "[", 0, 1); + x2s("[\\]]", "]", 0, 1); + x2s("[&]", "&", 0, 1); + x2s("[[ab]]", "b", 0, 1); + x2s("[[ab]c]", "c", 0, 1); + ns("[[^a]]", "a"); + ns("[^[a]]", "a"); + x2s("[[ab]&&bc]", "b", 0, 1); + ns("[[ab]&&bc]", "a"); + ns("[[ab]&&bc]", "c"); + x2s("[a-z&&b-y&&c-x]", "w", 0, 1); + ns("[^a-z&&b-y&&c-x]", "w"); + x2s("[[^a&&a]&&a-z]", "b", 0, 1); + ns("[[^a&&a]&&a-z]", "a"); + x2s("[[^a-z&&bcdef]&&[^c-g]]", "h", 0, 1); + ns("[[^a-z&&bcdef]&&[^c-g]]", "c"); + x2s("[^[^abc]&&[^cde]]", "c", 0, 1); + x2s("[^[^abc]&&[^cde]]", "e", 0, 1); + ns("[^[^abc]&&[^cde]]", "f"); + x2s("[a-&&-a]", "-", 0, 1); + ns("[a\\-&&\\-a]", "&"); + ns("\\wabc", " abc"); + x2s("a\\Wbc", "a bc", 0, 4); + x2s("a.b.c", "aabbc", 0, 5); + x2s(".\\wb\\W..c", "abb bcc", 0, 7); + x2s("\\s\\wzzz", " zzzz", 0, 5); + x2s("aa.b", "aabb", 0, 4); + ns(".a", "ab"); + x2s(".a", "aa", 0, 2); + x2s("^a", "a", 0, 1); + x2s("^a$", "a", 0, 1); + x2s("^\\w$", "a", 0, 1); + ns("^\\w$", " "); + x2s("^\\wab$", "zab", 0, 3); + x2s("^\\wabcdef$", "zabcdef", 0, 7); + x2s("^\\w...def$", "zabcdef", 0, 7); + x2s("\\w\\w\\s\\Waaa\\d", "aa aaa4", 0, 8); + x2s("\\A\\Z", "", 0, 0); + x2s("\\Axyz", "xyz", 0, 3); + x2s("xyz\\Z", "xyz", 0, 3); + x2s("xyz\\z", "xyz", 0, 3); + x2s("a\\Z", "a", 0, 1); + x2s("\\Gaz", "az", 0, 2); + ns("\\Gz", "bza"); + ns("az\\G", "az"); + ns("az\\A", "az"); + ns("a\\Az", "az"); + x2s("\\^\\$", "^$", 0, 2); + x2s("^x?y", "xy", 0, 2); + x2s("^(x?y)", "xy", 0, 2); + x2s("\\w", "_", 0, 1); + ns("\\W", "_"); + x2s("(?=z)z", "z", 0, 1); + ns("(?=z).", "a"); + x2s("(?!z)a", "a", 0, 1); + ns("(?!z)a", "z"); + x2s("(?i:a)", "a", 0, 1); + x2s("(?i:a)", "A", 0, 1); + x2s("(?i:A)", "a", 0, 1); + ns("(?i:A)", "b"); + x2s("(?i:[A-Z])", "a", 0, 1); + x2s("(?i:[f-m])", "H", 0, 1); + x2s("(?i:[f-m])", "h", 0, 1); + ns("(?i:[f-m])", "e"); + x2s("(?i:[A-c])", "D", 0, 1); + ns("(?i:[^a-z])", "A"); + ns("(?i:[^a-z])", "a"); + x2s("(?i:[!-k])", "Z", 0, 1); + x2s("(?i:[!-k])", "7", 0, 1); + x2s("(?i:[T-}])", "b", 0, 1); + x2s("(?i:[T-}])", "{", 0, 1); + x2s("(?i:\\?a)", "?A", 0, 2); + x2s("(?i:\\*A)", "*a", 0, 2); + ns(".", "\n"); + x2s("(?m:.)", "\n", 0, 1); + x2s("(?m:a.)", "a\n", 0, 2); + x2s("(?m:.b)", "a\nb", 1, 3); + x2s(".*abc", "dddabdd\nddabc", 8, 13); + x2s("(?m:.*abc)", "dddabddabc", 0, 10); + ns("(?i)(?-i)a", "A"); + ns("(?i)(?-i:a)", "A"); + x2s("a?", "", 0, 0); + x2s("a?", "b", 0, 0); + x2s("a?", "a", 0, 1); + x2s("a*", "", 0, 0); + x2s("a*", "a", 0, 1); + x2s("a*", "aaa", 0, 3); + x2s("a*", "baaaa", 0, 0); + ns("a+", ""); + x2s("a+", "a", 0, 1); + x2s("a+", "aaaa", 0, 4); + x2s("a+", "aabbb", 0, 2); + x2s("a+", "baaaa", 1, 5); + x2s(".?", "", 0, 0); + x2s(".?", "f", 0, 1); + x2s(".?", "\n", 0, 0); + x2s(".*", "", 0, 0); + x2s(".*", "abcde", 0, 5); + x2s(".+", "z", 0, 1); + x2s(".+", "zdswer\n", 0, 6); + x2s("(.*)a\\1f", "babfbac", 0, 4); + x2s("(.*)a\\1f", "bacbabf", 3, 7); + x2s("((.*)a\\2f)", "bacbabf", 3, 7); + x2s("(.*)a\\1f", "baczzzzzz\nbazz\nzzzzbabf", 19, 23); + x2s("a|b", "a", 0, 1); + x2s("a|b", "b", 0, 1); + x2s("|a", "a", 0, 0); + x2s("(|a)", "a", 0, 0); + x2s("ab|bc", "ab", 0, 2); + x2s("ab|bc", "bc", 0, 2); + x2s("z(?:ab|bc)", "zbc", 0, 3); + x2s("a(?:ab|bc)c", "aabc", 0, 4); + x2s("ab|(?:ac|az)", "az", 0, 2); + x2s("a|b|c", "dc", 1, 2); + x2s("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "pqr", 0, 2); + ns("a|b|cd|efg|h|ijk|lmn|o|pq|rstuvwx|yz", "mn"); + x2s("a|^z", "ba", 1, 2); + x2s("a|^z", "za", 0, 1); + x2s("a|\\Gz", "bza", 2, 3); + x2s("a|\\Gz", "za", 0, 1); + x2s("a|\\Az", "bza", 2, 3); + x2s("a|\\Az", "za", 0, 1); + x2s("a|b\\Z", "ba", 1, 2); + x2s("a|b\\Z", "b", 0, 1); + x2s("a|b\\z", "ba", 1, 2); + x2s("a|b\\z", "b", 0, 1); + x2s("\\w|\\s", " ", 0, 1); + ns("\\w|\\w", " "); + x2s("\\w|%", "%", 0, 1); + x2s("\\w|[&$]", "&", 0, 1); + x2s("[b-d]|[^e-z]", "a", 0, 1); + x2s("(?:a|[c-f])|bz", "dz", 0, 1); + x2s("(?:a|[c-f])|bz", "bz", 0, 2); + x2s("abc|(?=zz)..f", "zzf", 0, 3); + x2s("abc|(?!zz)..f", "abf", 0, 3); + x2s("(?=za)..a|(?=zz)..a", "zza", 0, 3); + ns("(?>a|abd)c", "abdc"); + x2s("(?>abd|a)c", "abdc", 0, 4); + x2s("a?|b", "a", 0, 1); + x2s("a?|b", "b", 0, 0); + x2s("a?|b", "", 0, 0); + x2s("a*|b", "aa", 0, 2); + x2s("a*|b*", "ba", 0, 0); + x2s("a*|b*", "ab", 0, 1); + x2s("a+|b*", "", 0, 0); + x2s("a+|b*", "bbb", 0, 3); + x2s("a+|b*", "abbb", 0, 1); + ns("a+|b+", ""); + x2s("(a|b)?", "b", 0, 1); + x2s("(a|b)*", "ba", 0, 2); + x2s("(a|b)+", "bab", 0, 3); + x2s("(ab|ca)+", "caabbc", 0, 4); + x2s("(ab|ca)+", "aabca", 1, 5); + x2s("(ab|ca)+", "abzca", 0, 2); + x2s("(a|bab)+", "ababa", 0, 5); + x2s("(a|bab)+", "ba", 1, 2); + x2s("(a|bab)+", "baaaba", 1, 4); + x2s("(?:a|b)(?:a|b)", "ab", 0, 2); + x2s("(?:a*|b*)(?:a*|b*)", "aaabbb", 0, 3); + x2s("(?:a*|b*)(?:a+|b+)", "aaabbb", 0, 6); + x2s("(?:a+|b+){2}", "aaabbb", 0, 6); + x2s("h{0,}", "hhhh", 0, 4); + x2s("(?:a+|b+){1,2}", "aaabbb", 0, 6); + ns("ax{2}*a", "0axxxa1"); + ns("a.{0,2}a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXa0"); + ns("a.{0,2}?a", "0aXXXXa0"); + x2s("^a{2,}?a$", "aaa", 0, 3); + x2s("^[a-z]{2,}?$", "aaa", 0, 3); + x2s("(?:a+|\\Ab*)cc", "cc", 0, 2); + ns("(?:a+|\\Ab*)cc", "abcc"); + x2s("(?:^a+|b+)*c", "aabbbabc", 6, 8); + x2s("(?:^a+|b+)*c", "aabbbbc", 0, 7); + x2s("a|(?i)c", "C", 0, 1); + x2s("(?i)c|a", "C", 0, 1); + x2s("(?i)c|a", "A", 0, 1); + x2s("(?i:c)|a", "C", 0, 1); + ns("(?i:c)|a", "A"); + x2s("[abc]?", "abc", 0, 1); + x2s("[abc]*", "abc", 0, 3); + x2s("[^abc]*", "abc", 0, 0); + ns("[^abc]+", "abc"); + x2s("a??", "aaa", 0, 0); + x2s("ba??b", "bab", 0, 3); + x2s("a*?", "aaa", 0, 0); + x2s("ba*?", "baa", 0, 1); + x2s("ba*?b", "baab", 0, 4); + x2s("a+?", "aaa", 0, 1); + x2s("ba+?", "baa", 0, 2); + x2s("ba+?b", "baab", 0, 4); + x2s("(?:a?)??", "a", 0, 0); + x2s("(?:a??)?", "a", 0, 0); + x2s("(?:a?)+?", "aaa", 0, 1); + x2s("(?:a+)??", "aaa", 0, 0); + x2s("(?:a+)??b", "aaab", 0, 4); + x2s("(?:ab)?{2}", "", 0, 0); + x2s("(?:ab)?{2}", "ababa", 0, 4); + x2s("(?:ab)*{0}", "ababa", 0, 0); + x2s("(?:ab){3,}", "abababab", 0, 8); + ns("(?:ab){3,}", "abab"); + x2s("(?:ab){2,4}", "ababab", 0, 6); + x2s("(?:ab){2,4}", "ababababab", 0, 8); + x2s("(?:ab){2,4}?", "ababababab", 0, 4); + x2s("(?:ab){,}", "ab{,}", 0, 5); + x2s("(?:abc)+?{2}", "abcabcabc", 0, 6); + x2s("(?:X*)(?i:xa)", "XXXa", 0, 4); + x2s("(d+)([^abc]z)", "dddz", 0, 4); + x2s("([^abc]*)([^abc]z)", "dddz", 0, 4); + x2s("(\\w+)(\\wz)", "dddz", 0, 4); + x3s("(a)", "a", 0, 1, 1); + x3s("(ab)", "ab", 0, 2, 1); + x2s("((ab))", "ab", 0, 2); + x3s("((ab))", "ab", 0, 2, 1); + x3s("((ab))", "ab", 0, 2, 2); + x3s("((((((((((((((((((((ab))))))))))))))))))))", "ab", 0, 2, 20); + x3s("(ab)(cd)", "abcd", 0, 2, 1); + x3s("(ab)(cd)", "abcd", 2, 4, 2); + x3s("()(a)bc(def)ghijk", "abcdefghijk", 3, 6, 3); + x3s("(()(a)bc(def)ghijk)", "abcdefghijk", 3, 6, 4); + x2s("(^a)", "a", 0, 1); + x3s("(a)|(a)", "ba", 1, 2, 1); + x3s("(^a)|(a)", "ba", 1, 2, 2); + x3s("(a?)", "aaa", 0, 1, 1); + x3s("(a*)", "aaa", 0, 3, 1); + x3s("(a*)", "", 0, 0, 1); + x3s("(a+)", "aaaaaaa", 0, 7, 1); + x3s("(a+|b*)", "bbbaa", 0, 3, 1); + x3s("(a+|b?)", "bbbaa", 0, 1, 1); + x3s("(abc)?", "abc", 0, 3, 1); + x3s("(abc)*", "abc", 0, 3, 1); + x3s("(abc)+", "abc", 0, 3, 1); + x3s("(xyz|abc)+", "abc", 0, 3, 1); + x3s("([xyz][abc]|abc)+", "abc", 0, 3, 1); + x3s("((?i:abc))", "AbC", 0, 3, 1); + x2s("(abc)(?i:\\1)", "abcABC", 0, 6); + x3s("((?m:a.c))", "a\nc", 0, 3, 1); + x3s("((?=az)a)", "azb", 0, 1, 1); + x3s("abc|(.abd)", "zabd", 0, 4, 1); + x2s("(?:abc)|(ABC)", "abc", 0, 3); + x3s("(?i:(abc))|(zzz)", "ABC", 0, 3, 1); + x3s("a*(.)", "aaaaz", 4, 5, 1); + x3s("a*?(.)", "aaaaz", 0, 1, 1); + x3s("a*?(c)", "aaaac", 4, 5, 1); + x3s("[bcd]a*(.)", "caaaaz", 5, 6, 1); + x3s("(\\Abb)cc", "bbcc", 0, 2, 1); + ns("(\\Abb)cc", "zbbcc"); + x3s("(^bb)cc", "bbcc", 0, 2, 1); + ns("(^bb)cc", "zbbcc"); + x3s("cc(bb$)", "ccbb", 2, 4, 1); + ns("cc(bb$)", "ccbbb"); + ns("(\\1)", ""); + ns("\\1(a)", "aa"); + ns("(a(b)\\1)\\2+", "ababb"); + ns("(?:(?:\\1|z)(a))+$", "zaa"); + x2s("(?:(?:\\1|z)(a))+$", "zaaa", 0, 4); + x2s("(a)(?=\\1)", "aa", 0, 1); + ns("(a)$|\\1", "az"); + x2s("(a)\\1", "aa", 0, 2); + ns("(a)\\1", "ab"); + x2s("(a?)\\1", "aa", 0, 2); + x2s("(a??)\\1", "aa", 0, 0); + x2s("(a*)\\1", "aaaaa", 0, 4); + x3s("(a*)\\1", "aaaaa", 0, 2, 1); + x2s("a(b*)\\1", "abbbb", 0, 5); + x2s("a(b*)\\1", "ab", 0, 1); + x2s("(a*)(b*)\\1\\2", "aaabbaaabb", 0, 10); + x2s("(a*)(b*)\\2", "aaabbbb", 0, 7); + x2s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 8); + x3s("(((((((a*)b))))))c\\7", "aaabcaaa", 0, 3, 7); + x2s("(a)(b)(c)\\2\\1\\3", "abcbac", 0, 6); + x2s("([a-d])\\1", "cc", 0, 2); + x2s("(\\w\\d\\s)\\1", "f5 f5 ", 0, 6); + ns("(\\w\\d\\s)\\1", "f5 f5"); + x2s("(who|[a-c]{3})\\1", "whowho", 0, 6); + x2s("...(who|[a-c]{3})\\1", "abcwhowho", 0, 9); + x2s("(who|[a-c]{3})\\1", "cbccbc", 0, 6); + x2s("(^a)\\1", "aa", 0, 2); + ns("(^a)\\1", "baa"); + ns("(a$)\\1", "aa"); + ns("(ab\\Z)\\1", "ab"); + x2s("(a*\\Z)\\1", "a", 1, 1); + x2s(".(a*\\Z)\\1", "ba", 1, 2); + x3s("(.(abc)\\2)", "zabcabc", 0, 7, 1); + x3s("(.(..\\d.)\\2)", "z12341234", 0, 9, 1); + x2s("((?i:az))\\1", "AzAz", 0, 4); + ns("((?i:az))\\1", "Azaz"); + x2s("(?<=a)b", "ab", 1, 2); + ns("(?<=a)b", "bb"); + x2s("(?<=a|b)b", "bb", 1, 2); + x2s("(?<=a|bc)b", "bcb", 2, 3); + x2s("(?<=a|bc)b", "ab", 1, 2); + x2s("(?<=a|bc||defghij|klmnopq|r)z", "rz", 1, 2); + x2s("(a)\\g<1>", "aa", 0, 2); + x2s("(?a)", "a", 0, 1); + x2s("(?ab)\\g", "abab", 0, 4); + x2s("(?.zv.)\\k", "azvbazvb", 0, 8); + x2s("(?<=\\g)|-\\zEND (?XyZ)", "XyZ", 3, 3); + x2s("(?|a\\g)+", "", 0, 0); + x2s("(?|\\(\\g\\))+$", "()(())", 0, 6); + x3s("\\g(?.){0}", "X", 0, 1, 1); + x2s("\\g(abc|df(?.YZ){2,8}){0}", "XYZ", 0, 3); + x2s("\\A(?(a\\g)|)\\z", "aaaa", 0, 4); + x2s("(?|\\g\\g)\\z|\\zEND (?a|(b)\\g)", "bbbbabba", 0, 8); + x2s("(?\\w+\\sx)a+\\k", " fg xaaaaaaaafg x", 2, 18); + x3s("(z)()()(?<_9>a)\\g<_9>", "zaa", 2, 3, 1); + x2s("(.)(((?<_>a)))\\k<_>", "zaa", 0, 3); + x2s("((?\\d)|(?\\w))(\\k|\\k)", "ff", 0, 2); + x2s("(?:(?)|(?efg))\\k", "", 0, 0); + x2s("(?:(?abc)|(?efg))\\k", "abcefgefg", 3, 9); + ns("(?:(?abc)|(?efg))\\k", "abcefg"); + x2s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "a-pyumpyum", 2, 10); + x3s("(?:(?.)|(?..)|(?...)|(?....)|(?.....)|(?......)|(?.......)|(?........)|(?.........)|(?..........)|(?...........)|(?............)|(?.............)|(?..............))\\k$", "xxxxabcdefghijklmnabcdefghijklmn", 4, 18, 14); + x3s("(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?)(?aaa)(?)$", "aaa", 0, 3, 16); + x2s("(?a|\\(\\g\\))", "a", 0, 1); + x2s("(?a|\\(\\g\\))", "((((((a))))))", 0, 13); + x3s("(?a|\\(\\g\\))", "((((((((a))))))))", 0, 17, 1); + x2s("\\g|\\zEND(?.*abc$)", "abcxxxabc", 0, 9); + x2s("\\g<1>|\\zEND(.a.)", "bac", 0, 3); + x3s("\\g<_A>\\g<_A>|\\zEND(.a.)(?<_A>.b.)", "xbxyby", 3, 6, 1); + x2s("\\A(?:\\g|\\g|\\zEND (?a|c\\gc)(?b|d\\gd))$", "cdcbcdc", 0, 7); + x2s("\\A(?|a\\g)\\z|\\zEND (?\\g)", "aaaa", 0, 4); + x2s("(?(a|b\\gc){3,5})", "baaaaca", 1, 5); + x2s("(?(a|b\\gc){3,5})", "baaaacaaaaa", 0, 10); + x2s("(?\\(([^\\(\\)]++|\\g)*+\\))", "((a))", 0, 5); + x2s("()*\\1", "", 0, 0); + x2s("(?:()|())*\\1\\2", "", 0, 0); + x3s("(?:\\1a|())*", "a", 0, 0, 1); + x2s("x((.)*)*x", "0x1x2x3", 1, 6); + x2s("x((.)*)*x(?i:\\1)\\Z", "0x1x2x1X2", 1, 9); + x2s("(?:()|()|()|()|()|())*\\2\\5", "", 0, 0); + x2s("(?:()|()|()|(x)|()|())*\\2b\\5", "b", 0, 1); + x2s("\\xED\\xF2", "\u00ed\u0148", 0, 2); + x2s("", "\u00a4\u02d8", 0, 0); + x2s("\u00a4\u02d8", "\u00a4\u02d8", 0, 2); + ns("\u00a4\u00a4", "\u00a4\u02d8"); + x2s("\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a6\u00a4\u00a6", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6); + x2s("\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", "\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", 0, 70); + x2s("\u00a4\u02d8", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x2s("\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 2, 6); + x2s("\\xca\\xb8", "\u0118\u00b8", 0, 2); + x2s(".", "\u00a4\u02d8", 0, 2); + x2s("..", "\u00a4\u00ab\u00a4\u00ad", 0, 4); + x2s("\\w", "\u00a4\u015e", 0, 2); + ns("\\W", "\u00a4\u02d8"); + x2s("[\\W]", "\u00a4\u00a6$", 2, 3); + x2s("\\S", "\u00a4\u02dd", 0, 2); + x2s("\\S", "\u00b4\u00c1", 0, 2); + x2s("\\b", "\u00b5\u00a4 ", 0, 0); + x2s("\\b", " \u00a4\u0170", 1, 1); + x2s("\\B", "\u00a4\u00bb\u00a4\u02dd ", 2, 2); + x2s("\\B", "\u00a4\u00a6 ", 3, 3); + x2s("\\B", " \u00a4\u00a4", 0, 0); + x2s("[\u00a4\u017c\u00a4\u00c1]", "\u00a4\u00c1", 0, 2); + ns("[\u00a4\u0118\u00a4\u00cb]", "\u00a4\u011a"); + x2s("[\u00a4\u00a6-\u00a4\u015e]", "\u00a4\u00a8", 0, 2); + ns("[^\u00a4\u00b1]", "\u00a4\u00b1"); + x2s("[\\w]", "\u00a4\u00cd", 0, 2); + ns("[\\d]", "\u00a4\u0150"); + x2s("[\\D]", "\u00a4\u010e", 0, 2); + ns("[\\s]", "\u00a4\u017b"); + x2s("[\\S]", "\u00a4\u0158", 0, 2); + x2s("[\\w\\d]", "\u00a4\u010d", 0, 2); + x2s("[\\w\\d]", " \u00a4\u010d", 3, 5); + ns("\\w\u00b5\u00b4\u013d\u00d6", " \u00b5\u00b4\u013d\u00d6"); + x2s("\u00b5\u00b4\\W\u013d\u00d6", "\u00b5\u00b4 \u013d\u00d6", 0, 5); + x2s("\u00a4\u02d8.\u00a4\u00a4.\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6", 0, 10); + x2s(".\\w\u00a4\u00a6\\W..\u00a4\u013e", "\u00a4\u00a8\u00a4\u00a6\u00a4\u00a6 \u00a4\u00a6\u00a4\u013e\u00a4\u013e", 0, 13); + x2s("\\s\\w\u00a4\u0142\u00a4\u0142\u00a4\u0142", " \u00a4\u0142\u00a4\u0142\u00a4\u0142\u00a4\u0142", 0, 9); + x2s("\u00a4\u02d8\u00a4\u02d8.\u00a4\u00b1", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00b1\u00a4\u00b1", 0, 8); + ns(".\u00a4\u00a4", "\u00a4\u00a4\u00a4\u00a8"); + x2s(".\u00a4\u015e", "\u00a4\u015e\u00a4\u015e", 0, 4); + x2s("^\u00a4\u02d8", "\u00a4\u02d8", 0, 2); + x2s("^\u00a4\u0155$", "\u00a4\u0155", 0, 2); + x2s("^\\w$", "\u00a4\u00cb", 0, 2); + x2s("^\\w\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142$", "z\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 0, 11); + x2s("^\\w...\u00a4\u00a6\u00a4\u00a8\u00a4\u015e$", "z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6\u00a4\u00a8\u00a4\u015e", 0, 13); + x2s("\\w\\w\\s\\W\u00a4\u015e\u00a4\u015e\u00a4\u015e\\d", "a\u00a4\u015e \u00a4\u015e\u00a4\u015e\u00a4\u015e4", 0, 12); + x2s("\\A\u00a4\u017c\u00a4\u00c1\u00a4\u00c4", "\u00a4\u017c\u00a4\u00c1\u00a4\u00c4", 0, 6); + x2s("\u00a4\u0155\u00a4\u00e1\u00a4\u00e2\\Z", "\u00a4\u0155\u00a4\u00e1\u00a4\u00e2", 0, 6); + x2s("\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\\z", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 6); + x2s("\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\\Z", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\n", 0, 6); + x2s("\\G\u00a4\u00dd\u00a4\u00d4", "\u00a4\u00dd\u00a4\u00d4", 0, 4); + ns("\\G\u00a4\u00a8", "\u00a4\u00a6\u00a4\u00a8\u00a4\u015e"); + ns("\u00a4\u010c\u00a4\u0106\\G", "\u00a4\u010c\u00a4\u0106"); + ns("\u00a4\u0162\u00a4\u00df\\A", "\u00a4\u0162\u00a4\u00df"); + ns("\u00a4\u0162\\A\u00a4\u00df", "\u00a4\u0162\u00a4\u00df"); + x2s("(?=\u00a4\u00bb)\u00a4\u00bb", "\u00a4\u00bb", 0, 2); + ns("(?=\u00a4\u00a6).", "\u00a4\u00a4"); + x2s("(?!\u00a4\u00a6)\u00a4\u00ab", "\u00a4\u00ab", 0, 2); + ns("(?!\u00a4\u010c)\u00a4\u02d8", "\u00a4\u010c"); + x2s("(?i:\u00a4\u02d8)", "\u00a4\u02d8", 0, 2); + x2s("(?i:\u00a4\u00d6\u00a4\u016e)", "\u00a4\u00d6\u00a4\u016e", 0, 4); + ns("(?i:\u00a4\u00a4)", "\u00a4\u00a6"); + x2s("(?m:\u00a4\u010d.)", "\u00a4\u010d\n", 0, 3); + x2s("(?m:.\u00a4\u00e1)", "\u00a4\u0162\n\u00a4\u00e1", 2, 5); + x2s("\u00a4\u02d8?", "", 0, 0); + x2s("\u0118\u0143?", "\u02db\u02dd", 0, 0); + x2s("\u0118\u0143?", "\u0118\u0143", 0, 2); + x2s("\u00ce\u011a*", "", 0, 0); + x2s("\u00ce\u011a*", "\u00ce\u011a", 0, 2); + x2s("\u00bb\u0147*", "\u00bb\u0147\u00bb\u0147\u00bb\u0147", 0, 6); + x2s("\u00c7\u010e*", "\u013d\u017b\u00c7\u010e\u00c7\u010e\u00c7\u010e\u00c7\u010e", 0, 0); + ns("\u00bb\u0142+", ""); + x2s("\u02db\u010e+", "\u02db\u010e", 0, 2); + x2s("\u00bb\u0163+", "\u00bb\u0163\u00bb\u0163\u00bb\u0163\u00bb\u0163", 0, 8); + x2s("\u00a4\u00a8+", "\u00a4\u00a8\u00a4\u00a8\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 0, 4); + x2s("\u00a4\u00a6+", "\u00a4\u015e\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 2, 10); + x2s(".?", "\u00a4\u017c", 0, 2); + x2s(".*", "\u00a4\u0143\u00a4\u00d4\u00a4\u00d7\u00a4\u00da", 0, 8); + x2s(".+", "\u00a4\u00ed", 0, 2); + x2s(".+", "\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u00ab\n", 0, 8); + x2s("\u00a4\u02d8|\u00a4\u00a4", "\u00a4\u02d8", 0, 2); + x2s("\u00a4\u02d8|\u00a4\u00a4", "\u00a4\u00a4", 0, 2); + x2s("\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a4\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a4\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a6", 0, 4); + x2s("\u00a4\u0148(?:\u00a4\u00ab\u00a4\u00ad|\u00a4\u00ad\u00a4\u017b)", "\u00a4\u0148\u00a4\u00ab\u00a4\u00ad", 0, 6); + x2s("\u00a4\u0148(?:\u00a4\u00ab\u00a4\u00ad|\u00a4\u00ad\u00a4\u017b)\u00a4\u00b1", "\u00a4\u0148\u00a4\u00ad\u00a4\u017b\u00a4\u00b1", 0, 8); + x2s("\u00a4\u02d8\u00a4\u00a4|(?:\u00a4\u02d8\u00a4\u00a6|\u00a4\u02d8\u00a4\u0148)", "\u00a4\u02d8\u00a4\u0148", 0, 4); + x2s("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6", "\u00a4\u00a8\u00a4\u00a6", 2, 4); + x2s("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8|\u00a4\u015e\u00a4\u00ab\u00a4\u00ad|\u00a4\u017b|\u00a4\u00b1\u00a4\u0142\u00a4\u00b5|\u00a4\u00b7\u00a4\u0105\u00a4\u00bb|\u00a4\u02dd|\u00a4\u017c\u00a4\u00c1|\u00a4\u00c4\u00a4\u0106\u00a4\u010c\u00a4\u0118\u00a4\u00cb|\u00a4\u011a\u00a4\u00cd", "\u00a4\u00b7\u00a4\u0105\u00a4\u00bb", 0, 6); + ns("\u00a4\u02d8|\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8|\u00a4\u015e\u00a4\u00ab\u00a4\u00ad|\u00a4\u017b|\u00a4\u00b1\u00a4\u0142\u00a4\u00b5|\u00a4\u00b7\u00a4\u0105\u00a4\u00bb|\u00a4\u02dd|\u00a4\u017c\u00a4\u00c1|\u00a4\u00c4\u00a4\u0106\u00a4\u010c\u00a4\u0118\u00a4\u00cb|\u00a4\u011a\u00a4\u00cd", "\u00a4\u0105\u00a4\u00bb"); + x2s("\u00a4\u02d8|^\u00a4\u010f", "\u00a4\u00d6\u00a4\u02d8", 2, 4); + x2s("\u00a4\u02d8|^\u00a4\u0148", "\u00a4\u0148\u00a4\u02d8", 0, 2); + x2s("\u00b5\u00b4|\\G\u013d\u00d6", "\u00a4\u00b1\u013d\u00d6\u00b5\u00b4", 4, 6); + x2s("\u00b5\u00b4|\\G\u013d\u00d6", "\u013d\u00d6\u00b5\u00b4", 0, 2); + x2s("\u00b5\u00b4|\\A\u013d\u00d6", "b\u013d\u00d6\u00b5\u00b4", 3, 5); + x2s("\u00b5\u00b4|\\A\u013d\u00d6", "\u013d\u00d6", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6\u00b5\u00b4", 2, 4); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\Z", "\u013d\u00d6\n", 0, 2); + x2s("\u00b5\u00b4|\u013d\u00d6\\z", "\u013d\u00d6\u00b5\u00b4", 2, 4); + x2s("\u00b5\u00b4|\u013d\u00d6\\z", "\u013d\u00d6", 0, 2); + x2s("\\w|\\s", "\u00a4\u015e", 0, 2); + x2s("\\w|%", "%\u00a4\u015e", 0, 1); + x2s("\\w|[&$]", "\u00a4\u00a6&", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]", "\u00a4\u00a6", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]|[^\u00a4\u00ab-\u00a4\u0142]", "\u00a4\u02d8", 0, 2); + x2s("[\u00a4\u00a4-\u00a4\u00b1]|[^\u00a4\u00ab-\u00a4\u0142]", "\u00a4\u00ab", 0, 2); + x2s("[^\u00a4\u02d8]", "\n", 0, 1); + x2s("(?:\u00a4\u02d8|[\u00a4\u00a6-\u00a4\u00ad])|\u00a4\u00a4\u00a4\u0148", "\u00a4\u00a6\u00a4\u0148", 0, 2); + x2s("(?:\u00a4\u02d8|[\u00a4\u00a6-\u00a4\u00ad])|\u00a4\u00a4\u00a4\u0148", "\u00a4\u00a4\u00a4\u0148", 0, 4); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(?=\u00a4\u00b1\u00a4\u00b1)..\u00a4\u0170", "\u00a4\u00b1\u00a4\u00b1\u00a4\u0170", 0, 6); + x2s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(?!\u00a4\u00b1\u00a4\u00b1)..\u00a4\u0170", "\u00a4\u02d8\u00a4\u00a4\u00a4\u0170", 0, 6); + // x2s("(?=\u00a4\u0148\u00a4\u02d8)..\u00a4\u02d8|(?=\u00a4\u0148\u00a4\u0148)..\u00a4\u02d8", "\u00a4\u0148\u00a4\u0148\u00a4\u02d8", 0, 6); + x2s("(?<=\u00a4\u02d8|\u00a4\u00a4\u00a4\u00a6)\u00a4\u00a4", "\u00a4\u00a4\u00a4\u00a6\u00a4\u00a4", 4, 6); + ns("(?>\u00a4\u02d8|\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8)\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8\u00a4\u00a6"); + x2s("(?>\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8|\u00a4\u02d8)\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8\u00a4\u00a6", 0, 8); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "\u00a4\u02d8", 0, 2); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "\u00a4\u00a4", 0, 0); + x2s("\u00a4\u02d8?|\u00a4\u00a4", "", 0, 0); + x2s("\u00a4\u02d8*|\u00a4\u00a4", "\u00a4\u02d8\u00a4\u02d8", 0, 4); + x2s("\u00a4\u02d8*|\u00a4\u00a4*", "\u00a4\u00a4\u00a4\u02d8", 0, 0); + x2s("\u00a4\u02d8*|\u00a4\u00a4*", "\u00a4\u02d8\u00a4\u00a4", 0, 2); + x2s("[a\u00a4\u02d8]*|\u00a4\u00a4*", "a\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 3); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "", 0, 0); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 6); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 2); + x2s("\u00a4\u02d8+|\u00a4\u00a4*", "a\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 0); + ns("\u00a4\u02d8+|\u00a4\u00a4+", ""); + x2s("(\u00a4\u02d8|\u00a4\u00a4)?", "\u00a4\u00a4", 0, 2); + x2s("(\u00a4\u02d8|\u00a4\u00a4)*", "\u00a4\u00a4\u00a4\u02d8", 0, 4); + x2s("(\u00a4\u02d8|\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4", 0, 6); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u00a6\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 0, 8); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u00a8)+", "\u00a4\u00a6\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 4, 12); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u02d8", 2, 10); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u0148\u00a4\u00a6\u00a4\u02d8", 0, 4); + x2s("(\u00a4\u02d8\u00a4\u00a4|\u00a4\u00a6\u00a4\u02d8)+", "$$zzzz\u00a4\u02d8\u00a4\u00a4\u00a4\u0148\u00a4\u00a6\u00a4\u02d8", 6, 10); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8", 0, 10); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x2s("(\u00a4\u02d8|\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4)+", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u02d8", 2, 8); + x2s("(?:\u00a4\u02d8|\u00a4\u00a4)(?:\u00a4\u02d8|\u00a4\u00a4)", "\u00a4\u02d8\u00a4\u00a4", 0, 4); + x2s("(?:\u00a4\u02d8*|\u00a4\u00a4*)(?:\u00a4\u02d8*|\u00a4\u00a4*)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 6); + x2s("(?:\u00a4\u02d8*|\u00a4\u00a4*)(?:\u00a4\u02d8+|\u00a4\u00a4+)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\u00a4\u00a4+){2}", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\u00a4\u00a4+){1,2}", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 12); + x2s("(?:\u00a4\u02d8+|\\A\u00a4\u00a4*)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a6\u00a4\u00a6", 0, 4); + ns("(?:\u00a4\u02d8+|\\A\u00a4\u00a4*)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x2s("(?:^\u00a4\u02d8+|\u00a4\u00a4+)*\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 12, 16); + x2s("(?:^\u00a4\u02d8+|\u00a4\u00a4+)*\u00a4\u00a6", "\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6", 0, 14); + x2s("\u00a4\u00a6{0,}", "\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6\u00a4\u00a6", 0, 8); + x2s("\u00a4\u02d8|(?i)c", "C", 0, 1); + x2s("(?i)c|\u00a4\u02d8", "C", 0, 1); + x2s("(?i:\u00a4\u02d8)|a", "a", 0, 1); + ns("(?i:\u00a4\u02d8)|a", "A"); + x2s("[\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]?", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 2); + x2s("[\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6); + x2s("[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 0); + ns("[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6"); + x2s("\u00a4\u02d8??", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 0); + x2s("\u00a4\u00a4\u00a4\u02d8??\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u00a4", 0, 6); + x2s("\u00a4\u02d8*?", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 0); + x2s("\u00a4\u00a4\u00a4\u02d8*?", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 2); + x2s("\u00a4\u00a4\u00a4\u02d8*?\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4", 0, 8); + x2s("\u00a4\u02d8+?", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 2); + x2s("\u00a4\u00a4\u00a4\u02d8+?", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 4); + x2s("\u00a4\u00a4\u00a4\u02d8+?\u00a4\u00a4", "\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4", 0, 8); + x2s("(?:\u0139\u00b7?)??", "\u0139\u00b7", 0, 0); + x2s("(?:\u0139\u00b7??)?", "\u0139\u00b7", 0, 0); + x2s("(?:\u011a\u00b4?)+?", "\u011a\u00b4\u011a\u00b4\u011a\u00b4", 0, 2); + x2s("(?:\u00c9\u00f7+)??", "\u00c9\u00f7\u00c9\u00f7\u00c9\u00f7", 0, 0); + x2s("(?:\u0154\u0103+)??\u00c1\u00fa", "\u0154\u0103\u0154\u0103\u0154\u0103\u00c1\u00fa", 0, 8); + x2s("(?:\u00a4\u02d8\u00a4\u00a4)?{2}", "", 0, 0); + x2s("(?:\u00b5\u00b4\u013d\u00d6)?{2}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4", 0, 8); + x2s("(?:\u00b5\u00b4\u013d\u00d6)*{0}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4", 0, 0); + x2s("(?:\u00b5\u00b4\u013d\u00d6){3,}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 16); + ns("(?:\u00b5\u00b4\u013d\u00d6){3,}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6"); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 12); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 16); + x2s("(?:\u00b5\u00b4\u013d\u00d6){2,4}?", "\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6\u00b5\u00b4\u013d\u00d6", 0, 8); + x2s("(?:\u00b5\u00b4\u013d\u00d6){,}", "\u00b5\u00b4\u013d\u00d6{,}", 0, 7); + x2s("(?:\u00a4\u00ab\u00a4\u00ad\u00a4\u017b)+?{2}", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 12); + x3s("(\u02db\u0110)", "\u02db\u0110", 0, 2, 1); + x3s("(\u02db\u0110\u017c\u013a)", "\u02db\u0110\u017c\u013a", 0, 4, 1); + x2s("((\u00bb\u0163\u00b4\u00d6))", "\u00bb\u0163\u00b4\u00d6", 0, 4); + x3s("((\u00c9\u00f7\u017c\u013a))", "\u00c9\u00f7\u017c\u013a", 0, 4, 1); + x3s("((\u015f\u0148\u0106\u00fc))", "\u015f\u0148\u0106\u00fc", 0, 4, 2); + x3s("((((((((((((((((((((\u00ce\u011a\u00bb\u0147))))))))))))))))))))", "\u00ce\u011a\u00bb\u0147", 0, 4, 20); + x3s("(\u00a4\u02d8\u00a4\u00a4)(\u00a4\u00a6\u00a4\u00a8)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 0, 4, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4)(\u00a4\u00a6\u00a4\u00a8)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8", 4, 8, 2); + x3s("()(\u00a4\u02d8)\u00a4\u00a4\u00a4\u00a6(\u00a4\u00a8\u00a4\u015e\u00a4\u00ab)\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 6, 12, 3); + x3s("(()(\u00a4\u02d8)\u00a4\u00a4\u00a4\u00a6(\u00a4\u00a8\u00a4\u015e\u00a4\u00ab)\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142)", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e\u00a4\u00ab\u00a4\u00ad\u00a4\u017b\u00a4\u00b1\u00a4\u0142", 6, 12, 4); + x3s(".*(\u0104\u0150\u0104\u00a9)\u0104\u00f3\u02c7\u00a6\u0104\u0162(\u0104\u00f3()\u0104\u00b7\u0104\u013a\u0104\u017c)\u0104\u00a4\u0104\u00f3", "\u0104\u0150\u0104\u00a9\u0104\u00f3\u02c7\u00a6\u0104\u0162\u0104\u00f3\u0104\u00b7\u0104\u013a\u0104\u017c\u0104\u00a4\u0104\u00f3", 10, 18, 2); + x2s("(^\u00a4\u02d8)", "\u00a4\u02d8", 0, 2); + x3s("(\u00a4\u02d8)|(\u00a4\u02d8)", "\u00a4\u00a4\u00a4\u02d8", 2, 4, 1); + x3s("(^\u00a4\u02d8)|(\u00a4\u02d8)", "\u00a4\u00a4\u00a4\u02d8", 2, 4, 2); + x3s("(\u00a4\u02d8?)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8", 0, 2, 1); + x3s("(\u00a4\u0162*)", "\u00a4\u0162\u00a4\u0162\u00a4\u0162", 0, 6, 1); + x3s("(\u00a4\u010c*)", "", 0, 0, 1); + x3s("(\u00a4\u00eb+)", "\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb", 0, 14, 1); + x3s("(\u00a4\u0150+|\u00a4\u0158*)", "\u00a4\u0150\u00a4\u0150\u00a4\u0150\u00a4\u0158\u00a4\u0158", 0, 6, 1); + x3s("(\u00a4\u02d8+|\u00a4\u00a4?)", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8", 0, 2, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)?", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)*", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("(\u00a4\u00b5\u00a4\u00b7\u00a4\u0105|\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6)+", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("([\u00a4\u0118\u00a4\u00cb\u00a4\u011a][\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]|\u00a4\u00ab\u00a4\u00ad\u00a4\u017b)+", "\u00a4\u00ab\u00a4\u00ad\u00a4\u017b", 0, 6, 1); + x3s("((?i:\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6))", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6", 0, 6, 1); + x3s("((?m:\u00a4\u02d8.\u00a4\u00a6))", "\u00a4\u02d8\n\u00a4\u00a6", 0, 5, 1); + x3s("((?=\u00a4\u02d8\u00a4\u00f3)\u00a4\u02d8)", "\u00a4\u02d8\u00a4\u00f3\u00a4\u00a4", 0, 2, 1); + x3s("\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6|(.\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8)", "\u00a4\u00f3\u00a4\u02d8\u00a4\u00a4\u00a4\u00a8", 0, 8, 1); + x3s("\u00a4\u02d8*(.)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 8, 10, 1); + x3s("\u00a4\u02d8*?(.)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 0, 2, 1); + x3s("\u00a4\u02d8*?(\u00a4\u00f3)", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 8, 10, 1); + x3s("[\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8]\u00a4\u02d8*(.)", "\u00a4\u00a8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00f3", 10, 12, 1); + x3s("(\\A\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6", 0, 4, 1); + ns("(\\A\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00f3\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x3s("(^\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6", 0, 4, 1); + ns("(^\u00a4\u00a4\u00a4\u00a4)\u00a4\u00a6\u00a4\u00a6", "\u00a4\u00f3\u00a4\u00a4\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6"); + x3s("\u00a4\u00ed\u00a4\u00ed(\u00a4\u00eb\u00a4\u00eb$)", "\u00a4\u00ed\u00a4\u00ed\u00a4\u00eb\u00a4\u00eb", 4, 8, 1); + ns("\u00a4\u00ed\u00a4\u00ed(\u00a4\u00eb\u00a4\u00eb$)", "\u00a4\u00ed\u00a4\u00ed\u00a4\u00eb\u00a4\u00eb\u00a4\u00eb"); + x2s("(\u011a\u00b5)\\1", "\u011a\u00b5\u011a\u00b5", 0, 4); + ns("(\u011a\u00b5)\\1", "\u011a\u00b5\u00c9\u0111"); + x2s("(\u00b6\u0151?)\\1", "\u00b6\u0151\u00b6\u0151", 0, 4); + x2s("(\u00b6\u0151??)\\1", "\u00b6\u0151\u00b6\u0151", 0, 0); + x2s("(\u00b6\u0151*)\\1", "\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151", 0, 8); + x3s("(\u00b6\u0151*)\\1", "\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151\u00b6\u0151", 0, 4, 1); + x2s("\u00a4\u02d8(\u00a4\u00a4*)\\1", "\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 10); + x2s("\u00a4\u02d8(\u00a4\u00a4*)\\1", "\u00a4\u02d8\u00a4\u00a4", 0, 2); + x2s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\1\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4", 0, 20); + x2s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 0, 14); + x3s("(\u00a4\u02d8*)(\u00a4\u00a4*)\\2", "\u00a4\u02d8\u00a4\u02d8\u00a4\u02d8\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4\u00a4", 6, 10, 2); + x2s("(((((((\u00a4\u00dd*)\u00a4\u00da))))))\u00a4\u00d4\\7", "\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd\u00a4\u00da\u00a4\u00d4\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd", 0, 16); + x3s("(((((((\u00a4\u00dd*)\u00a4\u00da))))))\u00a4\u00d4\\7", "\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd\u00a4\u00da\u00a4\u00d4\u00a4\u00dd\u00a4\u00dd\u00a4\u00dd", 0, 6, 7); + x2s("(\u00a4\u010e)(\u00a4\u0147)(\u00a4\u0150)\\2\\1\\3", "\u00a4\u010e\u00a4\u0147\u00a4\u0150\u00a4\u0147\u00a4\u010e\u00a4\u0150", 0, 12); + x2s("([\u00a4\u00ad-\u00a4\u00b1])\\1", "\u00a4\u017b\u00a4\u017b", 0, 4); + x2s("(\\w\\d\\s)\\1", "\u00a4\u02d85 \u00a4\u02d85 ", 0, 8); + ns("(\\w\\d\\s)\\1", "\u00a4\u02d85 \u00a4\u02d85"); + x2s("(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u0102\u017b\u02c7\u00a9\u0102\u017b\u02c7\u00a9", 0, 8); + x2s("...(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u00a4\u02d8a\u00a4\u02d8\u0102\u017b\u02c7\u00a9\u0102\u017b\u02c7\u00a9", 0, 13); + x2s("(\u0102\u017b\u02c7\u00a9|[\u00a4\u02d8-\u00a4\u00a6]{3})\\1", "\u00a4\u00a6\u00a4\u00a4\u00a4\u00a6\u00a4\u00a6\u00a4\u00a4\u00a4\u00a6", 0, 12); + x2s("(^\u00a4\u0142)\\1", "\u00a4\u0142\u00a4\u0142", 0, 4); + ns("(^\u00a4\u0155)\\1", "\u00a4\u00e1\u00a4\u0155\u00a4\u0155"); + ns("(\u00a4\u02d8$)\\1", "\u00a4\u02d8\u00a4\u02d8"); + ns("(\u00a4\u02d8\u00a4\u00a4\\Z)\\1", "\u00a4\u02d8\u00a4\u00a4"); + x2s("(\u00a4\u02d8*\\Z)\\1", "\u00a4\u02d8", 2, 2); + x2s(".(\u00a4\u02d8*\\Z)\\1", "\u00a4\u00a4\u00a4\u02d8", 2, 4); + x3s("(.(\u00a4\u00e4\u00a4\u00a4\u00a4\u0107)\\2)", "z\u00a4\u00e4\u00a4\u00a4\u00a4\u0107\u00a4\u00e4\u00a4\u00a4\u00a4\u0107", 0, 13, 1); + x3s("(.(..\\d.)\\2)", "\u00a4\u02d812341234", 0, 10, 1); + x2s("((?i:\u00a4\u02d8v\u00a4\u015f))\\1", "\u00a4\u02d8v\u00a4\u015f\u00a4\u02d8v\u00a4\u015f", 0, 10); + x2s("(?<\u00b6\u0148\u00a4\u00ab>\u0118\u0143|\\(\\g<\u00b6\u0148\u00a4\u00ab>\\))", "((((((\u0118\u0143))))))", 0, 14); + x2s("\\A(?:\\g<\u00b0\u00a4_1>|\\g<\u00b1\u013e_2>|\\z\u02dd\u015e\u00ce\u00bb (?<\u00b0\u00a4_1>\u00b4\u0143|\u013d\u00ab\\g<\u00b1\u013e_2>\u013d\u00ab)(?<\u00b1\u013e_2>\u015f\u00df|\u0118\u00ee\u00bb\u00a7\\g<\u00b0\u00a4_1>\u0118\u00ee\u00bb\u00a7))$", "\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u015f\u00df\u013d\u00ab\u0118\u00ee\u00bb\u00a7\u013d\u00ab\u0118\u00ee\u00bb\u00a7", 0, 26); + x2s("[[\u00a4\u0147\u00a4\u0150]]", "\u00a4\u0150", 0, 2); + x2s("[[\u00a4\u00a4\u00a4\u015e\u00a4\u00a6]\u00a4\u00ab]", "\u00a4\u00ab", 0, 2); + ns("[[^\u00a4\u02d8]]", "\u00a4\u02d8"); + ns("[^[\u00a4\u02d8]]", "\u00a4\u02d8"); + x2s("[^[^\u00a4\u02d8]]", "\u00a4\u02d8", 0, 2); + x2s("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u017b", 0, 2); + ns("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u00ab"); + ns("[[\u00a4\u00ab\u00a4\u00ad\u00a4\u017b]&&\u00a4\u00ad\u00a4\u017b]", "\u00a4\u00b1"); + x2s("[\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4-\u00a4\u0148&&\u00a4\u00a6-\u00a4\u0144]", "\u00a4\u0144", 0, 2); + ns("[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4-\u00a4\u0148&&\u00a4\u00a6-\u00a4\u0144]", "\u00a4\u0144"); + x2s("[[^\u00a4\u02d8&&\u00a4\u02d8]&&\u00a4\u02d8-\u00a4\u00f3]", "\u00a4\u00a4", 0, 2); + ns("[[^\u00a4\u02d8&&\u00a4\u02d8]&&\u00a4\u02d8-\u00a4\u00f3]", "\u00a4\u02d8"); + x2s("[[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]&&[^\u00a4\u00a6-\u00a4\u00ab]]", "\u00a4\u00ad", 0, 2); + ns("[[^\u00a4\u02d8-\u00a4\u00f3&&\u00a4\u00a4\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]&&[^\u00a4\u00a6-\u00a4\u00ab]]", "\u00a4\u00a4"); + x2s("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00a6", 0, 2); + x2s("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00a8", 0, 2); + ns("[^[^\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]]", "\u00a4\u00ab"); + x2s("[\u00a4\u02d8-&&-\u00a4\u02d8]", "-", 0, 1); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]q-w]", "\u00a4\u00a8", 0, 2); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "f", 0, 1); + x2s("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "g", 0, 1); + ns("[^[^a-z\u00a4\u02d8\u00a4\u00a4\u00a4\u00a6]&&[^bcdefg\u00a4\u00a6\u00a4\u00a8\u00a4\u015e]g-w]", "2"); + x2s("a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9<\\/b>", "a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9", 0, 32); + x2s(".\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9<\\/b>", "a\u0104\u0110\u02c7\u013d\u0104\u00b8\u0104\u00e7\u0104\u00f3\u00a4\u00ce\u0104\u0154\u0104\u00a6\u0104\u00f3\u0104\u00ed\u02c7\u013d\u0104\u00c9", 0, 32); + } + +} diff --git a/third_party/joni/test/org/joni/test/TestCallout.java b/third_party/joni/test/org/joni/test/TestCallout.java new file mode 100644 index 000000000..228c4ef29 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCallout.java @@ -0,0 +1,245 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.CalloutHandler; +import org.joni.CalloutResult; +import org.joni.MatchView; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.exception.TimeoutException; +import org.junit.Test; + +public class TestCallout { + private static Regex regex(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.US_ASCII); + return new Regex(bytes, 0, bytes.length, Option.NONE, ASCIIEncoding.INSTANCE, Syntax.RUBY); + } + + private static int search(Regex regex, String value) { + return search(regex, value, null); + } + + private static int search(Regex regex, String value, CalloutHandler handler) { + byte[] bytes = value.getBytes(StandardCharsets.US_ASCII); + Matcher matcher = regex.matcher(bytes); + matcher.setCalloutHandler(handler); + return matcher.search(0, bytes.length, Option.NONE); + } + + @Test + public void exposesCurrentPositionAndProvisionalCaptures() { + List events = new ArrayList<>(); + Regex regex = regex("(a)(?{=CALL:7})b"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add(id + ":" + match.currentBytePosition() + ":" + + match.captureBegin(0) + "-" + match.captureEnd(0) + ":" + + match.captureBegin(1) + "-" + match.captureEnd(1)); + return CalloutResult.CONTINUE; + } + + @Override + public void unwind(Object token) { + } + }; + + assertEquals(0, search(regex, "ab", handler)); + assertEquals(Arrays.asList("7:1:0-1:0-1"), events); + } + + @Test + public void unwindsBacktrackedAndSuccessfulPathsExactlyOnce() { + List events = new ArrayList<>(); + Regex regex = regex("(a(?{=CALL:1})b|a(?{=CALL:2})c)"); + CalloutHandler handler = recordingHandler(events, false); + + assertEquals(0, search(regex, "ac", handler)); + assertEquals(Arrays.asList("execute:1", "unwind:1", "execute:2", "unwind:2"), events); + } + + @Test + public void failResultBacktracksAfterUnwindingItsToken() { + List events = new ArrayList<>(); + Regex regex = regex("(a(?{=CALL:1})b|ac)"); + CalloutHandler handler = recordingHandler(events, true); + + assertEquals(0, search(regex, "ac", handler)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + @Test + public void unwindsInReverseOrderOnSuccess() { + List events = new ArrayList<>(); + Regex regex = regex("a(?{=CALL:1})(?{=CALL:2})b"); + CalloutHandler handler = recordingHandler(events, false); + + assertEquals(0, search(regex, "ab", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:2", "unwind:1"), events); + } + + @Test + public void preservesCalloutsInsideFixedQuantifiers() { + List events = new ArrayList<>(); + int[] execution = {0}; + Regex regex = regex("(?{=CALL:1}){2}"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + int token = ++execution[0]; + events.add("execute:" + token); + return CalloutResult.continueWith(token); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + + assertEquals(0, search(regex, "", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:2", "unwind:1"), events); + } + + @Test + public void unwindsPriorTokensWhenHandlerThrows() { + List events = new ArrayList<>(); + Regex regex = regex("a(?{=CALL:1})(?{=CALL:2})b"); + CalloutHandler handler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + if (id == 2) throw new IllegalStateException("boom"); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + + assertThrows(IllegalStateException.class, () -> search(regex, "ab", handler)); + assertEquals(Arrays.asList("execute:1", "execute:2", "unwind:1"), events); + } + + @Test + public void requiresAHandlerForCalloutPatterns() { + Regex regex = regex("(?{=CALL:1})"); + assertThrows(IllegalStateException.class, () -> search(regex, "")); + } + + @Test + public void keepsHandlersLocalToEachMatcher() { + Regex regex = regex("(?{=CALL:1})"); + byte[] input = new byte[0]; + List firstEvents = new ArrayList<>(); + List secondEvents = new ArrayList<>(); + Matcher first = regex.matcher(input); + Matcher second = regex.matcher(input); + first.setCalloutHandler(recordingHandler(firstEvents, false)); + second.setCalloutHandler(recordingHandler(secondEvents, false)); + + assertEquals(0, first.search(0, 0, Option.NONE)); + assertEquals(0, second.search(0, 0, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), firstEvents); + assertEquals(Arrays.asList("execute:1", "unwind:1"), secondEvents); + } + + @Test + public void unwindsActiveTokensWhenInterrupted() { + List events = new ArrayList<>(); + Regex regex = regex("(?{=CALL:1})a"); + byte[] input = "a".getBytes(StandardCharsets.US_ASCII); + Matcher matcher = regex.matcher(input); + matcher.setCalloutHandler(new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + matcher.interrupt(); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }); + + assertThrows(InterruptedException.class, + () -> matcher.searchInterruptible(0, input.length, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + @Test + public void unwindsActiveTokensOnTimeout() { + List events = new ArrayList<>(); + StringBuilder pattern = new StringBuilder("(?{=CALL:1})"); + for (int i = 0; i < 140; i++) pattern.append("()"); + Regex regex = regex(pattern.toString()); + byte[] input = new byte[0]; + Matcher matcher = regex.matcher(input); + matcher.setTimeout(1_000_000_000L); + matcher.setCalloutHandler(new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + matcher.setTimeout(0); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }); + + assertThrows(TimeoutException.class, + () -> matcher.searchInterruptible(0, input.length, Option.NONE)); + assertEquals(Arrays.asList("execute:1", "unwind:1"), events); + } + + private static CalloutHandler recordingHandler(List events, boolean fail) { + return new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + return fail ? CalloutResult.failWith(id) : CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + }; + } +} diff --git a/third_party/joni/test/org/joni/test/TestCaseInsensitive.java b/third_party/joni/test/org/joni/test/TestCaseInsensitive.java new file mode 100644 index 000000000..ef37c9719 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCaseInsensitive.java @@ -0,0 +1,32 @@ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestCaseInsensitive extends Test { + + @Override + public int option() { + return Option.IGNORECASE; + } + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + xx("^\\d\\d\\d-".getBytes(), new byte[]{-30, -126, -84, 48, 45}, 0, 0, 0, true); + x2s("ab", "\uD835\uDC4D ab", 5, 7); + } +} diff --git a/third_party/joni/test/org/joni/test/TestCornerCases.java b/third_party/joni/test/org/joni/test/TestCornerCases.java new file mode 100755 index 000000000..310477244 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCornerCases.java @@ -0,0 +1,59 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Config; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; + +public class TestCornerCases extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "cp1250"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + byte[] reg = "l.".getBytes(); + byte[] str = "hello,lo".getBytes(); + + Regex p = new Regex(reg,0,reg.length,Option.DEFAULT,ASCIIEncoding.INSTANCE,Syntax.DEFAULT); + int result = p.matcher(str, 0, str.length).search(3, 0, Option.NONE); + if(result != 3) { + Config.log.println("FAIL: /l./ 'hello,lo' - with reverse, 3,0"); + nfail++; + } + } +} diff --git a/third_party/joni/test/org/joni/test/TestCrnl.java b/third_party/joni/test/org/joni/test/TestCrnl.java new file mode 100755 index 000000000..85044304e --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCrnl.java @@ -0,0 +1,84 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Config; +import org.joni.Option; +import org.joni.Syntax; +import org.junit.Ignore; +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; + +@Ignore +public class TestCrnl extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "ascii"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + x2s("", "\r\n", 0, 0); + x2s(".", "\r\n", 0, 1); + ns("..", "\r\n"); + x2s("^", "\r\n", 0, 0); + x2s("\\n^", "\r\nf", 1, 2); + x2s("\\n^a", "\r\na", 1, 3); + x2s("$", "\r\n", 0, 0); + x2s("T$", "T\r\n", 0, 1); + x2s("T$", "T\raT\r\n", 3, 4); + x2s("\\z", "\r\n", 2, 2); + ns("a\\z", "a\r\n"); + x2s("\\Z", "\r\n", 0, 0); + x2s("\\Z", "\r\na", 3, 3); + x2s("\\Z", "\r\n\r\n\n", 4, 4); + x2s("\\Z", "\r\n\r\nX", 5, 5); + x2s("a\\Z", "a\r\n", 0, 1); + x2s("aaaaaaaaaaaaaaa\\Z", "aaaaaaaaaaaaaaa\r\n", 0, 15); + x2s("a|$", "b\r\n", 1, 1); + x2s("$|b", "\rb", 1, 2); + x2s("a$|ab$", "\r\nab\r\n", 2, 4); + + x2s("a|\\Z", "b\r\n", 1, 1); + x2s("\\Z|b", "\rb", 1, 2); + x2s("a\\Z|ab\\Z", "\r\nab\r\n", 2, 4); + x2s("(?=a$).", "a\r\n", 0, 1); + ns("(?=a$).", "a\r"); + x2s("(?!a$)..", "a\r", 0, 2); + x2s("(?<=a$).\\n", "a\r\n", 1, 3); + ns("(? 0 || nerror > 0) Config.err.println("make sure to enable USE_CRNL_AS_LINE_TERMINATOR"); + } +} diff --git a/third_party/joni/test/org/joni/test/TestError.java b/third_party/joni/test/org/joni/test/TestError.java new file mode 100755 index 000000000..4051611e5 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestError.java @@ -0,0 +1,106 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.ErrorMessages; + +public class TestError extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "iso-8859-2"; + } + + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + @Override + public void test() throws Exception { + xerrs("(", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("[[:WoRd:]]", ErrorMessages.INVALID_POSIX_BRACKET_TYPE); + xerrs("(0?0|(?(1)||)|(?(1)||))?", ErrorMessages.INVALID_CONDITION_PATTERN); + xerrs("[\\40000000000", ErrorMessages.TOO_BIG_NUMBER); + xerrs("[\\40000000000\n", ErrorMessages.TOO_BIG_NUMBER); + xerrs("[]", ErrorMessages.EMPTY_CHAR_CLASS); + xerrs("[c-a]", ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + xerrs("\\x{FFFFFFFF}", ErrorMessages.ERR_TOO_BIG_WIDE_CHAR_VALUE); + xerrs("\\x{100000000}", ErrorMessages.ERR_TOO_LONG_WIDE_CHAR_VALUE); + xerrs("\\u026x", ErrorMessages.TOO_SHORT_DIGITS); + xerrs("()(?\\!(?'a')\\1)", ErrorMessages.UNDEFINED_GROUP_OPTION); + xerrs("\\((", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("(|", ErrorMessages.END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("'/g\\\u00ff\u00ff\u00ff\u00ff&))", ErrorMessages.UNMATCHED_CLOSE_PARENTHESIS); + xerrs("0'/g\\\u00ff\u00ff\u00ff\u00ff&))", 1, 12, ErrorMessages.UNMATCHED_CLOSE_PARENTHESIS, WarnCallback.DEFAULT); + xerrs("0\\1**?", 1, 6, ErrorMessages.INVALID_BACKREF, WarnCallback.DEFAULT); + xerrs("[0-0-\u00ff ", ErrorMessages.PREMATURE_END_OF_CHAR_CLASS); // \xe2 + xerrs("\\p{foobarbaz}", ErrorMessages.ERR_INVALID_CHAR_PROPERTY_NAME.replace("%n", "foobarbaz")); + //xerrs("\\p{あ}", ErrorMessages.ERR_INVALID_CHAR_PROPERTY_NAME.replace("%n", "あ")); + + xerrs("a{100001}", ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + xerrs("a{0,100001}", ErrorMessages.TOO_BIG_NUMBER_FOR_REPEAT_RANGE); + xerrs("a{5,1}", ErrorMessages.UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE); + xerrs("[\\6000", ErrorMessages.TOO_BIG_NUMBER); // CVE-2017-9226 + xerrs("[\\H- ]", ErrorMessages.UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); // CVE-2017-9228 + xerrs("[a-\\d]", ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + + xerrs("(?:ab|cd)*\\1", ErrorMessages.INVALID_BACKREF); + xerrs("(ab|cd)*\\1", ErrorMessages.INVALID_BACKREF, Option.DONT_CAPTURE_GROUP); + + xerrs("(.(?=\\g<1>))", ErrorMessages.NEVER_ENDING_RECURSION); + xerrs("(a)(?b)\\g<1>\\g", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + + // xerrs("(?<", ErrorMessages.ERR_END_PATTERN_WITH_UNMATCHED_PARENTHESIS); + xerrs("(?<>)", ErrorMessages.EMPTY_GROUP_NAME); + //xerrs("(?<.>)", ErrorMessages.ERR_INVALID_CHAR_IN_GROUP_NAME); + xerrs("\\g<1->", ErrorMessages.INVALID_CHAR_IN_GROUP_NAME.replace("%n", "1->")); + xerrs("\\k<1/>", ErrorMessages.INVALID_GROUP_NAME.replace("%n", "1/")); + // xerrs("\\k<1-1/>", ErrorMessages.ERR_INVALID_GROUP_NAME.replace("%n", "1-1/>")); + // xerrs("\\k", ErrorMessages.ERR_INVALID_CHAR_IN_GROUP_NAME.replace("%n", "a/")); + // xerrs("\\g<1>", ErrorMessages.UNDEFINED_GROUP_REFERENCE); + + xerrs("*", ErrorMessages.TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + xerrs("{1}", ErrorMessages.TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + + xerrs("(?a)(?b)\\g", ErrorMessages.MULTIPLEX_DEFINITION_NAME_CALL.replace("%n", "a")); + + xerrs("(a)?(?b)?(?(1)a)(?()b)", ErrorMessages.NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED); + + xerrs("()(?(2))", ErrorMessages.INVALID_BACKREF); + xerrs("(?(700000))", ErrorMessages.INVALID_BACKREF); + + xerrs("(? " + acceptableMaximumTime); + } +} diff --git a/third_party/joni/test/org/joni/test/TestNSU8.java b/third_party/joni/test/org/joni/test/TestNSU8.java new file mode 100755 index 000000000..9f00ed4b7 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestNSU8.java @@ -0,0 +1,61 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.NonStrictUTF8Encoding; + +public class TestNSU8 extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return NonStrictUTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32, (byte)32, (byte)32}, 0, 5, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32, (byte)32}, 0, 4, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)240}, 0, 1, 1, false); + + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32, (byte)32, (byte)32}, 0, 4, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)224}, 0, 1, 1, false); + + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192, (byte)32, (byte)32}, 0, 3, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192, (byte)32}, 0, 2, 1, false); + xx("([^\\[\\]]+)".getBytes(), new byte[]{(byte)192}, 0, 1, 1, false); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerl.java b/third_party/joni/test/org/joni/test/TestPerl.java new file mode 100755 index 000000000..98bbf09f8 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerl.java @@ -0,0 +1,47 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; + +public class TestPerl extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "iso-8859-2"; + } + @Override + public Syntax syntax() { + return Syntax.PerlNG; + } + @Override + public void test() throws Exception { + } +} diff --git a/third_party/joni/test/org/joni/test/TestU.java b/third_party/joni/test/org/joni/test/TestU.java new file mode 100755 index 000000000..f8b027f23 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestU.java @@ -0,0 +1,802 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.joni.Option; +import org.joni.Syntax; +import org.jcodings.Encoding; +import org.jcodings.specific.UTF16BEEncoding; + +public class TestU extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return UTF16BEEncoding.INSTANCE; + } + @Override + public String testEncoding() { + return "iso-8859-1"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + + private int ulen(byte[]bytes) { + return encoding().strByteLengthNull(bytes, 0, bytes.length); + } + + private String uconv(byte []bytes, int len) { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < len; i += 2) { + int c = bytes[i] & 0xff; + // sb.append(String.format("\\%03o", c)); + if (c == 0) { + c = bytes[i+1] & 0xff; + if (c < 0x20 || c >= 0x7f || c == 0x5c || c == 0x22) { + sb.append(String.format("\\%03o", c)); + } else { + sb.append(new String(new byte[]{(byte)c})); + } + } else { + sb.append(String.format("\\%03o", c)); + c = bytes[i+1] & 0xff; + sb.append(String.format("\\%03o", c)); + } + } + + return sb.toString(); + } + + @Override + protected String repr(byte[]bytes) { + return uconv(bytes, ulen(bytes)); + } + + @Override + protected int length(byte[]bytes) { + return ulen(bytes); + } + + @Override + public void test() throws Exception { + x2s("\000\000", "\000\000", 0, 0); + x2s("\000^\000\000", "\000\000", 0, 0); + x2s("\000$\000\000", "\000\000", 0, 0); + x2s("\000\134\000G\000\000", "\000\000", 0, 0); + x2s("\000\134\000A\000\000", "\000\000", 0, 0); + x2s("\000\134\000Z\000\000", "\000\000", 0, 0); + x2s("\000\134\000z\000\000", "\000\000", 0, 0); + x2s("\000^\000$\000\000", "\000\000", 0, 0); + x2s("\000\134\000c\000a\000\000", "\000\001\000\000", 0, 2); + x2s("\000\134\000C\000-\000b\000\000", "\000\002\000\000", 0, 2); + x2s("\000\134\000c\000\134\000\134\000\000", "\000\034\000\000", 0, 2); + x2s("\000q\000[\000\134\000c\000\134\000\134\000]\000\000", "\000q\000\034\000\000", 0, 4); + x2s("\000\000", "\000a\000\000", 0, 0); + x2s("\000a\000\000", "\000a\000\000", 0, 2); + x2s("\000\134\000x\0000\0000\000\134\000x\0006\0001\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000a\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000a\000a\000a\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000\000", "\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000a\000\000", 0, 70); + x2s("\000a\000b\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000b\000\000", "\000a\000b\000\000", 2, 4); + x2s("\000b\000c\000\000", "\000a\000b\000c\000\000", 2, 6); + x2s("\000(\000?\000i\000:\000#\000R\000E\000T\000#\000)\000\000", "\000#\000I\000N\000S\000#\000#\000R\000E\000T\000#\000\000", 10, 20); + x2s("\000\134\0000\0000\0000\000\134\0001\0007\000\000", "\000\017\000\000", 0, 2); + x2s("\000\134\000x\0000\0000\000\134\000x\0001\000f\000\000", "\000\037\000\000", 0, 2); + x2s("\000a\000(\000?\000#\000.\000.\000.\000.\000\134\000\134\000J\000J\000J\000J\000)\000b\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000(\000?\000x\000)\000 \000 \000G\000 \000(\000o\000 \000O\000(\000?\000-\000x\000)\000o\000O\000)\000 \000g\000 \000L\000\000", "\000G\000o\000O\000o\000O\000g\000L\000e\000\000", 0, 14); + x2s("\000.\000\000", "\000a\000\000", 0, 2); + ns("\000.\000\000", "\000\000"); + x2s("\000.\000.\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000\134\000w\000\000", "\000e\000\000", 0, 2); + ns("\000\134\000W\000\000", "\000e\000\000"); + x2s("\000\134\000s\000\000", "\000 \000\000", 0, 2); + x2s("\000\134\000S\000\000", "\000b\000\000", 0, 2); + x2s("\000\134\000d\000\000", "\0004\000\000", 0, 2); + ns("\000\134\000D\000\000", "\0004\000\000"); + x2s("\000\134\000b\000\000", "\000z\000 \000\000", 0, 0); + x2s("\000\134\000b\000\000", "\000 \000z\000\000", 2, 2); + x2s("\000\134\000B\000\000", "\000z\000z\000 \000\000", 2, 2); + x2s("\000\134\000B\000\000", "\000z\000 \000\000", 4, 4); + x2s("\000\134\000B\000\000", "\000 \000z\000\000", 0, 0); + x2s("\000[\000a\000b\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000a\000b\000]\000\000", "\000c\000\000"); + x2s("\000[\000a\000-\000z\000]\000\000", "\000t\000\000", 0, 2); + ns("\000[\000^\000a\000]\000\000", "\000a\000\000"); + x2s("\000[\000^\000a\000]\000\000", "\000\012\000\000", 0, 2); + x2s("\000[\000]\000]\000\000", "\000]\000\000", 0, 2); + ns("\000[\000^\000]\000]\000\000", "\000]\000\000"); + x2s("\000[\000\134\000^\000]\000+\000\000", "\0000\000^\000^\0001\000\000", 2, 6); + x2s("\000[\000b\000-\000]\000\000", "\000b\000\000", 0, 2); + x2s("\000[\000b\000-\000]\000\000", "\000-\000\000", 0, 2); + x2s("\000[\000\134\000w\000]\000\000", "\000z\000\000", 0, 2); + ns("\000[\000\134\000w\000]\000\000", "\000 \000\000"); + x2s("\000[\000\134\000W\000]\000\000", "\000b\000$\000\000", 2, 4); + x2s("\000[\000\134\000d\000]\000\000", "\0005\000\000", 0, 2); + ns("\000[\000\134\000d\000]\000\000", "\000e\000\000"); + x2s("\000[\000\134\000D\000]\000\000", "\000t\000\000", 0, 2); + ns("\000[\000\134\000D\000]\000\000", "\0003\000\000"); + x2s("\000[\000\134\000s\000]\000\000", "\000 \000\000", 0, 2); + ns("\000[\000\134\000s\000]\000\000", "\000a\000\000"); + x2s("\000[\000\134\000S\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000\134\000S\000]\000\000", "\000 \000\000"); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "\0002\000\000", 0, 2); + ns("\000[\000\134\000w\000\134\000d\000]\000\000", "\000 \000\000"); + x2s("\000[\000[\000:\000u\000p\000p\000e\000r\000:\000]\000]\000\000", "\000B\000\000", 0, 2); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000+\000\000", 0, 2); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000G\000H\000I\000K\000K\000-\0009\000+\000*\000\000", 12, 14); + x2s("\000[\000*\000[\000:\000x\000d\000i\000g\000i\000t\000:\000]\000+\000]\000\000", "\000-\000@\000^\000+\000\000", 6, 8); + ns("\000[\000[\000:\000u\000p\000p\000e\000r\000]\000]\000\000", "\000A\000\000"); + x2s("\000[\000[\000:\000u\000p\000p\000e\000r\000]\000]\000\000", "\000:\000\000", 0, 2); + x2s("\000[\000\134\0000\0000\0000\000\134\0000\0004\0004\000-\000\134\0000\0000\0000\000\134\0000\0004\0007\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000\134\000x\0000\0000\000\134\000x\0005\000a\000-\000\134\000x\0000\0000\000\134\000x\0005\000c\000]\000\000", "\000[\000\000", 0, 2); + x2s("\000[\000\134\000x\0000\0000\000\134\000x\0006\000A\000-\000\134\000x\0000\0000\000\134\000x\0006\000D\000]\000\000", "\000l\000\000", 0, 2); + ns("\000[\000\134\000x\0000\0000\000\134\000x\0006\000A\000-\000\134\000x\0000\0000\000\134\000x\0006\000D\000]\000\000", "\000n\000\000"); + ns("\000^\000[\0000\000-\0009\000A\000-\000F\000]\000+\000 \0000\000+\000 \000U\000N\000D\000E\000F\000 \000\000", "\0007\0005\000F\000 \0000\0000\0000\0000\0000\0000\0000\0000\000 \000S\000E\000C\000T\0001\0004\000A\000 \000n\000o\000t\000y\000p\000e\000 \000(\000)\000 \000 \000 \000 \000E\000x\000t\000e\000r\000n\000a\000l\000 \000 \000 \000 \000|\000 \000_\000r\000b\000_\000a\000p\000p\000l\000y\000\000"); + x2s("\000[\000\134\000[\000]\000\000", "\000[\000\000", 0, 2); + x2s("\000[\000\134\000]\000]\000\000", "\000]\000\000", 0, 2); + x2s("\000[\000&\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000[\000a\000b\000]\000]\000\000", "\000b\000\000", 0, 2); + x2s("\000[\000[\000a\000b\000]\000c\000]\000\000", "\000c\000\000", 0, 2); + ns("\000[\000[\000^\000a\000]\000]\000\000", "\000a\000\000"); + ns("\000[\000^\000[\000a\000]\000]\000\000", "\000a\000\000"); + x2s("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000a\000\000"); + ns("\000[\000[\000a\000b\000]\000&\000&\000b\000c\000]\000\000", "\000c\000\000"); + x2s("\000[\000a\000-\000z\000&\000&\000b\000-\000y\000&\000&\000c\000-\000x\000]\000\000", "\000w\000\000", 0, 2); + ns("\000[\000^\000a\000-\000z\000&\000&\000b\000-\000y\000&\000&\000c\000-\000x\000]\000\000", "\000w\000\000"); + x2s("\000[\000[\000^\000a\000&\000&\000a\000]\000&\000&\000a\000-\000z\000]\000\000", "\000b\000\000", 0, 2); + ns("\000[\000[\000^\000a\000&\000&\000a\000]\000&\000&\000a\000-\000z\000]\000\000", "\000a\000\000"); + x2s("\000[\000[\000^\000a\000-\000z\000&\000&\000b\000c\000d\000e\000f\000]\000&\000&\000[\000^\000c\000-\000g\000]\000]\000\000", "\000h\000\000", 0, 2); + ns("\000[\000[\000^\000a\000-\000z\000&\000&\000b\000c\000d\000e\000f\000]\000&\000&\000[\000^\000c\000-\000g\000]\000]\000\000", "\000c\000\000"); + x2s("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000c\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000e\000\000", 0, 2); + ns("\000[\000^\000[\000^\000a\000b\000c\000]\000&\000&\000[\000^\000c\000d\000e\000]\000]\000\000", "\000f\000\000"); + x2s("\000[\000a\000-\000&\000&\000-\000a\000]\000\000", "\000-\000\000", 0, 2); + ns("\000[\000a\000\134\000-\000&\000&\000\134\000-\000a\000]\000\000", "\000&\000\000"); + ns("\000\134\000w\000a\000b\000c\000\000", "\000 \000a\000b\000c\000\000"); + x2s("\000a\000\134\000W\000b\000c\000\000", "\000a\000 \000b\000c\000\000", 0, 8); + x2s("\000a\000.\000b\000.\000c\000\000", "\000a\000a\000b\000b\000c\000\000", 0, 10); + x2s("\000.\000\134\000w\000b\000\134\000W\000.\000.\000c\000\000", "\000a\000b\000b\000 \000b\000c\000c\000\000", 0, 14); + x2s("\000\134\000s\000\134\000w\000z\000z\000z\000\000", "\000 \000z\000z\000z\000z\000\000", 0, 10); + x2s("\000a\000a\000.\000b\000\000", "\000a\000a\000b\000b\000\000", 0, 8); + ns("\000.\000a\000\000", "\000a\000b\000\000"); + x2s("\000.\000a\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000^\000a\000\000", "\000a\000\000", 0, 2); + x2s("\000^\000a\000$\000\000", "\000a\000\000", 0, 2); + x2s("\000^\000\134\000w\000$\000\000", "\000a\000\000", 0, 2); + ns("\000^\000\134\000w\000$\000\000", "\000 \000\000"); + x2s("\000^\000\134\000w\000a\000b\000$\000\000", "\000z\000a\000b\000\000", 0, 6); + x2s("\000^\000\134\000w\000a\000b\000c\000d\000e\000f\000$\000\000", "\000z\000a\000b\000c\000d\000e\000f\000\000", 0, 14); + x2s("\000^\000\134\000w\000.\000.\000.\000d\000e\000f\000$\000\000", "\000z\000a\000b\000c\000d\000e\000f\000\000", 0, 14); + x2s("\000\134\000w\000\134\000w\000\134\000s\000\134\000W\000a\000a\000a\000\134\000d\000\000", "\000a\000a\000 \000 \000a\000a\000a\0004\000\000", 0, 16); + x2s("\000\134\000A\000\134\000Z\000\000", "\000\000", 0, 0); + x2s("\000\134\000A\000x\000y\000z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000x\000y\000z\000\134\000Z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000x\000y\000z\000\134\000z\000\000", "\000x\000y\000z\000\000", 0, 6); + x2s("\000a\000\134\000Z\000\000", "\000a\000\000", 0, 2); + x2s("\000\134\000G\000a\000z\000\000", "\000a\000z\000\000", 0, 4); + ns("\000\134\000G\000z\000\000", "\000b\000z\000a\000\000"); + ns("\000a\000z\000\134\000G\000\000", "\000a\000z\000\000"); + ns("\000a\000z\000\134\000A\000\000", "\000a\000z\000\000"); + ns("\000a\000\134\000A\000z\000\000", "\000a\000z\000\000"); + x2s("\000\134\000^\000\134\000$\000\000", "\000^\000$\000\000", 0, 4); + x2s("\000^\000x\000?\000y\000\000", "\000x\000y\000\000", 0, 4); + x2s("\000^\000(\000x\000?\000y\000)\000\000", "\000x\000y\000\000", 0, 4); + x2s("\000\134\000w\000\000", "\000_\000\000", 0, 2); + ns("\000\134\000W\000\000", "\000_\000\000"); + x2s("\000(\000?\000=\000z\000)\000z\000\000", "\000z\000\000", 0, 2); + ns("\000(\000?\000=\000z\000)\000.\000\000", "\000a\000\000"); + x2s("\000(\000?\000!\000z\000)\000a\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000!\000z\000)\000a\000\000", "\000z\000\000"); + x2s("\000(\000?\000i\000:\000a\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000a\000)\000\000", "\000A\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000A\000)\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000i\000:\000A\000)\000\000", "\000b\000\000"); + x2s("\000(\000?\000i\000:\000[\000A\000-\000Z\000]\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000H\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000h\000\000", 0, 2); + ns("\000(\000?\000i\000:\000[\000f\000-\000m\000]\000)\000\000", "\000e\000\000"); + x2s("\000(\000?\000i\000:\000[\000A\000-\000c\000]\000)\000\000", "\000D\000\000", 0, 2); + ns("\000(\000?\000i\000:\000[\000^\000a\000-\000z\000]\000)\000\000", "\000A\000\000"); + ns("\000(\000?\000i\000:\000[\000^\000a\000-\000z\000]\000)\000\000", "\000a\000\000"); + x2s("\000(\000?\000i\000:\000[\000!\000-\000k\000]\000)\000\000", "\000Z\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000!\000-\000k\000]\000)\000\000", "\0007\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000T\000-\000}\000]\000)\000\000", "\000b\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000[\000T\000-\000}\000]\000)\000\000", "\000{\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000\134\000?\000a\000)\000\000", "\000?\000A\000\000", 0, 4); + x2s("\000(\000?\000i\000:\000\134\000*\000A\000)\000\000", "\000*\000a\000\000", 0, 4); + ns("\000.\000\000", "\000\012\000\000"); + x2s("\000(\000?\000m\000:\000.\000)\000\000", "\000\012\000\000", 0, 2); + x2s("\000(\000?\000m\000:\000a\000.\000)\000\000", "\000a\000\012\000\000", 0, 4); + x2s("\000(\000?\000m\000:\000.\000b\000)\000\000", "\000a\000\012\000b\000\000", 2, 6); + x2s("\000.\000*\000a\000b\000c\000\000", "\000d\000d\000d\000a\000b\000d\000d\000\012\000d\000d\000a\000b\000c\000\000", 16, 26); + x2s("\000(\000?\000m\000:\000.\000*\000a\000b\000c\000)\000\000", "\000d\000d\000d\000a\000b\000d\000d\000a\000b\000c\000\000", 0, 20); + ns("\000(\000?\000i\000)\000(\000?\000-\000i\000)\000a\000\000", "\000A\000\000"); + ns("\000(\000?\000i\000)\000(\000?\000-\000i\000:\000a\000)\000\000", "\000A\000\000"); + x2s("\000a\000?\000\000", "\000\000", 0, 0); + x2s("\000a\000?\000\000", "\000b\000\000", 0, 0); + x2s("\000a\000?\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000*\000\000", "\000\000", 0, 0); + x2s("\000a\000*\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000*\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000a\000*\000\000", "\000b\000a\000a\000a\000a\000\000", 0, 0); + ns("\000a\000+\000\000", "\000\000"); + x2s("\000a\000+\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000+\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000a\000+\000\000", "\000a\000a\000b\000b\000b\000\000", 0, 4); + x2s("\000a\000+\000\000", "\000b\000a\000a\000a\000a\000\000", 2, 10); + x2s("\000.\000?\000\000", "\000\000", 0, 0); + x2s("\000.\000?\000\000", "\000f\000\000", 0, 2); + x2s("\000.\000?\000\000", "\000\012\000\000", 0, 0); + x2s("\000.\000*\000\000", "\000\000", 0, 0); + x2s("\000.\000*\000\000", "\000a\000b\000c\000d\000e\000\000", 0, 10); + x2s("\000.\000+\000\000", "\000z\000\000", 0, 2); + x2s("\000.\000+\000\000", "\000z\000d\000s\000w\000e\000r\000\012\000\000", 0, 12); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000b\000f\000b\000a\000c\000\000", 0, 8); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000c\000b\000a\000b\000f\000\000", 6, 14); + x2s("\000(\000(\000.\000*\000)\000a\000\134\0002\000f\000)\000\000", "\000b\000a\000c\000b\000a\000b\000f\000\000", 6, 14); + x2s("\000(\000.\000*\000)\000a\000\134\0001\000f\000\000", "\000b\000a\000c\000z\000z\000z\000z\000z\000z\000\012\000b\000a\000z\000z\000\012\000z\000z\000z\000z\000b\000a\000b\000f\000\000", 38, 46); + x2s("\000a\000|\000b\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000|\000b\000\000", "\000b\000\000", 0, 2); + x2s("\000|\000a\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000|\000a\000)\000\000", "\000a\000\000", 0, 0); + x2s("\000a\000b\000|\000b\000c\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000a\000b\000|\000b\000c\000\000", "\000b\000c\000\000", 0, 4); + x2s("\000z\000(\000?\000:\000a\000b\000|\000b\000c\000)\000\000", "\000z\000b\000c\000\000", 0, 6); + x2s("\000a\000(\000?\000:\000a\000b\000|\000b\000c\000)\000c\000\000", "\000a\000a\000b\000c\000\000", 0, 8); + x2s("\000a\000b\000|\000(\000?\000:\000a\000c\000|\000a\000z\000)\000\000", "\000a\000z\000\000", 0, 4); + x2s("\000a\000|\000b\000|\000c\000\000", "\000d\000c\000\000", 2, 4); + x2s("\000a\000|\000b\000|\000c\000d\000|\000e\000f\000g\000|\000h\000|\000i\000j\000k\000|\000l\000m\000n\000|\000o\000|\000p\000q\000|\000r\000s\000t\000u\000v\000w\000x\000|\000y\000z\000\000", "\000p\000q\000r\000\000", 0, 4); + ns("\000a\000|\000b\000|\000c\000d\000|\000e\000f\000g\000|\000h\000|\000i\000j\000k\000|\000l\000m\000n\000|\000o\000|\000p\000q\000|\000r\000s\000t\000u\000v\000w\000x\000|\000y\000z\000\000", "\000m\000n\000\000"); + x2s("\000a\000|\000^\000z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000^\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000\134\000G\000z\000\000", "\000b\000z\000a\000\000", 4, 6); + x2s("\000a\000|\000\134\000G\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000\134\000A\000z\000\000", "\000b\000z\000a\000\000", 4, 6); + x2s("\000a\000|\000\134\000A\000z\000\000", "\000z\000a\000\000", 0, 2); + x2s("\000a\000|\000b\000\134\000Z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000b\000\134\000Z\000\000", "\000b\000\000", 0, 2); + x2s("\000a\000|\000b\000\134\000z\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000a\000|\000b\000\134\000z\000\000", "\000b\000\000", 0, 2); + x2s("\000\134\000w\000|\000\134\000s\000\000", "\000 \000\000", 0, 2); + ns("\000\134\000w\000|\000\134\000w\000\000", "\000 \000\000"); + x2s("\000\134\000w\000|\000%\000\000", "\000%\000\000", 0, 2); + x2s("\000\134\000w\000|\000[\000&\000$\000]\000\000", "\000&\000\000", 0, 2); + x2s("\000[\000b\000-\000d\000]\000|\000[\000^\000e\000-\000z\000]\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000|\000[\000c\000-\000f\000]\000)\000|\000b\000z\000\000", "\000d\000z\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000|\000[\000c\000-\000f\000]\000)\000|\000b\000z\000\000", "\000b\000z\000\000", 0, 4); + x2s("\000a\000b\000c\000|\000(\000?\000=\000z\000z\000)\000.\000.\000f\000\000", "\000z\000z\000f\000\000", 0, 6); + x2s("\000a\000b\000c\000|\000(\000?\000!\000z\000z\000)\000.\000.\000f\000\000", "\000a\000b\000f\000\000", 0, 6); + x2s("\000(\000?\000=\000z\000a\000)\000.\000.\000a\000|\000(\000?\000=\000z\000z\000)\000.\000.\000a\000\000", "\000z\000z\000a\000\000", 0, 6); + ns("\000(\000?\000>\000a\000|\000a\000b\000d\000)\000c\000\000", "\000a\000b\000d\000c\000\000"); + x2s("\000(\000?\000>\000a\000b\000d\000|\000a\000)\000c\000\000", "\000a\000b\000d\000c\000\000", 0, 8); + x2s("\000a\000?\000|\000b\000\000", "\000a\000\000", 0, 2); + x2s("\000a\000?\000|\000b\000\000", "\000b\000\000", 0, 0); + x2s("\000a\000?\000|\000b\000\000", "\000\000", 0, 0); + x2s("\000a\000*\000|\000b\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000a\000*\000|\000b\000*\000\000", "\000b\000a\000\000", 0, 0); + x2s("\000a\000*\000|\000b\000*\000\000", "\000a\000b\000\000", 0, 2); + x2s("\000a\000+\000|\000b\000*\000\000", "\000\000", 0, 0); + x2s("\000a\000+\000|\000b\000*\000\000", "\000b\000b\000b\000\000", 0, 6); + x2s("\000a\000+\000|\000b\000*\000\000", "\000a\000b\000b\000b\000\000", 0, 2); + ns("\000a\000+\000|\000b\000+\000\000", "\000\000"); + x2s("\000(\000a\000|\000b\000)\000?\000\000", "\000b\000\000", 0, 2); + x2s("\000(\000a\000|\000b\000)\000*\000\000", "\000b\000a\000\000", 0, 4); + x2s("\000(\000a\000|\000b\000)\000+\000\000", "\000b\000a\000b\000\000", 0, 6); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000c\000a\000a\000b\000b\000c\000\000", 0, 8); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000a\000a\000b\000c\000a\000\000", 2, 10); + x2s("\000(\000a\000b\000|\000c\000a\000)\000+\000\000", "\000a\000b\000z\000c\000a\000\000", 0, 4); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 10); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000b\000a\000\000", 2, 4); + x2s("\000(\000a\000|\000b\000a\000b\000)\000+\000\000", "\000b\000a\000a\000a\000b\000a\000\000", 2, 8); + x2s("\000(\000?\000:\000a\000|\000b\000)\000(\000?\000:\000a\000|\000b\000)\000\000", "\000a\000b\000\000", 0, 4); + x2s("\000(\000?\000:\000a\000*\000|\000b\000*\000)\000(\000?\000:\000a\000*\000|\000b\000*\000)\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 6); + x2s("\000(\000?\000:\000a\000*\000|\000b\000*\000)\000(\000?\000:\000a\000+\000|\000b\000+\000)\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + x2s("\000(\000?\000:\000a\000+\000|\000b\000+\000)\000{\0002\000}\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + x2s("\000h\000{\0000\000,\000}\000\000", "\000h\000h\000h\000h\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000+\000|\000b\000+\000)\000{\0001\000,\0002\000}\000\000", "\000a\000a\000a\000b\000b\000b\000\000", 0, 12); + ns("\000a\000x\000{\0002\000}\000*\000a\000\000", "\0000\000a\000x\000x\000x\000a\0001\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000a\000\000", "\0000\000a\000X\000X\000X\000a\0000\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000?\000a\000\000", "\0000\000a\000X\000X\000X\000a\0000\000\000"); + ns("\000a\000.\000{\0000\000,\0002\000}\000?\000a\000\000", "\0000\000a\000X\000X\000X\000X\000a\0000\000\000"); + x2s("\000^\000a\000{\0002\000,\000}\000?\000a\000$\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000^\000[\000a\000-\000z\000]\000{\0002\000,\000}\000?\000$\000\000", "\000a\000a\000a\000\000", 0, 6); + x2s("\000(\000?\000:\000a\000+\000|\000\134\000A\000b\000*\000)\000c\000c\000\000", "\000c\000c\000\000", 0, 4); + ns("\000(\000?\000:\000a\000+\000|\000\134\000A\000b\000*\000)\000c\000c\000\000", "\000a\000b\000c\000c\000\000"); + x2s("\000(\000?\000:\000^\000a\000+\000|\000b\000+\000)\000*\000c\000\000", "\000a\000a\000b\000b\000b\000a\000b\000c\000\000", 12, 16); + x2s("\000(\000?\000:\000^\000a\000+\000|\000b\000+\000)\000*\000c\000\000", "\000a\000a\000b\000b\000b\000b\000c\000\000", 0, 14); + x2s("\000a\000|\000(\000?\000i\000)\000c\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|\000a\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|\000a\000\000", "\000A\000\000", 0, 2); + x2s("\000(\000?\000i\000:\000c\000)\000|\000a\000\000", "\000C\000\000", 0, 2); + ns("\000(\000?\000i\000:\000c\000)\000|\000a\000\000", "\000A\000\000"); + x2s("\000[\000a\000b\000c\000]\000?\000\000", "\000a\000b\000c\000\000", 0, 2); + x2s("\000[\000a\000b\000c\000]\000*\000\000", "\000a\000b\000c\000\000", 0, 6); + x2s("\000[\000^\000a\000b\000c\000]\000*\000\000", "\000a\000b\000c\000\000", 0, 0); + ns("\000[\000^\000a\000b\000c\000]\000+\000\000", "\000a\000b\000c\000\000"); + x2s("\000a\000?\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000b\000a\000?\000?\000b\000\000", "\000b\000a\000b\000\000", 0, 6); + x2s("\000a\000*\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000b\000a\000*\000?\000\000", "\000b\000a\000a\000\000", 0, 2); + x2s("\000b\000a\000*\000?\000b\000\000", "\000b\000a\000a\000b\000\000", 0, 8); + x2s("\000a\000+\000?\000\000", "\000a\000a\000a\000\000", 0, 2); + x2s("\000b\000a\000+\000?\000\000", "\000b\000a\000a\000\000", 0, 4); + x2s("\000b\000a\000+\000?\000b\000\000", "\000b\000a\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000?\000)\000?\000?\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000?\000?\000)\000?\000\000", "\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000?\000)\000+\000?\000\000", "\000a\000a\000a\000\000", 0, 2); + x2s("\000(\000?\000:\000a\000+\000)\000?\000?\000\000", "\000a\000a\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000+\000)\000?\000?\000b\000\000", "\000a\000a\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000?\000{\0002\000}\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000b\000)\000?\000{\0002\000}\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000*\000{\0000\000}\000\000", "\000a\000b\000a\000b\000a\000\000", 0, 0); + x2s("\000(\000?\000:\000a\000b\000)\000{\0003\000,\000}\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 16); + ns("\000(\000?\000:\000a\000b\000)\000{\0003\000,\000}\000\000", "\000a\000b\000a\000b\000\000"); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000\000", "\000a\000b\000a\000b\000a\000b\000\000", 0, 12); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 16); + x2s("\000(\000?\000:\000a\000b\000)\000{\0002\000,\0004\000}\000?\000\000", "\000a\000b\000a\000b\000a\000b\000a\000b\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000:\000a\000b\000)\000{\000,\000}\000\000", "\000a\000b\000{\000,\000}\000\000", 0, 10); + x2s("\000(\000?\000:\000a\000b\000c\000)\000+\000?\000{\0002\000}\000\000", "\000a\000b\000c\000a\000b\000c\000a\000b\000c\000\000", 0, 12); + x2s("\000(\000?\000:\000X\000*\000)\000(\000?\000i\000:\000x\000a\000)\000\000", "\000X\000X\000X\000a\000\000", 0, 8); + x2s("\000(\000d\000+\000)\000(\000[\000^\000a\000b\000c\000]\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x2s("\000(\000[\000^\000a\000b\000c\000]\000*\000)\000(\000[\000^\000a\000b\000c\000]\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x2s("\000(\000\134\000w\000+\000)\000(\000\134\000w\000z\000)\000\000", "\000d\000d\000d\000z\000\000", 0, 8); + x3s("\000(\000a\000)\000\000", "\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000b\000)\000\000", "\000a\000b\000\000", 0, 4, 1); + x2s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4); + x3s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 1); + x3s("\000(\000(\000a\000b\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 2); + x3s("\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000a\000b\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000\000", "\000a\000b\000\000", 0, 4, 20); + x3s("\000(\000a\000b\000)\000(\000c\000d\000)\000\000", "\000a\000b\000c\000d\000\000", 0, 4, 1); + x3s("\000(\000a\000b\000)\000(\000c\000d\000)\000\000", "\000a\000b\000c\000d\000\000", 4, 8, 2); + x3s("\000(\000)\000(\000a\000)\000b\000c\000(\000d\000e\000f\000)\000g\000h\000i\000j\000k\000\000", "\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000\000", 6, 12, 3); + x3s("\000(\000(\000)\000(\000a\000)\000b\000c\000(\000d\000e\000f\000)\000g\000h\000i\000j\000k\000)\000\000", "\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000\000", 6, 12, 4); + x2s("\000(\000^\000a\000)\000\000", "\000a\000\000", 0, 2); + x3s("\000(\000a\000)\000|\000(\000a\000)\000\000", "\000b\000a\000\000", 2, 4, 1); + x3s("\000(\000^\000a\000)\000|\000(\000a\000)\000\000", "\000b\000a\000\000", 2, 4, 2); + x3s("\000(\000a\000?\000)\000\000", "\000a\000a\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000*\000)\000\000", "\000a\000a\000a\000\000", 0, 6, 1); + x3s("\000(\000a\000*\000)\000\000", "\000\000", 0, 0, 1); + x3s("\000(\000a\000+\000)\000\000", "\000a\000a\000a\000a\000a\000a\000a\000\000", 0, 14, 1); + x3s("\000(\000a\000+\000|\000b\000*\000)\000\000", "\000b\000b\000b\000a\000a\000\000", 0, 6, 1); + x3s("\000(\000a\000+\000|\000b\000?\000)\000\000", "\000b\000b\000b\000a\000a\000\000", 0, 2, 1); + x3s("\000(\000a\000b\000c\000)\000?\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000a\000b\000c\000)\000*\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000x\000y\000z\000|\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000[\000x\000y\000z\000]\000[\000a\000b\000c\000]\000|\000a\000b\000c\000)\000+\000\000", "\000a\000b\000c\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000i\000:\000a\000b\000c\000)\000)\000\000", "\000A\000b\000C\000\000", 0, 6, 1); + x2s("\000(\000a\000b\000c\000)\000(\000?\000i\000:\000\134\0001\000)\000\000", "\000a\000b\000c\000A\000B\000C\000\000", 0, 12); + x3s("\000(\000(\000?\000m\000:\000a\000.\000c\000)\000)\000\000", "\000a\000\012\000c\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000=\000a\000z\000)\000a\000)\000\000", "\000a\000z\000b\000\000", 0, 2, 1); + x3s("\000a\000b\000c\000|\000(\000.\000a\000b\000d\000)\000\000", "\000z\000a\000b\000d\000\000", 0, 8, 1); + x2s("\000(\000?\000:\000a\000b\000c\000)\000|\000(\000A\000B\000C\000)\000\000", "\000a\000b\000c\000\000", 0, 6); + x3s("\000(\000?\000i\000:\000(\000a\000b\000c\000)\000)\000|\000(\000z\000z\000z\000)\000\000", "\000A\000B\000C\000\000", 0, 6, 1); + x3s("\000a\000*\000(\000.\000)\000\000", "\000a\000a\000a\000a\000z\000\000", 8, 10, 1); + x3s("\000a\000*\000?\000(\000.\000)\000\000", "\000a\000a\000a\000a\000z\000\000", 0, 2, 1); + x3s("\000a\000*\000?\000(\000c\000)\000\000", "\000a\000a\000a\000a\000c\000\000", 8, 10, 1); + x3s("\000[\000b\000c\000d\000]\000a\000*\000(\000.\000)\000\000", "\000c\000a\000a\000a\000a\000z\000\000", 10, 12, 1); + x3s("\000(\000\134\000A\000b\000b\000)\000c\000c\000\000", "\000b\000b\000c\000c\000\000", 0, 4, 1); + ns("\000(\000\134\000A\000b\000b\000)\000c\000c\000\000", "\000z\000b\000b\000c\000c\000\000"); + x3s("\000(\000^\000b\000b\000)\000c\000c\000\000", "\000b\000b\000c\000c\000\000", 0, 4, 1); + ns("\000(\000^\000b\000b\000)\000c\000c\000\000", "\000z\000b\000b\000c\000c\000\000"); + x3s("\000c\000c\000(\000b\000b\000$\000)\000\000", "\000c\000c\000b\000b\000\000", 4, 8, 1); + ns("\000c\000c\000(\000b\000b\000$\000)\000\000", "\000c\000c\000b\000b\000b\000\000"); + ns("\000(\000\134\0001\000)\000\000", "\000\000"); + ns("\000\134\0001\000(\000a\000)\000\000", "\000a\000a\000\000"); + ns("\000(\000a\000(\000b\000)\000\134\0001\000)\000\134\0002\000+\000\000", "\000a\000b\000a\000b\000b\000\000"); + ns("\000(\000?\000:\000(\000?\000:\000\134\0001\000|\000z\000)\000(\000a\000)\000)\000+\000$\000\000", "\000z\000a\000a\000\000"); + x2s("\000(\000?\000:\000(\000?\000:\000\134\0001\000|\000z\000)\000(\000a\000)\000)\000+\000$\000\000", "\000z\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000a\000)\000(\000?\000=\000\134\0001\000)\000\000", "\000a\000a\000\000", 0, 2); + ns("\000(\000a\000)\000$\000|\000\134\0001\000\000", "\000a\000z\000\000"); + x2s("\000(\000a\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + ns("\000(\000a\000)\000\134\0001\000\000", "\000a\000b\000\000"); + x2s("\000(\000a\000?\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000(\000a\000?\000?\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 0); + x2s("\000(\000a\000*\000)\000\134\0001\000\000", "\000a\000a\000a\000a\000a\000\000", 0, 8); + x3s("\000(\000a\000*\000)\000\134\0001\000\000", "\000a\000a\000a\000a\000a\000\000", 0, 4, 1); + x2s("\000a\000(\000b\000*\000)\000\134\0001\000\000", "\000a\000b\000b\000b\000b\000\000", 0, 10); + x2s("\000a\000(\000b\000*\000)\000\134\0001\000\000", "\000a\000b\000\000", 0, 2); + x2s("\000(\000a\000*\000)\000(\000b\000*\000)\000\134\0001\000\134\0002\000\000", "\000a\000a\000a\000b\000b\000a\000a\000a\000b\000b\000\000", 0, 20); + x2s("\000(\000a\000*\000)\000(\000b\000*\000)\000\134\0002\000\000", "\000a\000a\000a\000b\000b\000b\000b\000\000", 0, 14); + x2s("\000(\000(\000(\000(\000(\000(\000(\000a\000*\000)\000b\000)\000)\000)\000)\000)\000)\000c\000\134\0007\000\000", "\000a\000a\000a\000b\000c\000a\000a\000a\000\000", 0, 16); + x3s("\000(\000(\000(\000(\000(\000(\000(\000a\000*\000)\000b\000)\000)\000)\000)\000)\000)\000c\000\134\0007\000\000", "\000a\000a\000a\000b\000c\000a\000a\000a\000\000", 0, 6, 7); + x2s("\000(\000a\000)\000(\000b\000)\000(\000c\000)\000\134\0002\000\134\0001\000\134\0003\000\000", "\000a\000b\000c\000b\000a\000c\000\000", 0, 12); + x2s("\000(\000[\000a\000-\000d\000]\000)\000\134\0001\000\000", "\000c\000c\000\000", 0, 4); + x2s("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "\000f\0005\000 \000f\0005\000 \000\000", 0, 12); + ns("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "\000f\0005\000 \000f\0005\000\000"); + x2s("\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000w\000h\000o\000w\000h\000o\000\000", 0, 12); + x2s("\000.\000.\000.\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000a\000b\000c\000w\000h\000o\000w\000h\000o\000\000", 0, 18); + x2s("\000(\000w\000h\000o\000|\000[\000a\000-\000c\000]\000{\0003\000}\000)\000\134\0001\000\000", "\000c\000b\000c\000c\000b\000c\000\000", 0, 12); + x2s("\000(\000^\000a\000)\000\134\0001\000\000", "\000a\000a\000\000", 0, 4); + ns("\000(\000^\000a\000)\000\134\0001\000\000", "\000b\000a\000a\000\000"); + ns("\000(\000a\000$\000)\000\134\0001\000\000", "\000a\000a\000\000"); + ns("\000(\000a\000b\000\134\000Z\000)\000\134\0001\000\000", "\000a\000b\000\000"); + x2s("\000(\000a\000*\000\134\000Z\000)\000\134\0001\000\000", "\000a\000\000", 2, 2); + x2s("\000.\000(\000a\000*\000\134\000Z\000)\000\134\0001\000\000", "\000b\000a\000\000", 2, 4); + x3s("\000(\000.\000(\000a\000b\000c\000)\000\134\0002\000)\000\000", "\000z\000a\000b\000c\000a\000b\000c\000\000", 0, 14, 1); + x3s("\000(\000.\000(\000.\000.\000\134\000d\000.\000)\000\134\0002\000)\000\000", "\000z\0001\0002\0003\0004\0001\0002\0003\0004\000\000", 0, 18, 1); + x2s("\000(\000(\000?\000i\000:\000a\000z\000)\000)\000\134\0001\000\000", "\000A\000z\000A\000z\000\000", 0, 8); + ns("\000(\000(\000?\000i\000:\000a\000z\000)\000)\000\134\0001\000\000", "\000A\000z\000a\000z\000\000"); + x2s("\000(\000?\000<\000=\000a\000)\000b\000\000", "\000a\000b\000\000", 2, 4); + ns("\000(\000?\000<\000=\000a\000)\000b\000\000", "\000b\000b\000\000"); + x2s("\000(\000?\000<\000=\000a\000|\000b\000)\000b\000\000", "\000b\000b\000\000", 2, 4); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000)\000b\000\000", "\000b\000c\000b\000\000", 4, 6); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000)\000b\000\000", "\000a\000b\000\000", 2, 4); + x2s("\000(\000?\000<\000=\000a\000|\000b\000c\000|\000|\000d\000e\000f\000g\000h\000i\000j\000|\000k\000l\000m\000n\000o\000p\000q\000|\000r\000)\000z\000\000", "\000r\000z\000\000", 2, 4); + x2s("\000(\000a\000)\000\134\000g\000<\0001\000>\000\000", "\000a\000a\000\000", 0, 4); + x2s("\000(\000?\000<\000!\000a\000)\000b\000\000", "\000c\000b\000\000", 2, 4); + ns("\000(\000?\000<\000!\000a\000)\000b\000\000", "\000a\000b\000\000"); + x2s("\000(\000?\000<\000!\000a\000|\000b\000c\000)\000b\000\000", "\000b\000b\000b\000\000", 0, 2); + ns("\000(\000?\000<\000!\000a\000|\000b\000c\000)\000z\000\000", "\000b\000c\000z\000\000"); + x2s("\000(\000?\000<\000n\000a\000m\000e\0001\000>\000a\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000<\000n\000a\000m\000e\000_\0002\000>\000a\000b\000)\000\134\000g\000<\000n\000a\000m\000e\000_\0002\000>\000\000", "\000a\000b\000a\000b\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000a\000m\000e\000_\0003\000>\000.\000z\000v\000.\000)\000\134\000k\000<\000n\000a\000m\000e\000_\0003\000>\000\000", "\000a\000z\000v\000b\000a\000z\000v\000b\000\000", 0, 16); + x2s("\000(\000?\000<\000=\000\134\000g\000<\000a\000b\000>\000)\000|\000-\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000a\000b\000>\000X\000y\000Z\000)\000\000", "\000X\000y\000Z\000\000", 6, 6); + x2s("\000(\000?\000<\000n\000>\000|\000a\000\134\000g\000<\000n\000>\000)\000+\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000<\000n\000>\000|\000\134\000(\000\134\000g\000<\000n\000>\000\134\000)\000)\000+\000$\000\000", "\000(\000)\000(\000(\000)\000)\000\000", 0, 12); + x3s("\000\134\000g\000<\000n\000>\000(\000?\000<\000n\000>\000.\000)\000{\0000\000}\000\000", "\000X\000\000", 0, 2, 1); + x2s("\000\134\000g\000<\000n\000>\000(\000a\000b\000c\000|\000d\000f\000(\000?\000<\000n\000>\000.\000Y\000Z\000)\000{\0002\000,\0008\000}\000)\000{\0000\000}\000\000", "\000X\000Y\000Z\000\000", 0, 6); + x2s("\000\134\000A\000(\000?\000<\000n\000>\000(\000a\000\134\000g\000<\000n\000>\000)\000|\000)\000\134\000z\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000>\000|\000\134\000g\000<\000m\000>\000\134\000g\000<\000n\000>\000)\000\134\000z\000|\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000m\000>\000a\000|\000(\000b\000)\000\134\000g\000<\000m\000>\000)\000\000", "\000b\000b\000b\000b\000a\000b\000b\000a\000\000", 0, 16); + x2s("\000(\000?\000<\000n\000a\000m\000e\0001\0002\0004\0000\000>\000\134\000w\000+\000\134\000s\000x\000)\000a\000+\000\134\000k\000<\000n\000a\000m\000e\0001\0002\0004\0000\000>\000\000", "\000 \000 \000f\000g\000 \000x\000a\000a\000a\000a\000a\000a\000a\000a\000f\000g\000 \000x\000\000", 4, 36); + x3s("\000(\000z\000)\000(\000)\000(\000)\000(\000?\000<\000_\0009\000>\000a\000)\000\134\000g\000<\000_\0009\000>\000\000", "\000z\000a\000a\000\000", 4, 6, 1); + x2s("\000(\000.\000)\000(\000(\000(\000?\000<\000_\000>\000a\000)\000)\000)\000\134\000k\000<\000_\000>\000\000", "\000z\000a\000a\000\000", 0, 6); + x2s("\000(\000(\000?\000<\000n\000a\000m\000e\0001\000>\000\134\000d\000)\000|\000(\000?\000<\000n\000a\000m\000e\0002\000>\000\134\000w\000)\000)\000(\000\134\000k\000<\000n\000a\000m\000e\0001\000>\000|\000\134\000k\000<\000n\000a\000m\000e\0002\000>\000)\000\000", "\000f\000f\000\000", 0, 4); + x2s("\000(\000?\000:\000(\000?\000<\000x\000>\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000?\000<\000x\000>\000a\000b\000c\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000a\000b\000c\000e\000f\000g\000e\000f\000g\000\000", 6, 18); + ns("\000(\000?\000:\000(\000?\000<\000x\000>\000a\000b\000c\000)\000|\000(\000?\000<\000x\000>\000e\000f\000g\000)\000)\000\134\000k\000<\000x\000>\000\000", "\000a\000b\000c\000e\000f\000g\000\000"); + x2s("\000(\000?\000:\000(\000?\000<\000n\0001\000>\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000)\000\134\000k\000<\000n\0001\000>\000$\000\000", "\000a\000-\000p\000y\000u\000m\000p\000y\000u\000m\000\000", 4, 20); + x3s("\000(\000?\000:\000(\000?\000<\000n\0001\000>\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000|\000(\000?\000<\000n\0001\000>\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000.\000)\000)\000\134\000k\000<\000n\0001\000>\000$\000\000", "\000x\000x\000x\000x\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000l\000m\000n\000a\000b\000c\000d\000e\000f\000g\000h\000i\000j\000k\000l\000m\000n\000\000", 8, 36, 14); + x3s("\000(\000?\000<\000n\000a\000m\000e\0001\000>\000)\000(\000?\000<\000n\000a\000m\000e\0002\000>\000)\000(\000?\000<\000n\000a\000m\000e\0003\000>\000)\000(\000?\000<\000n\000a\000m\000e\0004\000>\000)\000(\000?\000<\000n\000a\000m\000e\0005\000>\000)\000(\000?\000<\000n\000a\000m\000e\0006\000>\000)\000(\000?\000<\000n\000a\000m\000e\0007\000>\000)\000(\000?\000<\000n\000a\000m\000e\0008\000>\000)\000(\000?\000<\000n\000a\000m\000e\0009\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0000\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0001\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0002\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0003\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0004\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0005\000>\000)\000(\000?\000<\000n\000a\000m\000e\0001\0006\000>\000a\000a\000a\000)\000(\000?\000<\000n\000a\000m\000e\0001\0007\000>\000)\000$\000\000", "\000a\000a\000a\000\000", 0, 6, 16); + x2s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000a\000\000", 0, 2); + x2s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(\000a\000)\000)\000)\000)\000)\000)\000\000", 0, 26); + x3s("\000(\000?\000<\000f\000o\000o\000>\000a\000|\000\134\000(\000\134\000g\000<\000f\000o\000o\000>\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(\000(\000(\000a\000)\000)\000)\000)\000)\000)\000)\000)\000\000", 0, 34, 1); + x2s("\000\134\000g\000<\000b\000a\000r\000>\000|\000\134\000z\000E\000N\000D\000(\000?\000<\000b\000a\000r\000>\000.\000*\000a\000b\000c\000$\000)\000\000", "\000a\000b\000c\000x\000x\000x\000a\000b\000c\000\000", 0, 18); + x2s("\000\134\000g\000<\0001\000>\000|\000\134\000z\000E\000N\000D\000(\000.\000a\000.\000)\000\000", "\000b\000a\000c\000\000", 0, 6); + x3s("\000\134\000g\000<\000_\000A\000>\000\134\000g\000<\000_\000A\000>\000|\000\134\000z\000E\000N\000D\000(\000.\000a\000.\000)\000(\000?\000<\000_\000A\000>\000.\000b\000.\000)\000\000", "\000x\000b\000x\000y\000b\000y\000\000", 6, 12, 1); + x2s("\000\134\000A\000(\000?\000:\000\134\000g\000<\000p\000o\000n\000>\000|\000\134\000g\000<\000p\000a\000n\000>\000|\000\134\000z\000E\000N\000D\000 \000 \000(\000?\000<\000p\000a\000n\000>\000a\000|\000c\000\134\000g\000<\000p\000o\000n\000>\000c\000)\000(\000?\000<\000p\000o\000n\000>\000b\000|\000d\000\134\000g\000<\000p\000a\000n\000>\000d\000)\000)\000$\000\000", "\000c\000d\000c\000b\000c\000d\000c\000\000", 0, 14); + x2s("\000\134\000A\000(\000?\000<\000n\000>\000|\000a\000\134\000g\000<\000m\000>\000)\000\134\000z\000|\000\134\000z\000E\000N\000D\000 \000(\000?\000<\000m\000>\000\134\000g\000<\000n\000>\000)\000\000", "\000a\000a\000a\000a\000\000", 0, 8); + x2s("\000(\000?\000<\000n\000>\000(\000a\000|\000b\000\134\000g\000<\000n\000>\000c\000)\000{\0003\000,\0005\000}\000)\000\000", "\000b\000a\000a\000a\000a\000c\000a\000\000", 2, 10); + x2s("\000(\000?\000<\000n\000>\000(\000a\000|\000b\000\134\000g\000<\000n\000>\000c\000)\000{\0003\000,\0005\000}\000)\000\000", "\000b\000a\000a\000a\000a\000c\000a\000a\000a\000a\000a\000\000", 0, 20); + x2s("\000(\000?\000<\000p\000a\000r\000e\000>\000\134\000(\000(\000[\000^\000\134\000(\000\134\000)\000]\000+\000+\000|\000\134\000g\000<\000p\000a\000r\000e\000>\000)\000*\000+\000\134\000)\000)\000\000", "\000(\000(\000a\000)\000)\000\000", 0, 10); + x2s("\000(\000)\000*\000\134\0001\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000)\000*\000\134\0001\000\134\0002\000\000", "\000\000", 0, 0); + x3s("\000(\000?\000:\000\134\0001\000a\000|\000(\000)\000)\000*\000\000", "\000a\000\000", 0, 0, 1); + x2s("\000x\000(\000(\000.\000)\000*\000)\000*\000x\000\000", "\0000\000x\0001\000x\0002\000x\0003\000\000", 2, 12); + x2s("\000x\000(\000(\000.\000)\000*\000)\000*\000x\000(\000?\000i\000:\000\134\0001\000)\000\134\000Z\000\000", "\0000\000x\0001\000x\0002\000x\0001\000X\0002\000\000", 2, 18); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000)\000)\000*\000\134\0002\000\134\0005\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\000(\000)\000|\000(\000)\000|\000(\000)\000|\000(\000x\000)\000|\000(\000)\000|\000(\000)\000)\000*\000\134\0002\000b\000\134\0005\000\000", "\000b\000\000", 0, 2); + x2s("\217\372\000\000", "\217\372\000\000", 0, 2); + x2s("\000\000", "0B\000\000", 0, 0); + x2s("0B\000\000", "0B\000\000", 0, 2); + ns("0D\000\000", "0B\000\000"); + x2s("0F0F\000\000", "0F0F\000\000", 0, 4); + x2s("0B0D0F\000\000", "0B0D0F\000\000", 0, 6); + x2s("0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S\000\000", "0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S0S\000\000", 0, 70); + x2s("0B\000\000", "0D0B\000\000", 2, 4); + x2s("0D0F\000\000", "0B0D0F\000\000", 2, 6); + x2s("e\207\000\000", "e\207\000\000", 0, 2); + x2s("\000.\000\000", "0B\000\000", 0, 2); + x2s("\000.\000.\000\000", "0K0M\000\000", 0, 4); + x2s("\000\134\000w\000\000", "0J\000\000", 0, 2); + ns("\000\134\000W\000\000", "0B\000\000"); + x2s("\000[\000\134\000W\000]\000\000", "0F\000$\000\000", 2, 4); + x2s("\000\134\000S\000\000", "0]\000\000", 0, 2); + x2s("\000\134\000S\000\000", "o\042\000\000", 0, 2); + x2s("\000\134\000b\000\000", "l\027\000 \000\000", 0, 0); + x2s("\000\134\000b\000\000", "\000 0{\000\000", 2, 2); + x2s("\000\134\000B\000\000", "0[0]\000 \000\000", 2, 2); + x2s("\000\134\000B\000\000", "0F\000 \000\000", 4, 4); + x2s("\000\134\000B\000\000", "\000 0D\000\000", 0, 0); + x2s("\000[0_0a\000]\000\000", "0a\000\000", 0, 2); + ns("\000[0j0k\000]\000\000", "0l\000\000"); + x2s("\000[0F\000-0J\000]\000\000", "0H\000\000", 0, 2); + ns("\000[\000^0Q\000]\000\000", "0Q\000\000"); + x2s("\000[\000\134\000w\000]\000\000", "0m\000\000", 0, 2); + ns("\000[\000\134\000d\000]\000\000", "0u\000\000"); + x2s("\000[\000\134\000D\000]\000\000", "0o\000\000", 0, 2); + ns("\000[\000\134\000s\000]\000\000", "0O\000\000"); + x2s("\000[\000\134\000S\000]\000\000", "0x\000\000", 0, 2); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "0\210\000\000", 0, 2); + x2s("\000[\000\134\000w\000\134\000d\000]\000\000", "\000 \000 \000 0\210\000\000", 6, 8); + ns("\000\134\000w\233<\216\312\000\000", "\000 \233<\216\312\000\000"); + x2s("\233<\000\134\000W\216\312\000\000", "\233<\000 \216\312\000\000", 0, 6); + x2s("0B\000.0D\000.0F\000\000", "0B0B0D0D0F\000\000", 0, 10); + x2s("\000.\000\134\000w0F\000\134\000W\000.\000.0^\000\000", "0H0F0F\000 0F0^0^\000\000", 0, 14); + x2s("\000\134\000s\000\134\000w0S0S0S\000\000", "\000 0S0S0S0S\000\000", 0, 10); + x2s("0B0B\000.0Q\000\000", "0B0B0Q0Q\000\000", 0, 8); + ns("\000.0D\000\000", "0D0H\000\000"); + x2s("\000.0J\000\000", "0J0J\000\000", 0, 4); + x2s("\000^0B\000\000", "0B\000\000", 0, 2); + x2s("\000^0\200\000$\000\000", "0\200\000\000", 0, 2); + x2s("\000^\000\134\000w\000$\000\000", "0k\000\000", 0, 2); + x2s("\000^\000\134\000w0K0M0O0Q0S\000$\000\000", "\000z0K0M0O0Q0S\000\000", 0, 12); + x2s("\000^\000\134\000w\000.\000.\000.0F0H0J\000$\000\000", "\000z0B0D0F0F0H0J\000\000", 0, 14); + x2s("\000\134\000w\000\134\000w\000\134\000s\000\134\000W0J0J0J\000\134\000d\000\000", "\000a0J\000 \000 0J0J0J\0004\000\000", 0, 16); + x2s("\000\134\000A0_0a0d\000\000", "0_0a0d\000\000", 0, 6); + x2s("0\2000\2010\202\000\134\000Z\000\000", "0\2000\2010\202\000\000", 0, 6); + x2s("0K0M0O\000\134\000z\000\000", "0K0M0O\000\000", 0, 6); + x2s("0K0M0O\000\134\000Z\000\000", "0K0M0O\000\012\000\000", 0, 6); + x2s("\000\134\000G0}0t\000\000", "0}0t\000\000", 0, 4); + ns("\000\134\000G0H\000\000", "0F0H0J\000\000"); + ns("0h0f\000\134\000G\000\000", "0h0f\000\000"); + ns("0~0\177\000\134\000A\000\000", "0~0\177\000\000"); + ns("0~\000\134\000A0\177\000\000", "0~0\177\000\000"); + x2s("\000(\000?\000=0[\000)0[\000\000", "0[\000\000", 0, 2); + ns("\000(\000?\000=0F\000)\000.\000\000", "0D\000\000"); + x2s("\000(\000?\000!0F\000)0K\000\000", "0K\000\000", 0, 2); + ns("\000(\000?\000!0h\000)0B\000\000", "0h\000\000"); + x2s("\000(\000?\000i\000:0B\000)\000\000", "0B\000\000", 0, 2); + x2s("\000(\000?\000i\000:0v0y\000)\000\000", "0v0y\000\000", 0, 4); + ns("\000(\000?\000i\000:0D\000)\000\000", "0F\000\000"); + x2s("\000(\000?\000m\000:0\210\000.\000)\000\000", "0\210\000\012\000\000", 0, 4); + x2s("\000(\000?\000m\000:\000.0\201\000)\000\000", "0~\000\0120\201\000\000", 2, 6); + x2s("0B\000?\000\000", "\000\000", 0, 0); + x2s("Y\011\000?\000\000", "S\026\000\000", 0, 0); + x2s("Y\011\000?\000\000", "Y\011\000\000", 0, 2); + x2s("\221\317\000*\000\000", "\000\000", 0, 0); + x2s("\221\317\000*\000\000", "\221\317\000\000", 0, 2); + x2s("[P\000*\000\000", "[P[P[P\000\000", 0, 6); + x2s("\231\254\000*\000\000", "\236\177\231\254\231\254\231\254\231\254\000\000", 0, 0); + ns("\134q\000+\000\000", "\000\000"); + x2s("l\263\000+\000\000", "l\263\000\000", 0, 2); + x2s("fB\000+\000\000", "fBfBfBfB\000\000", 0, 8); + x2s("0H\000+\000\000", "0H0H0F0F0F\000\000", 0, 4); + x2s("0F\000+\000\000", "0J0F0F0F0F\000\000", 2, 10); + x2s("\000.\000?\000\000", "0_\000\000", 0, 2); + x2s("\000.\000*\000\000", "0q0t0w0z\000\000", 0, 8); + x2s("\000.\000+\000\000", "0\215\000\000", 0, 2); + x2s("\000.\000+\000\000", "0D0F0H0K\000\012\000\000", 0, 8); + x2s("0B\000|0D\000\000", "0B\000\000", 0, 2); + x2s("0B\000|0D\000\000", "0D\000\000", 0, 2); + x2s("0B0D\000|0D0F\000\000", "0B0D\000\000", 0, 4); + x2s("0B0D\000|0D0F\000\000", "0D0F\000\000", 0, 4); + x2s("0\222\000(\000?\000:0K0M\000|0M0O\000)\000\000", "0\2220K0M\000\000", 0, 6); + x2s("0\222\000(\000?\000:0K0M\000|0M0O\000)0Q\000\000", "0\2220M0O0Q\000\000", 0, 8); + x2s("0B0D\000|\000(\000?\000:0B0F\000|0B0\222\000)\000\000", "0B0\222\000\000", 0, 4); + x2s("0B\000|0D\000|0F\000\000", "0H0F\000\000", 2, 4); + x2s("0B\000|0D\000|0F0H\000|0J0K0M\000|0O\000|0Q0S0U\000|0W0Y0[\000|0]\000|0_0a\000|0d0f0h0j0k\000|0l0m\000\000", "0W0Y0[\000\000", 0, 6); + ns("0B\000|0D\000|0F0H\000|0J0K0M\000|0O\000|0Q0S0U\000|0W0Y0[\000|0]\000|0_0a\000|0d0f0h0j0k\000|0l0m\000\000", "0Y0[\000\000"); + x2s("0B\000|\000^0\217\000\000", "0v0B\000\000", 2, 4); + x2s("0B\000|\000^0\222\000\000", "0\2220B\000\000", 0, 2); + x2s("\233<\000|\000\134\000G\216\312\000\000", "0Q\216\312\233<\000\000", 4, 6); + x2s("\233<\000|\000\134\000G\216\312\000\000", "\216\312\233<\000\000", 0, 2); + x2s("\233<\000|\000\134\000A\216\312\000\000", "\000b\216\312\233<\000\000", 4, 6); + x2s("\233<\000|\000\134\000A\216\312\000\000", "\216\312\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\233<\000\000", 2, 4); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000Z\000\000", "\216\312\000\012\000\000", 0, 2); + x2s("\233<\000|\216\312\000\134\000z\000\000", "\216\312\233<\000\000", 2, 4); + x2s("\233<\000|\216\312\000\134\000z\000\000", "\216\312\000\000", 0, 2); + x2s("\000\134\000w\000|\000\134\000s\000\000", "0J\000\000", 0, 2); + x2s("\000\134\000w\000|\000%\000\000", "\000%0J\000\000", 0, 2); + x2s("\000\134\000w\000|\000[\000&\000$\000]\000\000", "0F\000&\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000\000", "0F\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000|\000[\000^0K\000-0S\000]\000\000", "0B\000\000", 0, 2); + x2s("\000[0D\000-0Q\000]\000|\000[\000^0K\000-0S\000]\000\000", "0K\000\000", 0, 2); + x2s("\000[\000^0B\000]\000\000", "\000\012\000\000", 0, 2); + x2s("\000(\000?\000:0B\000|\000[0F\000-0M\000]\000)\000|0D0\222\000\000", "0F0\222\000\000", 0, 2); + x2s("\000(\000?\000:0B\000|\000[0F\000-0M\000]\000)\000|0D0\222\000\000", "0D0\222\000\000", 0, 4); + x2s("0B0D0F\000|\000(\000?\000=0Q0Q\000)\000.\000.0{\000\000", "0Q0Q0{\000\000", 0, 6); + x2s("0B0D0F\000|\000(\000?\000!0Q0Q\000)\000.\000.0{\000\000", "0B0D0{\000\000", 0, 6); + x2s("\000(\000?\000=0\2220B\000)\000.\000.0B\000|\000(\000?\000=0\2220\222\000)\000.\000.0B\000\000", "0\2220\2220B\000\000", 0, 6); + x2s("\000(\000?\000<\000=0B\000|0D0F\000)0D\000\000", "0D0F0D\000\000", 4, 6); + ns("\000(\000?\000>0B\000|0B0D0H\000)0F\000\000", "0B0D0H0F\000\000"); + x2s("\000(\000?\000>0B0D0H\000|0B\000)0F\000\000", "0B0D0H0F\000\000", 0, 8); + x2s("0B\000?\000|0D\000\000", "0B\000\000", 0, 2); + x2s("0B\000?\000|0D\000\000", "0D\000\000", 0, 0); + x2s("0B\000?\000|0D\000\000", "\000\000", 0, 0); + x2s("0B\000*\000|0D\000\000", "0B0B\000\000", 0, 4); + x2s("0B\000*\000|0D\000*\000\000", "0D0B\000\000", 0, 0); + x2s("0B\000*\000|0D\000*\000\000", "0B0D\000\000", 0, 2); + x2s("\000[\000a0B\000]\000*\000|0D\000*\000\000", "\000a0B0D0D0D\000\000", 0, 4); + x2s("0B\000+\000|0D\000*\000\000", "\000\000", 0, 0); + x2s("0B\000+\000|0D\000*\000\000", "0D0D0D\000\000", 0, 6); + x2s("0B\000+\000|0D\000*\000\000", "0B0D0D0D\000\000", 0, 2); + x2s("0B\000+\000|0D\000*\000\000", "\000a0B0D0D0D\000\000", 0, 0); + ns("0B\000+\000|0D\000+\000\000", "\000\000"); + x2s("\000(0B\000|0D\000)\000?\000\000", "0D\000\000", 0, 2); + x2s("\000(0B\000|0D\000)\000*\000\000", "0D0B\000\000", 0, 4); + x2s("\000(0B\000|0D\000)\000+\000\000", "0D0B0D\000\000", 0, 6); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0F0B0B0D0F0H\000\000", 0, 8); + x2s("\000(0B0D\000|0F0H\000)\000+\000\000", "0F0B0B0D0F0H\000\000", 4, 12); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0B0B0D0F0B\000\000", 2, 10); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "0B0D0\2220F0B\000\000", 0, 4); + x2s("\000(0B0D\000|0F0B\000)\000+\000\000", "\000$\000$\000z\000z\000z\000z0B0D0\2220F0B\000\000", 12, 16); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0B0D0B0D0B\000\000", 0, 10); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0D0B\000\000", 2, 4); + x2s("\000(0B\000|0D0B0D\000)\000+\000\000", "0D0B0B0B0D0B\000\000", 2, 8); + x2s("\000(\000?\000:0B\000|0D\000)\000(\000?\000:0B\000|0D\000)\000\000", "0B0D\000\000", 0, 4); + x2s("\000(\000?\000:0B\000*\000|0D\000*\000)\000(\000?\000:0B\000*\000|0D\000*\000)\000\000", "0B0B0B0D0D0D\000\000", 0, 6); + x2s("\000(\000?\000:0B\000*\000|0D\000*\000)\000(\000?\000:0B\000+\000|0D\000+\000)\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|0D\000+\000)\000{\0002\000}\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|0D\000+\000)\000{\0001\000,\0002\000}\000\000", "0B0B0B0D0D0D\000\000", 0, 12); + x2s("\000(\000?\000:0B\000+\000|\000\134\000A0D\000*\000)0F0F\000\000", "0F0F\000\000", 0, 4); + ns("\000(\000?\000:0B\000+\000|\000\134\000A0D\000*\000)0F0F\000\000", "0B0D0F0F\000\000"); + x2s("\000(\000?\000:\000^0B\000+\000|0D\000+\000)\000*0F\000\000", "0B0B0D0D0D0B0D0F\000\000", 12, 16); + x2s("\000(\000?\000:\000^0B\000+\000|0D\000+\000)\000*0F\000\000", "0B0B0D0D0D0D0F\000\000", 0, 14); + x2s("0F\000{\0000\000,\000}\000\000", "0F0F0F0F\000\000", 0, 8); + x2s("0B\000|\000(\000?\000i\000)\000c\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000)\000c\000|0B\000\000", "\000C\000\000", 0, 2); + x2s("\000(\000?\000i\000:0B\000)\000|\000a\000\000", "\000a\000\000", 0, 2); + ns("\000(\000?\000i\000:0B\000)\000|\000a\000\000", "\000A\000\000"); + x2s("\000[0B0D0F\000]\000?\000\000", "0B0D0F\000\000", 0, 2); + x2s("\000[0B0D0F\000]\000*\000\000", "0B0D0F\000\000", 0, 6); + x2s("\000[\000^0B0D0F\000]\000*\000\000", "0B0D0F\000\000", 0, 0); + ns("\000[\000^0B0D0F\000]\000+\000\000", "0B0D0F\000\000"); + x2s("0B\000?\000?\000\000", "0B0B0B\000\000", 0, 0); + x2s("0D0B\000?\000?0D\000\000", "0D0B0D\000\000", 0, 6); + x2s("0B\000*\000?\000\000", "0B0B0B\000\000", 0, 0); + x2s("0D0B\000*\000?\000\000", "0D0B0B\000\000", 0, 2); + x2s("0D0B\000*\000?0D\000\000", "0D0B0B0D\000\000", 0, 8); + x2s("0B\000+\000?\000\000", "0B0B0B\000\000", 0, 2); + x2s("0D0B\000+\000?\000\000", "0D0B0B\000\000", 0, 4); + x2s("0D0B\000+\000?0D\000\000", "0D0B0B0D\000\000", 0, 8); + x2s("\000(\000?\000:Y)\000?\000)\000?\000?\000\000", "Y)\000\000", 0, 0); + x2s("\000(\000?\000:Y)\000?\000?\000)\000?\000\000", "Y)\000\000", 0, 0); + x2s("\000(\000?\000:Y\042\000?\000)\000+\000?\000\000", "Y\042Y\042Y\042\000\000", 0, 2); + x2s("\000(\000?\000:\230\250\000+\000)\000?\000?\000\000", "\230\250\230\250\230\250\000\000", 0, 0); + x2s("\000(\000?\000:\226\352\000+\000)\000?\000?\227\034\000\000", "\226\352\226\352\226\352\227\034\000\000", 0, 8); + x2s("\000(\000?\000:0B0D\000)\000?\000{\0002\000}\000\000", "\000\000", 0, 0); + x2s("\000(\000?\000:\233<\216\312\000)\000?\000{\0002\000}\000\000", "\233<\216\312\233<\216\312\233<\000\000", 0, 8); + x2s("\000(\000?\000:\233<\216\312\000)\000*\000{\0000\000}\000\000", "\233<\216\312\233<\216\312\233<\000\000", 0, 0); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0003\000,\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 16); + ns("\000(\000?\000:\233<\216\312\000)\000{\0003\000,\000}\000\000", "\233<\216\312\233<\216\312\000\000"); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 12); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 16); + x2s("\000(\000?\000:\233<\216\312\000)\000{\0002\000,\0004\000}\000?\000\000", "\233<\216\312\233<\216\312\233<\216\312\233<\216\312\233<\216\312\000\000", 0, 8); + x2s("\000(\000?\000:\233<\216\312\000)\000{\000,\000}\000\000", "\233<\216\312\000{\000,\000}\000\000", 0, 10); + x2s("\000(\000?\000:0K0M0O\000)\000+\000?\000{\0002\000}\000\000", "0K0M0O0K0M0O0K0M0O\000\000", 0, 12); + x3s("\000(pk\000)\000\000", "pk\000\000", 0, 2, 1); + x3s("\000(pkl4\000)\000\000", "pkl4\000\000", 0, 4, 1); + x2s("\000(\000(fB\225\223\000)\000)\000\000", "fB\225\223\000\000", 0, 4); + x3s("\000(\000(\230\250l4\000)\000)\000\000", "\230\250l4\000\000", 0, 4, 1); + x3s("\000(\000(f(e\345\000)\000)\000\000", "f(e\345\000\000", 0, 4, 2); + x3s("\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\000(\221\317[P\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000)\000\000", "\221\317[P\000\000", 0, 4, 20); + x3s("\000(0B0D\000)\000(0F0H\000)\000\000", "0B0D0F0H\000\000", 0, 4, 1); + x3s("\000(0B0D\000)\000(0F0H\000)\000\000", "0B0D0F0H\000\000", 4, 8, 2); + x3s("\000(\000)\000(0B\000)0D0F\000(0H0J0K\000)0M0O0Q0S\000\000", "0B0D0F0H0J0K0M0O0Q0S\000\000", 6, 12, 3); + x3s("\000(\000(\000)\000(0B\000)0D0F\000(0H0J0K\000)0M0O0Q0S\000)\000\000", "0B0D0F0H0J0K0M0O0Q0S\000\000", 6, 12, 4); + x3s("\000.\000*\000(0\3250\251\000)0\3630\3730\336\000(0\363\000(\000)0\2670\3450\277\000)0\2440\363\000\000", "0\3250\2510\3630\3730\3360\3630\2670\3450\2770\2440\363\000\000", 10, 18, 2); + x2s("\000(\000^0B\000)\000\000", "0B\000\000", 0, 2); + x3s("\000(0B\000)\000|\000(0B\000)\000\000", "0D0B\000\000", 2, 4, 1); + x3s("\000(\000^0B\000)\000|\000(0B\000)\000\000", "0D0B\000\000", 2, 4, 2); + x3s("\000(0B\000?\000)\000\000", "0B0B0B\000\000", 0, 2, 1); + x3s("\000(0~\000*\000)\000\000", "0~0~0~\000\000", 0, 6, 1); + x3s("\000(0h\000*\000)\000\000", "\000\000", 0, 0, 1); + x3s("\000(0\213\000+\000)\000\000", "0\2130\2130\2130\2130\2130\2130\213\000\000", 0, 14, 1); + x3s("\000(0u\000+\000|0x\000*\000)\000\000", "0u0u0u0x0x\000\000", 0, 6, 1); + x3s("\000(0B\000+\000|0D\000?\000)\000\000", "0D0D0D0B0B\000\000", 0, 2, 1); + x3s("\000(0B0D0F\000)\000?\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0B0D0F\000)\000*\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0B0D0F\000)\000+\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(0U0W0Y\000|0B0D0F\000)\000+\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(\000[0j0k0l\000]\000[0K0M0O\000]\000|0K0M0O\000)\000+\000\000", "0K0M0O\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000i\000:0B0D0F\000)\000)\000\000", "0B0D0F\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000m\000:0B\000.0F\000)\000)\000\000", "0B\000\0120F\000\000", 0, 6, 1); + x3s("\000(\000(\000?\000=0B0\223\000)0B\000)\000\000", "0B0\2230D\000\000", 0, 2, 1); + x3s("0B0D0F\000|\000(\000.0B0D0H\000)\000\000", "0\2230B0D0H\000\000", 0, 8, 1); + x3s("0B\000*\000(\000.\000)\000\000", "0B0B0B0B0\223\000\000", 8, 10, 1); + x3s("0B\000*\000?\000(\000.\000)\000\000", "0B0B0B0B0\223\000\000", 0, 2, 1); + x3s("0B\000*\000?\000(0\223\000)\000\000", "0B0B0B0B0\223\000\000", 8, 10, 1); + x3s("\000[0D0F0H\000]0B\000*\000(\000.\000)\000\000", "0H0B0B0B0B0\223\000\000", 10, 12, 1); + x3s("\000(\000\134\000A0D0D\000)0F0F\000\000", "0D0D0F0F\000\000", 0, 4, 1); + ns("\000(\000\134\000A0D0D\000)0F0F\000\000", "0\2230D0D0F0F\000\000"); + x3s("\000(\000^0D0D\000)0F0F\000\000", "0D0D0F0F\000\000", 0, 4, 1); + ns("\000(\000^0D0D\000)0F0F\000\000", "0\2230D0D0F0F\000\000"); + x3s("0\2150\215\000(0\2130\213\000$\000)\000\000", "0\2150\2150\2130\213\000\000", 4, 8, 1); + ns("0\2150\215\000(0\2130\213\000$\000)\000\000", "0\2150\2150\2130\2130\213\000\000"); + x2s("\000(q!\000)\000\134\0001\000\000", "q!q!\000\000", 0, 4); + ns("\000(q!\000)\000\134\0001\000\000", "q!kf\000\000"); + x2s("\000(zz\000?\000)\000\134\0001\000\000", "zzzz\000\000", 0, 4); + x2s("\000(zz\000?\000?\000)\000\134\0001\000\000", "zzzz\000\000", 0, 0); + x2s("\000(zz\000*\000)\000\134\0001\000\000", "zzzzzzzzzz\000\000", 0, 8); + x3s("\000(zz\000*\000)\000\134\0001\000\000", "zzzzzzzzzz\000\000", 0, 4, 1); + x2s("0B\000(0D\000*\000)\000\134\0001\000\000", "0B0D0D0D0D\000\000", 0, 10); + x2s("0B\000(0D\000*\000)\000\134\0001\000\000", "0B0D\000\000", 0, 2); + x2s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0001\000\134\0002\000\000", "0B0B0B0D0D0B0B0B0D0D\000\000", 0, 20); + x2s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0002\000\000", "0B0B0B0D0D0D0D\000\000", 0, 14); + x3s("\000(0B\000*\000)\000(0D\000*\000)\000\134\0002\000\000", "0B0B0B0D0D0D0D\000\000", 6, 10, 2); + x2s("\000(\000(\000(\000(\000(\000(\000(0}\000*\000)0z\000)\000)\000)\000)\000)\000)0t\000\134\0007\000\000", "0}0}0}0z0t0}0}0}\000\000", 0, 16); + x3s("\000(\000(\000(\000(\000(\000(\000(0}\000*\000)0z\000)\000)\000)\000)\000)\000)0t\000\134\0007\000\000", "0}0}0}0z0t0}0}0}\000\000", 0, 6, 7); + x2s("\000(0o\000)\000(0r\000)\000(0u\000)\000\134\0002\000\134\0001\000\134\0003\000\000", "0o0r0u0r0o0u\000\000", 0, 12); + x2s("\000(\000[0M\000-0Q\000]\000)\000\134\0001\000\000", "0O0O\000\000", 0, 4); + x2s("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "0B\0005\000 0B\0005\000 \000\000", 0, 12); + ns("\000(\000\134\000w\000\134\000d\000\134\000s\000)\000\134\0001\000\000", "0B\0005\000 0B\0005\000\000"); + x2s("\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "\212\260\377\037\212\260\377\037\000\000", 0, 8); + x2s("\000.\000.\000.\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "0B\000a0B\212\260\377\037\212\260\377\037\000\000", 0, 14); + x2s("\000(\212\260\377\037\000|\000[0B\000-0F\000]\000{\0003\000}\000)\000\134\0001\000\000", "0F0D0F0F0D0F\000\000", 0, 12); + x2s("\000(\000^0S\000)\000\134\0001\000\000", "0S0S\000\000", 0, 4); + ns("\000(\000^0\200\000)\000\134\0001\000\000", "0\2010\2000\200\000\000"); + ns("\000(0B\000$\000)\000\134\0001\000\000", "0B0B\000\000"); + ns("\000(0B0D\000\134\000Z\000)\000\134\0001\000\000", "0B0D\000\000"); + x2s("\000(0B\000*\000\134\000Z\000)\000\134\0001\000\000", "0B\000\000", 2, 2); + x2s("\000.\000(0B\000*\000\134\000Z\000)\000\134\0001\000\000", "0D0B\000\000", 2, 4); + x3s("\000(\000.\000(0\2040D0\206\000)\000\134\0002\000)\000\000", "\000z0\2040D0\2060\2040D0\206\000\000", 0, 14, 1); + x3s("\000(\000.\000(\000.\000.\000\134\000d\000.\000)\000\134\0002\000)\000\000", "0B\0001\0002\0003\0004\0001\0002\0003\0004\000\000", 0, 18, 1); + x2s("\000(\000(\000?\000i\000:0B\000v0Z\000)\000)\000\134\0001\000\000", "0B\000v0Z0B\000v0Z\000\000", 0, 12); + x2s("\000(\000?\000Y\011\000|\000\134\000(\000\134\000g\000\000\134\000)\000)\000\000", "\000(\000(\000(\000(\000(\000(Y\011\000)\000)\000)\000)\000)\000)\000\000", 0, 26); + x2s("\000\134\000A\000(\000?\000:\000\134\000g\000<\226?\000_\0001\000>\000|\000\134\000g\000\000|\000\134\000z}BN\206\000 \000 \000(\000?\000<\226?\000_\0001\000>\211\263\000|\201\352\000\134\000g\000\201\352\000)\000(\000?\000W(\000|\203\351\205\251\000\134\000g\000<\226?\000_\0001\000>\203\351\205\251\000)\000)\000$\000\000", "\203\351\205\251\201\352\203\351\205\251\201\352W(\201\352\203\351\205\251\201\352\203\351\205\251\000\000", 0, 26); + x2s("\000[\000[0r0u\000]\000]\000\000", "0u\000\000", 0, 2); + x2s("\000[\000[0D0J0F\000]0K\000]\000\000", "0K\000\000", 0, 2); + ns("\000[\000[\000^0B\000]\000]\000\000", "0B\000\000"); + ns("\000[\000^\000[0B\000]\000]\000\000", "0B\000\000"); + x2s("\000[\000^\000[\000^0B\000]\000]\000\000", "0B\000\000", 0, 2); + x2s("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0O\000\000", 0, 2); + ns("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0K\000\000"); + ns("\000[\000[0K0M0O\000]\000&\000&0M0O\000]\000\000", "0Q\000\000"); + x2s("\000[0B\000-0\223\000&\000&0D\000-0\222\000&\000&0F\000-0\221\000]\000\000", "0\221\000\000", 0, 2); + ns("\000[\000^0B\000-0\223\000&\000&0D\000-0\222\000&\000&0F\000-0\221\000]\000\000", "0\221\000\000"); + x2s("\000[\000[\000^0B\000&\000&0B\000]\000&\000&0B\000-0\223\000]\000\000", "0D\000\000", 0, 2); + ns("\000[\000[\000^0B\000&\000&0B\000]\000&\000&0B\000-0\223\000]\000\000", "0B\000\000"); + x2s("\000[\000[\000^0B\000-0\223\000&\000&0D0F0H0J\000]\000&\000&\000[\000^0F\000-0K\000]\000]\000\000", "0M\000\000", 0, 2); + ns("\000[\000[\000^0B\000-0\223\000&\000&0D0F0H0J\000]\000&\000&\000[\000^0F\000-0K\000]\000]\000\000", "0D\000\000"); + x2s("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0F\000\000", 0, 2); + x2s("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0H\000\000", 0, 2); + ns("\000[\000^\000[\000^0B0D0F\000]\000&\000&\000[\000^0F0H0J\000]\000]\000\000", "0K\000\000"); + x2s("\000[0B\000-\000&\000&\000-0B\000]\000\000", "\000-\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000q\000-\000w\000]\000\000", "0H\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\000f\000\000", 0, 2); + x2s("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\000g\000\000", 0, 2); + ns("\000[\000^\000[\000^\000a\000-\000z0B0D0F\000]\000&\000&\000[\000^\000b\000c\000d\000e\000f\000g0F0H0J\000]\000g\000-\000w\000]\000\000", "\0002\000\000"); + x2s("\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000\134\000/\000b\000>\000\000", "\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000/\000b\000>\000\000", 0, 40); + x2s("\000.\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000\134\000/\000b\000>\000\000", "\000a\000<\000b\000>0\3200\3740\2700\3470\3630n0\3000\2460\3630\3550\3740\311\000<\000/\000b\000>\000\000", 0, 40); + + // Unicode case folding tests + // common case folding: u+0041 and u+0061 + x2s("\u0000\u0041\000\000", "\u0000\u0061\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0000\u0061\000\000", "\u0000\u0041\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+00C0 and u+00E0 + x2s("\u0000\u00C0\000\000", "\u0000\u00E0\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0000\u00E0\000\000", "\u0000\u00C0\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+00B5 and u+03BC + x2s("\u0000\u00B5\000\000", "\u0003\u00BC\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0003\u00BC\000\000", "\u0000\u00B5\000\000", 0, 2, Option.IGNORECASE); + // common case folding: u+0073 and u+017F + x2s("\u0000\u0073\000\000", "\u0001\u007F\000\000", 0, 2, Option.IGNORECASE); + x2s("\u0001\u007F\000\000", "\u0000\u0073\000\000", 0, 2, Option.IGNORECASE); + // full case folding: u+1FA0 and u+1F60 u+03B9 + x2s("\u001F\u00A0\000\000", "\u001F\u0060\u0003\u00B9\000\000", 0, 4, Option.IGNORECASE); + x2s("\u001F\u0060\u0003\u00B9\000\000", "\u001F\u00A0\000\000", 0, 2, Option.IGNORECASE); + // full case folding: u+1FA8 and u+1F60 u+03B9 + x2s("\u001F\u00A8\000\000", "\u001F\u0060\u0003\u00B9\000\000", 0, 4, Option.IGNORECASE); + x2s("\u001F\u0060\u0003\u00B9\000\000", "\u001F\u00A8\000\000", 0, 2, Option.IGNORECASE); + // simple case folding: u+1FA8 and u+1FA0 + x2s("\u001F\u00A8\000\000", "\u001F\u00A0\000\000", 0, 2, Option.IGNORECASE); + x2s("\u001F\u00A0\000\000", "\u001F\u00A8\000\000", 0, 2, Option.IGNORECASE); + // FIXME: Case folding for 'LATIN CAPITAL LETTER SHARP S' not supported + // full case folding: u+1E9E and u+0073 u+0073 + x2s("\u001E\u009E\000\000", "\u0000\u0073\u0000\u0073\000\000", 0, 4, Option.IGNORECASE); + // simple case folding: u+1E9E and u+00DF + x2s("\u001E\u009E\000\000", "\u0000\u00DF\000\000", 0, 2, Option.IGNORECASE); + + // Case fold exceeding Analyser#THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION (= 8) + x2s("\u0000\u0041\u0000\u0041\u0000\u0041\u0000\u0041\000\000", "\u0000\u0061\u0000\u0061\u0000\u0061\u0000\u0061\000\000", 0, 8, Option.IGNORECASE); + + x2s("\000[\000\134\000x\000{\0000\000}\000-\000X\000]\000\000", "\0000\000\000", 0, 2, Option.IGNORECASE); + } +} diff --git a/third_party/joni/test/org/joni/test/TestU8.java b/third_party/joni/test/org/joni/test/TestU8.java new file mode 100755 index 000000000..e243a4b49 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestU8.java @@ -0,0 +1,359 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestU8 extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + @Override + public String testEncoding() { + return "utf-8"; + } + @Override + public Syntax syntax() { + return Syntax.TEST; + } + @Override + public void test() throws Exception { + xx("^\\d\\d\\d-".getBytes(), new byte []{-30, -126, -84, 48, 45}, 0, 0, 0, true); + x2s("x{2}", "xx", 0, 2, Option.IGNORECASE); + x2s("x{2}", "XX", 0, 2, Option.IGNORECASE); + x2s("x{3}", "XxX", 0, 3, Option.IGNORECASE); + ns("x{2}", "x", Option.IGNORECASE); + ns("x{2}", "X", Option.IGNORECASE); + + byte[] pat = new byte[] {(byte)227, (byte)131, (byte)160, (byte)40, (byte)46, (byte)41}; + byte[] str = new byte[]{(byte)227, (byte)130, (byte)185, (byte)227, (byte)131, (byte)145, (byte)227, (byte)131, (byte)160, (byte)227, (byte)131, (byte)143, (byte)227, (byte)131, (byte)179, (byte)227, (byte)130, (byte)175}; + + x2(pat, str, 6, 12); + + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 35, Option.IGNORECASE); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 0, 35, Option.IGNORECASE); + x2s("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAA", 0, 35, Option.IGNORECASE); + + pat = new byte[]{94, 40, (byte)239, (byte)188, (byte)161, 41, 92, 49, 36}; + str = new byte[]{(byte)239, (byte)188, (byte)161, 65}; + + n(pat, str, Option.IGNORECASE); + + pat = new byte[]{94, (byte)195, (byte)159, 123, 50, 125, 36}; + str = new byte[]{(byte)195, (byte)159, 115, 115}; + + x2(pat, str, 0, 4, Option.IGNORECASE); + + String str2 = new String(new byte[]{-61, -123, -61, -123}); + String pat2 = new String(new byte[]{'^', -61, -123, '{', '2', '}', '$'}); + + // x2s(pat2, str2, 4, 4); + // x2s(pat2, str2, 4, 4, Option.IGNORECASE); + + ns("(?i-mx:ak)a", "ema"); + + x2s("(?i:!\\[CDAT)", "![CDAT", 0, 6); + x2s("(?i:\\!\\[CDAa)", "\\![CDAa", 1, 7); + x2s("(?i:\\!\\[CDAb)", "\\![CDAb", 1, 7); + + x2s("\\R", "\u0085", 0, 2); + x2s("\\R", "\u2028", 0, 3); + x2s("\\R", "\u2029", 0, 3); + + x2s("\\A\\R\\z", "\r", 0, 1); + x2s("\\A\\R\\z", "\n", 0, 1); + x2s("\\A\\R\\z", "\r\n", 0, 2); + + x2s("foo\\b", "foo", 0, 3); + + x2s("(x?)x*\\1", "x", 0, 1, Option.IGNORECASE); + x2s("(x?)x*\\k<1+0>", "x", 0, 1, Option.IGNORECASE); + x2s("(?x?)(?x?)\\k", "x", 0, 1, Option.IGNORECASE); + + x2s("(?=((?)(\\k)))", "", 0, 0); + + x2s("a\\g<0>*z", "aaazzz", 0, 6); + + x2s("ab\\Kcd", "abcd", 2, 4); + x2s("ab\\Kc(\\Kd|z)", "abcd", 3, 4); + x2s("ab\\Kc(\\Kz|d)", "abcd", 2, 4); + x2s("(a\\K)*", "aaab", 3, 3); + x3s("(a\\K)*", "aaab", 2, 3, 1); + // x2s("a\\K?a", "aa", 0, 2); // error: differ from perl + x2s("ab(?=c\\Kd)", "abcd", 2, 2); // This behaviour is currently not well defined. (see: perlre) + x2s("(?<=a\\Kb|aa)cd", "abcd", 1, 4); // ... + x2s("(?<=ab|a\\Ka)cd", "abcd", 2, 4); // ... + + x2s("\\X", "\n", 0, 1); + x2s("\\X", "\r", 0, 1); + x2s("\\X{3}", "\r\r\n\n", 0, 4); + x2s("\\X", "\u306F\u309A\n", 0, 6); + x2s("\\A\\X\\z", "\u0020\u200d", 0, 4); + x2s("\\A\\X\\z", "\u0600\u0600", 0, 4); + x2s("\\A\\X\\z", "\u0600\u0020", 0, 3); + + x2s("\\A\\X\\z", " ‍", 0, 4); + x2s("\\A\\X\\z", "؀؀", 0, 4); + x2s("\\A\\X\\z", "؀", 0, 2); + x2s("\\A\\X\\z", "☝🏻", 0, 7); + x2s("\\A\\X\\z", "😀", 0, 4); + x2s("\\A\\X\\z", " ̈", 0, 3); // u{1f600} + + // u{20 200d} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)32, (byte)226, (byte)128, (byte)141}, 0, 4); + // u{600 600} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)216, (byte)128, (byte)216, (byte)128}, 0, 4); + // u{600 20} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)216, (byte)128, (byte)32}, 0, 3); + // u{261d 1F3FB} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)226, (byte)152, (byte)157, (byte)240, (byte)159, (byte)143, (byte)187}, 0, 7); + // u{1f600} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)152, (byte)128}, 0, 4); + // u{20 308} + x2s("\\A\\X\\z", " \u0308", 0, 3); + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)32, (byte)204, (byte)136}, 0, 3); + // u{a 308} + x2s("\\A\\X\\z", "a\u0308", 0, 3); + x2("\\A\\X\\X\\z".getBytes(), new byte[] {(byte)10, (byte)204, (byte)136}, 0, 3); + // u{d 308} + x2s("\\A\\X\\z", "d\u0308", 0, 3); + x2("\\A\\X\\X\\z".getBytes(), new byte[] {(byte)13, (byte)204, (byte)136}, 0, 3); + // u{1F477 1F3FF 200D 2640 FE0F} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)183, (byte)240, (byte)159, (byte)143, (byte)191, (byte)226, (byte)128, (byte)141, (byte)226, (byte)153, (byte)128, (byte)239, (byte)184, (byte)143}, 0, 17); + // u{1F468 200D 1F393} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)168, (byte)226, (byte)128, (byte)141, (byte)240, (byte)159, (byte)142, (byte)147}, 0, 11); + // u{1F46F 200D 2642 FE0F} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)175, (byte)226, (byte)128, (byte)141, (byte)226, (byte)153, (byte)130, (byte)239, (byte)184, (byte)143}, 0, 13); + // u{1f469 200d 2764 fe0f 200d 1f469} + x2("\\A\\X\\z".getBytes(), new byte[] {(byte)240, (byte)159, (byte)145, (byte)169, (byte)226, (byte)128, (byte)141, (byte)226, (byte)157, (byte)164, (byte)239, (byte)184, (byte)143, (byte)226, (byte)128, (byte)141, (byte)240, (byte)159, (byte)145, (byte)169}, 0, 20); + + x2s("\\A\\X\\X\\z", "\r\u0308", 0, 3); + x2s("\\A\\X\\X\\z", "\n\u0308", 0, 3); + + x2s("[0-9-a]+", " 0123456789-a ", 1, 13); + x2s("[0-9-\\s]+", " 0123456789-a ", 0, 12); + x2s("[0-9-あ\\\\/\u0001]+", " 0123456789-あ\\/\u0001 ", 1, 18); + x2s("[a-b-]+", "ab-", 0, 3); + x2s("[a-b-&&-]+", "ab-", 2, 3); + x2s("(?i)[a[b-あ]]+", "abあ", 0, 5); + x2s("(?i)[\\d[:^graph:]]+", "0あ", 0, 1); + x2s("(?ia)[\\d[:^print:]]+", "0あ", 0, 4); + + x2s("(?i:a) B", "a B", 0, 3); + x2s("(?i:a )B", "a B", 0, 3); + x2s("B (?i:a)", "B a", 0, 3); + x2s("B(?i: a)", "B a", 0, 3); + + x2s("(?a)[\\p{Space}\\d]", "\u00a0", 0, 2); + x2s("(?a)[\\d\\p{Space}]", "\u00a0", 0, 2); + ns("(?a)[^\\p{Space}\\d]", "\u00a0"); + ns("(?a)[^\\d\\p{Space}]", "\u00a0"); + x2s("(?d)[[:space:]\\d]", "\u00a0", 0, 2); + ns("(?d)[^\\d[:space:]]", "\u00a0"); + + x2s("\\p{In_Unified_Canadian_Aboriginal_Syllabics_Extended}+", "\u18B0\u18FF", 0, 6); + x2s("(?i)\u1ffc", "\u2126\u1fbe", 0, 6); + x2s("(?i)\u1ffc", "\u1ff3", 0, 3); + x2s("(?i)\u0390", "\u03b9\u0308\u0301", 0, 6); + x2s("(?i)\u03b9\u0308\u0301", "\u0390", 0, 2); + x2s("(?i)ff", "\ufb00", 0, 3); + x2s("(?i)\ufb01", "fi", 0, 2); + + x2s("(?i)\u0149\u0149", "\u0149\u0149", 0, 4); + x2s("(?i)(?<=\u0149)a", "\u02bcna", 3, 4); + + x2s("(?m:.*abc)", "dddabdd\nddabc", 0, 13); + x2s("(?m:.+abc)", "dddabdd\nddabc", 0, 13); + x2s("(?-m:.*abc)", "dddabdd\nddabc", 8, 13); + ns("(?-m:.*ab[x-z])", "dddabdd\nddabc"); + x2s("(?-m:.*(?:abc|\\Gabc))", "dddabdd\nddabc", 8, 13); + x2s("(?-m:.+abc)", "dddabdd\nddabc", 8, 13); + x2s("(?-m:.*abc)", "dddabdd\nabc", 8, 11); + ns("(?-m:.+abc)", "dddabdd\nabc"); + x2s("(?m:.*\\Z)", "dddabdd\nddabc", 0, 13); + x2s("(?-m:.*\\Z)", "dddabdd\nddabc", 8, 13); + x2s("(.*)X\\1", "1234X2345", 1, 8); + + x2s("(?<=(?i)ab)cd", "ABcd", 2, 4); + x2s("(?<=(?i:ab))cd", "ABcd", 2, 4); + ns("(?<=(?i)ab)cd", "ABCD"); + ns("(?<=(?i:ab))cd", "ABCD"); + x2s("(?)->", "<- ->->", 0, 5); + x2s("<-(?~->)->\n", "<-1->2<-3->\n", 6, 12); + x2s("<-(?~->)->.*<-(?~->)->", "<-1->2<-3->4<-5->", 0, 17); + x2s("<-(?~->)->.*?<-(?~->)->", "<-1->2<-3->4<-5->", 0, 11); + x2s("(?~abc)c", "abc", 0, 3); + x2s("(?~abc)bc", "abc", 0, 3); + x2s("(?~abc)abc", "abc", 0, 3); + + // ns("(?~)", " "); + ns("(?~)", ""); + ns(" (?~)", " "); + ns(" (?~)", " "); + // x2s("(?~(?~))", "abc", 0, 3); + x2s("(?~a)", "", 0, 0); + x2s("(?~a)a", "a", 0, 1); + x2s("(?~a)", "x", 0, 1); + x2s("(?~a)a", "xa", 0, 2); + x2s("(?~.)", "", 0, 0); + x2s("(?~.)a", "a", 0, 1); + x2s("(?~.)", "x", 0, 0); + x2s("(?~.)a", "xa", 1, 2); + x2s("(?~abc)", "abc", 0, 2); + x2s("(?~b)", "abc", 0, 1); + x2s("(?~abc|b)", "abc", 0, 1); + // ns("(?~|abc)", "abc"); // ? + x2s("(?~abc|)", "abc", 0, 1); // ? + x2s("(?~abc|def)x", "abcx", 1, 4); + x2s("(?~abc|def)x", "defx", 1, 4); + x2s("^(?~\\S+)TEST", "TEST", 0, 4); + x3s("(?~(a)c)", "aab", -1, -1, 1); // # $1 should not match. + + x2s("𠜎𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎?𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎*𠜱", "𠜎𠜱", 0, 8); + x2s("𠜎{3}", "𠜎𠜎𠜎", 0, 12); + + x2s("[^a\\x{80}]", "x", 0, 1); + ns("[^a\\x{80}]", "a"); + ns("[a\\x{80}]", "x", Option.CR_7_BIT); + x2s("[a\\x{80}]", "a", 0, 1, Option.CR_7_BIT); + x2s("[^a\\x{80}]", "x", 0, 1, Option.CR_7_BIT); + ns("[^a\\x{80}]", "a", Option.CR_7_BIT); + + ns("(\\2)(\\1)", ""); + + x2s("(?<=fo).*", "foo", 2, 3); + x2s("(?m)(?<=fo).*", "foo", 2, 3); + x2s("(?m)(?<=fo).+", "foo", 2, 3); + + x3s("\\(((?:[^(]|\\g<0>)*)\\)", "(abc)(abc)", 1, 4, 1); + x3s("\\(((?:[^(]|\\g<0>)*)\\)", "((abc)(abc))", 1, 11, 1); + x3s("\\(((?:[^(]|(\\g<0>))*)\\)", "((abc)(abc))", 6, 11, 2); + + x2s("^.+$", "a\n", 0, 1); + x2s("^.+$", "\na\n", 1, 2); + ns("^.+$", "\n"); + + ns("💌", "aa"); + ns("aa", "💌"); + + x2s("\\P{In_Supplemental_Symbols_and_Pictographs}?", "", 0, 0); + x2s("\\P{In_Transport_and_Map_Symbols}?", "", 0, 0); + + x2s("^(\"|)(.*)\\1$", "X6", 0, 2); + } +} diff --git a/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java b/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java new file mode 100644 index 000000000..69796af5b --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestUtf8CaseFoldingLatin1Supplement.java @@ -0,0 +1,56 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestUtf8CaseFoldingLatin1Supplement extends Test { + + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return UTF8Encoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "utf-8"; + } + + @Override + public Syntax syntax() { + return Syntax.Java; + } + + @Override + public void test() throws InterruptedException { + // test ignorecase for Latin-1 Supplement + x2s("[\\u00e0-\\u00e5]", "\u00c2", 0, 2, Option.IGNORECASE); + x2s("[\\u00e2]", "\u00c2", 0, 2, Option.IGNORECASE); + x2s("\\u00e2", "\u00c2", 0, 2, Option.IGNORECASE); + } +} diff --git a/third_party/licenses/jcodings-LICENSE.txt b/third_party/licenses/jcodings-LICENSE.txt new file mode 100644 index 000000000..ed2a963f7 --- /dev/null +++ b/third_party/licenses/jcodings-LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2025 JRuby Team + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +