diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f6d1c1..d88667b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 with: repository: regtab/jregtab - ref: v0.5.0 + ref: v0.7.1 token: ${{ secrets.JREGTAB_TOKEN }} path: jregtab - uses: actions/setup-java@v4 diff --git a/Cargo.lock b/Cargo.lock index 0bbf2b3..acf464c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,11 +167,12 @@ dependencies = [ [[package]] name = "pyregtab" -version = "0.5.1" +version = "0.7.1" dependencies = [ "indexmap", "pyo3", "regex", + "rustc-hash", ] [[package]] @@ -212,6 +213,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustversion" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index c4c6724..7779a5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyregtab" -version = "0.5.1" +version = "0.7.1" edition = "2021" description = "Native core of pyRegTab: RTL compiler, ATP matcher and table interpreter" license = "MIT" @@ -20,6 +20,7 @@ python = ["dep:pyo3"] pyo3 = { version = "0.23", features = ["abi3-py310"], optional = true } regex = "1" indexmap = "2" +rustc-hash = "2" [profile.release] lto = true diff --git a/README.md b/README.md index d6c3ad5..c56bd3d 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,26 @@ and interprets the match into a relational **recordset**: TableSyntax → RtlCompiler/TablePattern → AtpMatcher → TableInterpreter → Recordset ``` -**pyRegTab 0.5.1 ≙ jRegTab 0.5.1** (same API, same semantics, same test +**pyRegTab 0.7.1 ≙ jRegTab 0.7.1** (same API, same semantics, same test corpus), including the embedded RTL DSL `pyregtab.dsl` — a port of jRegTab's `ru.icc.regtab.dsl` (added upstream in jRegTab 0.3.0). Python-side extras on top of the Java API: `AtpMatcher.match_many` (parallel batch matching), `Recordset.to_pandas()`, -`Recordset.to_csv()`, `RtlCompileError.line`/`.col` attributes, and -`CellDerivedItem.span` (the item's byte range within the raw cell text). +`Recordset.to_csv()`, `RtlCompileError.line`/`.col` attributes, +`CellDerivedItem.span` (the item's byte range within the raw cell text), and +the CLI runner `python -m pyregtab.runner` (see below). + +Since 0.5.1 the language and the interpreter follow jRegTab 0.6.0–0.7.1: +`JOIN(K)` is the **record product** (a cross product for `K = ∅`, an equi-join +on the key `K` otherwise), the former folding operation is spelled +`CONCAT(K)` — a pattern written for 0.5.x must replace `JOIN(K)` by +`CONCAT(K)`; the key `K` may name attributes (`CONCAT(0, 1, 'A', 'B')`, +`JOIN('Year')`); skipped `CONCAT`/`JOIN` actions (a key mismatch, a shared +attribute, a forgotten `REC`) are reported through +`TableInterpreter.diagnostics()`; zero-width subrows match the empty +sequence and `{0}`/`{1}` are accepted; cell-derived providers use a spatial +index, so interpretation is linear in the number of cells (a 1.19 M-cell +table interprets in seconds). ## Installation @@ -143,7 +156,7 @@ Rust (`pyregtab._core`, built with [PyO3](https://pyo3.rs) and ## Testing -`pytest tests` runs (1 925 tests): +`pytest tests` runs (1 981 tests): - the full benchmark suite — tasks 001–150 (Foofah, RegTab, Baikal), every fixture variant, **both** via RTL patterns and via ATP patterns @@ -159,13 +172,20 @@ Rust (`pyregtab._core`, built with [PyO3](https://pyo3.rs) and - RTL↔ATP round-trip for tasks 001–050; - API unit tests (syntax layer, extractors, EXT bindings, custom predicates, transformations, interpreter options, GIL-released batch - matching from a thread pool and via `AtpMatcher.match_many`). - -`cargo test` additionally runs the conformance corpus and an end-to-end -smoke test against the native core alone. Differential testing against the -Java reference (`tools/differential.py` + `tools/RecordsetDumpMain.java`) -compares recordsets cell-by-cell on all 750 task variants — zero -mismatches against jRegTab v0.5.0. + matching from a thread pool and via `AtpMatcher.match_many`); +- `CONCAT`/`JOIN` semantics, named keys, diagnostics, strict preconditions, + zero-width subrows and `{0}`/`{1}` (`tests/test_join_concat.py`, a port of + jRegTab's `TableInterpreterMultiRecordTest`). + +`cargo test` additionally runs the conformance corpus, an end-to-end smoke +test against the native core alone, the working-state unit tests for +`CONCAT`/`JOIN` (ports of `WorkingStateConcatTest`/`WorkingStateJoinTest`), the +zero-width matcher cases, and a randomized equivalence test of the indexed +provider against the reference definition Υ^{J,k}_{τ,κ} (full scan + sort). +Differential testing against the Java reference (`tools/differential.py` + +`tools/RecordsetDumpMain.java`) compares recordsets cell-by-cell on all 750 +task variants; the 244 ATBench solutions of regtab-eval-on-atbench are +byte-identical between the jRegTab 0.7.1 runner and `python -m pyregtab.runner`. ## IDE support @@ -180,6 +200,21 @@ IntelliJ/PyCharm and other TextMate editors is under [`ide/`](ide/README.md). RTL is also validated at compile time: `RtlCompiler.compile(...)` raises `RtlCompileError` with a `line:col` position on an invalid pattern. +## Command-line runner + +`python -m pyregtab.runner ` (or `python -m pyregtab `) +reads `./input.csv`, applies the pattern (`RtlCompiler.compile` → +`AtpMatcher.match` → `TableInterpreter` with `RECORD_FIRST` → +`pattern.transform`) and writes `./output.csv` — the schema as the header row, +every field double-quoted, a missing value as the empty string, UTF-8, LF. It +is the drop-in counterpart of the `regtab-runner` jar used by +[regtab-eval-on-atbench](https://github.com/regtab/regtab-eval-on-atbench): +same I/O contract, same exit codes (0 — success, 1 — the pattern did not +match, 2 — invalid usage), same CSV parsing (RFC 4180 with multi-line quoted +cells, ragged rows padded); on the 244 ATBench solutions of that project the +two runners produce byte-identical `output.csv`. `--input`/`--output` override +the file names, `--strict` turns interpreter diagnostics into errors. + ## Development ``` diff --git a/conformance/README.md b/conformance/README.md index 737b2fd..547074c 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -103,6 +103,29 @@ compound specification). > Changed in jRegTab 0.5.0. Earlier versions trimmed each substring and silently > dropped empty ones; patterns relying on that must add `=TRIM` to the delimited atom. +## Zero-width subrows and `{n}` with n < 2 + +An explicit subrow (or subtable) whose children are all optional may consume no cells at +all, e.g. `{ [BLANK]* }` in a row without blank cells. It then matches the *empty sequence*, +as `*` does in regular expressions: + +- the empty match is tried at the end of the row as well, so a trailing `{ [BLANK]* }` + does not make the row fail; +- a repeated empty match (`{ [BLANK]* }+`, `{ [BLANK]* }*`, `{ [BLANK]* }{3}`) counts as a + single empty iteration — implementations must not loop; +- an empty subrow covers no cells and must not appear in the interpretable table. + +The quantifier `{n}` accepts any `n ≥ 0`: `{1}` is equivalent to no quantifier, `{0}` to +zero occurrences (an empty match). Only a negative `n` is a compile error. Positive case +`quantifier_small_n` pins the canonical form; the executable checks are +`semantic/subrow_zero_width_mid`, `semantic/subrow_zero_width_tail`, +`semantic/subrow_zero_width_repeated` (`+`), `semantic/subrow_zero_width_repeated_star` (`*`), +`semantic/quantifier_exactly_one`, `semantic/quantifier_exactly_zero` and the negative case +`quantifier_exactly_negative`. + +> Changed in jRegTab after 0.7.0. Earlier versions raised an error for a zero-width subrow in the +> middle of a row, silently failed on one at the end of a row, and rejected `{0}` / `{1}`. + In jRegTab items 1–4 of the contract are executed by `ru.icc.regtab.conformance.RtlConformanceTest` and item 5 by `ru.icc.regtab.conformance.RtlSemanticConformanceTest`; diff --git a/conformance/UPSTREAM b/conformance/UPSTREAM index 67f8315..5b3a87b 100644 --- a/conformance/UPSTREAM +++ b/conformance/UPSTREAM @@ -1,3 +1,3 @@ -commit: c12633763b309fd00f65d8b236a4ab91795303b4 -tag: v0.5.1 +commit: f0093339139f828353db31ad822dd73f68c72eab +tag: v0.7.1 path: conformance/ diff --git a/conformance/VERSION b/conformance/VERSION index 1820d49..d1bdc4d 100644 --- a/conformance/VERSION +++ b/conformance/VERSION @@ -1,2 +1,8 @@ -generated: 2026-08-26 +generated: 2026-08-29 sources: RtlTask001..150 + curated extras +note: key K of CONCAT(K)/JOIN(K) may name attributes (CONCAT(0, 1, 'A', 'B'), JOIN('k')), resolved per record; + canonical form sorts positions, then names, single-quoted; curated extra named_key and + semantic cases concat_named_key, join_named_key, negative concat_empty_key_name added; + zero-width subrows match the empty sequence (never repeated) and {n} accepts n in {0, 1}: + curated extra quantifier_small_n, semantic cases subrow_zero_width_{mid,tail,repeated,repeated_star}, + quantifier_exactly_{one,zero}, negative quantifier_exactly_negative added diff --git a/conformance/negative/concat_empty_key_name.rtl b/conformance/negative/concat_empty_key_name.rtl new file mode 100644 index 0000000..a3edaec --- /dev/null +++ b/conformance/negative/concat_empty_key_name.rtl @@ -0,0 +1 @@ +[ [VAL : RT->REC, BW*->CONCAT('')] ] diff --git a/conformance/negative/quantifier_exactly_negative.rtl b/conformance/negative/quantifier_exactly_negative.rtl new file mode 100644 index 0000000..b846154 --- /dev/null +++ b/conformance/negative/quantifier_exactly_negative.rtl @@ -0,0 +1 @@ +[ [VAL]{-1} ] diff --git a/conformance/positive/named_key.expected.rtl b/conformance/positive/named_key.expected.rtl new file mode 100644 index 0000000..647c454 --- /dev/null +++ b/conformance/positive/named_key.expected.rtl @@ -0,0 +1 @@ +[ [] [] [ ATTR ]+ ] [ [ VAL : RT*->REC, (BW & STR)*->CONCAT(0, 1, 'A', 'B') ] [ VAL ] [ VAL ] [ VAL : COL->AVP ]{2} ]+ [ [ VAL : COL->AVP, RT->REC, C2*->JOIN('k') ] [ VAL : COL->AVP ] [ VAL : COL->AVP, RT->REC ] [ VAL : COL->AVP ] ]+ diff --git a/conformance/positive/named_key.rtl b/conformance/positive/named_key.rtl new file mode 100644 index 0000000..0ed9e86 --- /dev/null +++ b/conformance/positive/named_key.rtl @@ -0,0 +1,3 @@ +[ [] [] [ATTR]+ ] +[ [VAL: RT*->REC, (BW&STR)*->CONCAT("B", 'A', 1, 0)] [VAL] [VAL] [VAL: COL->AVP]{2} ]+ +[ [VAL: COL->AVP, RT->REC, C2*->JOIN('k')] [VAL: COL->AVP] [VAL: COL->AVP, RT->REC] [VAL: COL->AVP] ]+ diff --git a/conformance/positive/quantifier_small_n.expected.rtl b/conformance/positive/quantifier_small_n.expected.rtl new file mode 100644 index 0000000..6c01062 --- /dev/null +++ b/conformance/positive/quantifier_small_n.expected.rtl @@ -0,0 +1 @@ +[ [ VAL ]{1} [ VAL ]{0} { [ BLANK ]* }{1} ] diff --git a/conformance/positive/quantifier_small_n.rtl b/conformance/positive/quantifier_small_n.rtl new file mode 100644 index 0000000..0095224 --- /dev/null +++ b/conformance/positive/quantifier_small_n.rtl @@ -0,0 +1 @@ +[ [VAL]{1} [VAL]{0} { [BLANK]* }{1} ] diff --git a/conformance/positive/task_016.expected.rtl b/conformance/positive/task_016.expected.rtl index 40267fa..175e577 100644 --- a/conformance/positive/task_016.expected.rtl +++ b/conformance/positive/task_016.expected.rtl @@ -1 +1 @@ -[ [ VAL : RT->REC, (BW & STR)*->JOIN(0) ] [ VAL ] ]+ +[ [ VAL : RT->REC, (BW & STR)*->CONCAT(0) ] [ VAL ] ]+ diff --git a/conformance/positive/task_016.rtl b/conformance/positive/task_016.rtl index 42add13..86bd6e6 100644 --- a/conformance/positive/task_016.rtl +++ b/conformance/positive/task_016.rtl @@ -1 +1 @@ -[ [VAL : RT->REC, BW&STR*->JOIN(0)] [VAL] ]+ +[ [VAL : RT->REC, BW&STR*->CONCAT(0)] [VAL] ]+ diff --git a/conformance/positive/task_023.expected.rtl b/conformance/positive/task_023.expected.rtl index 7aa7166..24865b5 100644 --- a/conformance/positive/task_023.expected.rtl +++ b/conformance/positive/task_023.expected.rtl @@ -1 +1 @@ -{ [ [ VAL : ''->AVP, SR*->REC, (BW & STR)*->JOIN(0) ] [ ATTR : RT->SUFFIX ] [ AUX ] [ VAL : SR->AVP ] ]{3} }+ +{ [ [ VAL : ''->AVP, SR*->REC, (BW & STR)*->CONCAT(0) ] [ ATTR : RT->SUFFIX ] [ AUX ] [ VAL : SR->AVP ] ]{3} }+ diff --git a/conformance/positive/task_023.rtl b/conformance/positive/task_023.rtl index e433702..ac2a946 100644 --- a/conformance/positive/task_023.rtl +++ b/conformance/positive/task_023.rtl @@ -1 +1 @@ -{ [ [VAL : ''->AVP, SR*->REC, BW&STR*->JOIN(0)] [ATTR : RT->SUFFIX] [AUX] [VAL : SR->AVP] ]{3} }+ +{ [ [VAL : ''->AVP, SR*->REC, BW&STR*->CONCAT(0)] [ATTR : RT->SUFFIX] [AUX] [VAL : SR->AVP] ]{3} }+ diff --git a/conformance/positive/task_025.expected.rtl b/conformance/positive/task_025.expected.rtl index f4789e3..b587f19 100644 --- a/conformance/positive/task_025.expected.rtl +++ b/conformance/positive/task_025.expected.rtl @@ -1 +1 @@ - [ [ VAL : RT->SUFFIX("/"), (RT & C+2..)*->REC('/'), (BW & STR)*->JOIN(0) ] [ VAL ]+ ]+ + [ [ VAL : RT->SUFFIX("/"), (RT & C+2..)*->REC('/'), (BW & STR)*->CONCAT(0) ] [ VAL ]+ ]+ diff --git a/conformance/positive/task_025.rtl b/conformance/positive/task_025.rtl index 27dd7e8..c69aaae 100644 --- a/conformance/positive/task_025.rtl +++ b/conformance/positive/task_025.rtl @@ -1 +1 @@ -[ [VAL : RT->SUFFIX('/'), RT&C+2..*->REC('/'), BW&STR*->JOIN(0)] [VAL]+ ]+ +[ [VAL : RT->SUFFIX('/'), RT&C+2..*->REC('/'), BW&STR*->CONCAT(0)] [VAL]+ ]+ diff --git a/conformance/positive/task_033.expected.rtl b/conformance/positive/task_033.expected.rtl index 2a91346..bd6c2d3 100644 --- a/conformance/positive/task_033.expected.rtl +++ b/conformance/positive/task_033.expected.rtl @@ -1 +1 @@ -[ [ VAL : SR*->REC, (BW & STR)*->JOIN(0) ] [ VAL ]+ ]+ +[ [ VAL : SR*->REC, (BW & STR)*->CONCAT(0) ] [ VAL ]+ ]+ diff --git a/conformance/positive/task_033.rtl b/conformance/positive/task_033.rtl index 40670b7..ce80007 100644 --- a/conformance/positive/task_033.rtl +++ b/conformance/positive/task_033.rtl @@ -1 +1 @@ -[ [VAL : SR*->REC, BW&STR*->JOIN(0)] [VAL]+ ]+ +[ [VAL : SR*->REC, BW&STR*->CONCAT(0)] [VAL]+ ]+ diff --git a/conformance/positive/task_046.expected.rtl b/conformance/positive/task_046.expected.rtl index df99b50..ec0c856 100644 --- a/conformance/positive/task_046.expected.rtl +++ b/conformance/positive/task_046.expected.rtl @@ -1 +1 @@ -{ [ [ !BLANK? VAL : ''->AVP, SR*->REC, (BW & STR)*->JOIN(0) ] [ !BLANK? ATTR ] [ !BLANK? VAL : SR->AVP ] ]+ }+ +{ [ [ !BLANK? VAL : ''->AVP, SR*->REC, (BW & STR)*->CONCAT(0) ] [ !BLANK? ATTR ] [ !BLANK? VAL : SR->AVP ] ]+ }+ diff --git a/conformance/positive/task_046.rtl b/conformance/positive/task_046.rtl index 606b0c2..546cb6b 100644 --- a/conformance/positive/task_046.rtl +++ b/conformance/positive/task_046.rtl @@ -1 +1 @@ -{ [ [!BLANK? VAL : ''->AVP, SR*->REC, BW&STR*->JOIN(0)] [!BLANK? ATTR] [!BLANK? VAL : SR->AVP] ]+ }+ +{ [ [!BLANK? VAL : ''->AVP, SR*->REC, BW&STR*->CONCAT(0)] [!BLANK? ATTR] [!BLANK? VAL : SR->AVP] ]+ }+ diff --git a/conformance/positive/task_047.expected.rtl b/conformance/positive/task_047.expected.rtl index f43e797..feaf3f0 100644 --- a/conformance/positive/task_047.expected.rtl +++ b/conformance/positive/task_047.expected.rtl @@ -1 +1 @@ -{ [ [ !BLANK? VAL : SR*->REC, (BW & STR)*->JOIN(0) ] [ !BLANK? VAL ] ]+ }+ +{ [ [ !BLANK? VAL : SR*->REC, (BW & STR)*->CONCAT(0) ] [ !BLANK? VAL ] ]+ }+ diff --git a/conformance/positive/task_047.rtl b/conformance/positive/task_047.rtl index beb3d23..bea0aa8 100644 --- a/conformance/positive/task_047.rtl +++ b/conformance/positive/task_047.rtl @@ -1 +1 @@ -{ [ [!BLANK? VAL : SR*->REC, BW&STR*->JOIN(0)] [!BLANK? VAL] ]+ }+ +{ [ [!BLANK? VAL : SR*->REC, BW&STR*->CONCAT(0)] [!BLANK? VAL] ]+ }+ diff --git a/conformance/positive/task_050.expected.rtl b/conformance/positive/task_050.expected.rtl index 3381e19..f74bb47 100644 --- a/conformance/positive/task_050.expected.rtl +++ b/conformance/positive/task_050.expected.rtl @@ -1 +1 @@ -[ [ !BLANK? VAL : ''->AVP, SR*->REC, (BW & STR)*->JOIN(0) ] [ !BLANK? ATTR ] [ !BLANK? VAL : SR->AVP ] ]+ +[ [ !BLANK? VAL : ''->AVP, SR*->REC, (BW & STR)*->CONCAT(0) ] [ !BLANK? ATTR ] [ !BLANK? VAL : SR->AVP ] ]+ diff --git a/conformance/positive/task_050.rtl b/conformance/positive/task_050.rtl index c9ffe93..593f633 100644 --- a/conformance/positive/task_050.rtl +++ b/conformance/positive/task_050.rtl @@ -1 +1 @@ -[ [!BLANK? VAL : ''->AVP, SR*->REC, BW&STR*->JOIN(0)] [!BLANK? ATTR] [!BLANK? VAL : SR->AVP] ]+ +[ [!BLANK? VAL : ''->AVP, SR*->REC, BW&STR*->CONCAT(0)] [!BLANK? ATTR] [!BLANK? VAL : SR->AVP] ]+ diff --git a/conformance/positive/task_053.expected.rtl b/conformance/positive/task_053.expected.rtl index e1a8e87..3792441 100644 --- a/conformance/positive/task_053.expected.rtl +++ b/conformance/positive/task_053.expected.rtl @@ -1 +1 @@ -[ [] [ AUX ]+ ] [ [ VAL : ROW*->REC, (BW & STR)->JOIN(0), 'ID'->AVP ] { [ ATTR : AV->PREFIX("_") ] [ VAL : SR->AVP ] }+ ]+ +[ [] [ AUX ]+ ] [ [ VAL : ROW*->REC, (BW & STR)->CONCAT(0), 'ID'->AVP ] { [ ATTR : AV->PREFIX("_") ] [ VAL : SR->AVP ] }+ ]+ diff --git a/conformance/positive/task_053.rtl b/conformance/positive/task_053.rtl index cab6584..0724d42 100644 --- a/conformance/positive/task_053.rtl +++ b/conformance/positive/task_053.rtl @@ -1,3 +1,3 @@ [ [] [AUX]+ ] -[ [VAL : ROW*->REC, BW&STR->JOIN(0), 'ID'->AVP] +[ [VAL : ROW*->REC, BW&STR->CONCAT(0), 'ID'->AVP] {[ATTR : AV->PREFIX('_')] [VAL : SR->AVP]}+]+ diff --git a/conformance/positive/task_069.expected.rtl b/conformance/positive/task_069.expected.rtl index 1eae93e..f0fc959 100644 --- a/conformance/positive/task_069.expected.rtl +++ b/conformance/positive/task_069.expected.rtl @@ -1 +1 @@ -[ { [ ATTR : SR->AVP, BW*->REC ] [ VAL #'1' : SR->AVP, BW*->REC, (ROW & #'1')*->JOIN ] [ VAL #'2' : SR->AVP, BW*->REC, (ROW & #'2')*->JOIN ] }* ] [ { [ ATTR : SR->AVP ] [ VAL : SR->AVP ]{2} }* ]* +[ { [ ATTR : SR->AVP, BW*->REC ] [ VAL #'1' : SR->AVP, BW*->REC, (ROW & #'1')*->CONCAT ] [ VAL #'2' : SR->AVP, BW*->REC, (ROW & #'2')*->CONCAT ] }* ] [ { [ ATTR : SR->AVP ] [ VAL : SR->AVP ]{2} }* ]* diff --git a/conformance/positive/task_069.rtl b/conformance/positive/task_069.rtl index d055ce9..d9ccba5 100644 --- a/conformance/positive/task_069.rtl +++ b/conformance/positive/task_069.rtl @@ -1,3 +1,3 @@ SR->AVP -[ BW*->REC { [ATTR] [VAL#'1': ROW&#'1'*->JOIN][VAL#'2': ROW&#'2'*->JOIN] }* ] +[ BW*->REC { [ATTR] [VAL#'1': ROW&#'1'*->CONCAT][VAL#'2': ROW&#'2'*->CONCAT] }* ] [ { [ATTR] [VAL]{2} }* ]* diff --git a/conformance/positive/task_094.expected.rtl b/conformance/positive/task_094.expected.rtl index d2d1e73..d63701c 100644 --- a/conformance/positive/task_094.expected.rtl +++ b/conformance/positive/task_094.expected.rtl @@ -1 +1 @@ -{ [ { [ !BLANK? VAL : COL*->REC, (ROW & C+1.. & STR)*->JOIN(0) ]+ [ BLANK ]? }+ ] } { [ { [ !BLANK? VAL ]+ [ BLANK ]? }+ ]+ [ [ BLANK ]+ ]? }+ +{ [ { [ !BLANK? VAL : COL*->REC, (ROW & C+1.. & STR)*->CONCAT(0) ]+ [ BLANK ]? }+ ] } { [ { [ !BLANK? VAL ]+ [ BLANK ]? }+ ]+ [ [ BLANK ]+ ]? }+ diff --git a/conformance/positive/task_094.rtl b/conformance/positive/task_094.rtl index 1786fde..cba7a15 100644 --- a/conformance/positive/task_094.rtl +++ b/conformance/positive/task_094.rtl @@ -1,3 +1,3 @@ - [ { [!BLANK? VAL: COL*->REC, ROW&C+1..&STR*->JOIN(0)]+ [BLANK]? }+ ] + [ { [!BLANK? VAL: COL*->REC, ROW&C+1..&STR*->CONCAT(0)]+ [BLANK]? }+ ] { [ { [!BLANK? VAL]+ [BLANK]? }+ ]+ [ [BLANK]+ ]? }+ diff --git a/conformance/positive/task_097.expected.rtl b/conformance/positive/task_097.expected.rtl index 494b9ea..4c3a492 100644 --- a/conformance/positive/task_097.expected.rtl +++ b/conformance/positive/task_097.expected.rtl @@ -1 +1 @@ -[ [ VAL : RT*->REC, (BW & STR)*->JOIN(0, 1) ] [ VAL ]+ ]+ +[ [ VAL : RT*->REC, (BW & STR)*->CONCAT(0, 1) ] [ VAL ]+ ]+ diff --git a/conformance/positive/task_097.rtl b/conformance/positive/task_097.rtl index 7315103..cf59eda 100644 --- a/conformance/positive/task_097.rtl +++ b/conformance/positive/task_097.rtl @@ -1 +1 @@ -[ [VAL: RT*->REC, (BW&STR)*->JOIN(0,1)] [VAL]+ ]+ +[ [VAL: RT*->REC, (BW&STR)*->CONCAT(0,1)] [VAL]+ ]+ diff --git a/conformance/positive/task_098.expected.rtl b/conformance/positive/task_098.expected.rtl index 606004f..7f3bfd2 100644 --- a/conformance/positive/task_098.expected.rtl +++ b/conformance/positive/task_098.expected.rtl @@ -1 +1 @@ -[ [] [] [ ATTR ]+ ] [ [ VAL : RT*->REC, (BW & STR)*->JOIN(0, 1) ] [ VAL ] [ VAL : COL->AVP ]{2} [ VAL ]+ ]+ +[ [] [] [ ATTR ]+ ] [ [ VAL : RT*->REC, (BW & STR)*->CONCAT(0, 1, 2, 3) ] [ VAL ] [ VAL : COL->AVP ]{2} [ VAL ]+ ]+ diff --git a/conformance/positive/task_098.rtl b/conformance/positive/task_098.rtl index 9cfbd4a..6a4252b 100644 --- a/conformance/positive/task_098.rtl +++ b/conformance/positive/task_098.rtl @@ -1,2 +1,2 @@ [ [] [] [ATTR]+ ] -[ [VAL: RT*->REC, (BW&STR)*->JOIN(0,1)] [VAL] [VAL: COL->AVP]{2} [VAL]+ ]+ +[ [VAL: RT*->REC, (BW&STR)*->CONCAT(0,1,2,3)] [VAL] [VAL: COL->AVP]{2} [VAL]+ ]+ diff --git a/conformance/semantic/concat_by_key/expected.csv b/conformance/semantic/concat_by_key/expected.csv new file mode 100644 index 0000000..efc5836 --- /dev/null +++ b/conformance/semantic/concat_by_key/expected.csv @@ -0,0 +1,3 @@ +"book","5","6","7" +"cat","2","3","8" +"dog","2","3","4" diff --git a/conformance/semantic/concat_by_key/input.csv b/conformance/semantic/concat_by_key/input.csv new file mode 100644 index 0000000..848a3af --- /dev/null +++ b/conformance/semantic/concat_by_key/input.csv @@ -0,0 +1,9 @@ +"book","5" +"book","6" +"book","7" +"cat","2" +"cat","3" +"cat","8" +"dog","2" +"dog","3" +"dog","4" diff --git a/conformance/semantic/concat_by_key/options.json b/conformance/semantic/concat_by_key/options.json new file mode 100644 index 0000000..0a01d42 --- /dev/null +++ b/conformance/semantic/concat_by_key/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": false } diff --git a/conformance/semantic/concat_by_key/pattern.rtl b/conformance/semantic/concat_by_key/pattern.rtl new file mode 100644 index 0000000..86bd6e6 --- /dev/null +++ b/conformance/semantic/concat_by_key/pattern.rtl @@ -0,0 +1 @@ +[ [VAL : RT->REC, BW&STR*->CONCAT(0)] [VAL] ]+ diff --git a/conformance/semantic/concat_named_key/expected.csv b/conformance/semantic/concat_named_key/expected.csv new file mode 100644 index 0000000..1dc31a7 --- /dev/null +++ b/conformance/semantic/concat_named_key/expected.csv @@ -0,0 +1,2 @@ +"k1","k11","c1","a1","b1","c2" +"k2","k22","c3","a2","b2","c4" diff --git a/conformance/semantic/concat_named_key/input.csv b/conformance/semantic/concat_named_key/input.csv new file mode 100644 index 0000000..25e2aaf --- /dev/null +++ b/conformance/semantic/concat_named_key/input.csv @@ -0,0 +1,5 @@ +"","","C","A","B" +"k1","k11","c1","a1","b1" +"k1","k11","c2","a1","b1" +"k2","k22","c3","a2","b2" +"k2","k22","c4","a2","b2" diff --git a/conformance/semantic/concat_named_key/options.json b/conformance/semantic/concat_named_key/options.json new file mode 100644 index 0000000..0a01d42 --- /dev/null +++ b/conformance/semantic/concat_named_key/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": false } diff --git a/conformance/semantic/concat_named_key/pattern.rtl b/conformance/semantic/concat_named_key/pattern.rtl new file mode 100644 index 0000000..ccf654a --- /dev/null +++ b/conformance/semantic/concat_named_key/pattern.rtl @@ -0,0 +1,2 @@ +[ [] [] [ATTR]+ ] +[ [VAL: RT*->REC, (BW&STR)*->CONCAT(0, 1, 'A', 'B')] [VAL] [VAL] [VAL: COL->AVP]{2} ]+ diff --git a/conformance/semantic/join_equi_key/expected.csv b/conformance/semantic/join_equi_key/expected.csv new file mode 100644 index 0000000..d813e43 --- /dev/null +++ b/conformance/semantic/join_equi_key/expected.csv @@ -0,0 +1,3 @@ +"k","v","u" +"X","5","kg" +"Y","8","pc" diff --git a/conformance/semantic/join_equi_key/input.csv b/conformance/semantic/join_equi_key/input.csv new file mode 100644 index 0000000..c2c6a4f --- /dev/null +++ b/conformance/semantic/join_equi_key/input.csv @@ -0,0 +1,3 @@ +"k","v","k","u" +"X","5","X","kg" +"Y","8","Y","pc" diff --git a/conformance/semantic/join_equi_key/options.json b/conformance/semantic/join_equi_key/options.json new file mode 100644 index 0000000..41659ef --- /dev/null +++ b/conformance/semantic/join_equi_key/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": true } diff --git a/conformance/semantic/join_equi_key/pattern.rtl b/conformance/semantic/join_equi_key/pattern.rtl new file mode 100644 index 0000000..c8c1fa9 --- /dev/null +++ b/conformance/semantic/join_equi_key/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, RT->REC, C2*->JOIN(0)] [VAL: COL->AVP] [VAL: COL->AVP, RT->REC] [VAL: COL->AVP] ]+ diff --git a/conformance/semantic/join_named_key/expected.csv b/conformance/semantic/join_named_key/expected.csv new file mode 100644 index 0000000..d813e43 --- /dev/null +++ b/conformance/semantic/join_named_key/expected.csv @@ -0,0 +1,3 @@ +"k","v","u" +"X","5","kg" +"Y","8","pc" diff --git a/conformance/semantic/join_named_key/input.csv b/conformance/semantic/join_named_key/input.csv new file mode 100644 index 0000000..c2c6a4f --- /dev/null +++ b/conformance/semantic/join_named_key/input.csv @@ -0,0 +1,3 @@ +"k","v","k","u" +"X","5","X","kg" +"Y","8","Y","pc" diff --git a/conformance/semantic/join_named_key/options.json b/conformance/semantic/join_named_key/options.json new file mode 100644 index 0000000..41659ef --- /dev/null +++ b/conformance/semantic/join_named_key/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": true } diff --git a/conformance/semantic/join_named_key/pattern.rtl b/conformance/semantic/join_named_key/pattern.rtl new file mode 100644 index 0000000..2f81897 --- /dev/null +++ b/conformance/semantic/join_named_key/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, RT->REC, C2*->JOIN('k')] [VAL: COL->AVP] [VAL: COL->AVP, RT->REC] [VAL: COL->AVP] ]+ diff --git a/conformance/semantic/join_product/expected.csv b/conformance/semantic/join_product/expected.csv new file mode 100644 index 0000000..e18920b --- /dev/null +++ b/conformance/semantic/join_product/expected.csv @@ -0,0 +1,7 @@ +"id","value","var" +"a","1","x" +"a","2","y" +"b","1","x" +"b","2","y" +"c","3","x" +"c","4","y" diff --git a/conformance/semantic/join_product/input.csv b/conformance/semantic/join_product/input.csv new file mode 100644 index 0000000..20d7ad1 --- /dev/null +++ b/conformance/semantic/join_product/input.csv @@ -0,0 +1,3 @@ +"id","x","y" +"a;b","1","2" +"c","3","4" diff --git a/conformance/semantic/join_product/options.json b/conformance/semantic/join_product/options.json new file mode 100644 index 0000000..41659ef --- /dev/null +++ b/conformance/semantic/join_product/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": true } diff --git a/conformance/semantic/join_product/pattern.rtl b/conformance/semantic/join_product/pattern.rtl new file mode 100644 index 0000000..35732b3 --- /dev/null +++ b/conformance/semantic/join_product/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR] [VAL: 'var'->AVP]+ ] +[ [(VAL: COL->AVP, ()->REC, RT*->JOIN){';'}] [VAL: 'value'->AVP, COL->REC]+ ]+ diff --git a/conformance/semantic/quantifier_exactly_one/expected.csv b/conformance/semantic/quantifier_exactly_one/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/quantifier_exactly_one/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/quantifier_exactly_one/input.csv b/conformance/semantic/quantifier_exactly_one/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/quantifier_exactly_one/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/quantifier_exactly_one/pattern.rtl b/conformance/semantic/quantifier_exactly_one/pattern.rtl new file mode 100644 index 0000000..a6de863 --- /dev/null +++ b/conformance/semantic/quantifier_exactly_one/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* }{1} [VAL: COL->AVP] ]+ diff --git a/conformance/semantic/quantifier_exactly_zero/expected.csv b/conformance/semantic/quantifier_exactly_zero/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/quantifier_exactly_zero/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/quantifier_exactly_zero/input.csv b/conformance/semantic/quantifier_exactly_zero/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/quantifier_exactly_zero/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/quantifier_exactly_zero/pattern.rtl b/conformance/semantic/quantifier_exactly_zero/pattern.rtl new file mode 100644 index 0000000..430165c --- /dev/null +++ b/conformance/semantic/quantifier_exactly_zero/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* }{0} [VAL: COL->AVP] ]+ diff --git a/conformance/semantic/subrow_zero_width_mid/expected.csv b/conformance/semantic/subrow_zero_width_mid/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/subrow_zero_width_mid/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_mid/input.csv b/conformance/semantic/subrow_zero_width_mid/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/subrow_zero_width_mid/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_mid/pattern.rtl b/conformance/semantic/subrow_zero_width_mid/pattern.rtl new file mode 100644 index 0000000..4543f9f --- /dev/null +++ b/conformance/semantic/subrow_zero_width_mid/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* } [VAL: COL->AVP] ]+ diff --git a/conformance/semantic/subrow_zero_width_repeated/expected.csv b/conformance/semantic/subrow_zero_width_repeated/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_repeated/input.csv b/conformance/semantic/subrow_zero_width_repeated/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_repeated/pattern.rtl b/conformance/semantic/subrow_zero_width_repeated/pattern.rtl new file mode 100644 index 0000000..77639d3 --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* }+ [VAL: COL->AVP] { [BLANK]* }+ ]+ diff --git a/conformance/semantic/subrow_zero_width_repeated_star/expected.csv b/conformance/semantic/subrow_zero_width_repeated_star/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated_star/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_repeated_star/input.csv b/conformance/semantic/subrow_zero_width_repeated_star/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated_star/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_repeated_star/pattern.rtl b/conformance/semantic/subrow_zero_width_repeated_star/pattern.rtl new file mode 100644 index 0000000..787a76f --- /dev/null +++ b/conformance/semantic/subrow_zero_width_repeated_star/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* }* [VAL: COL->AVP] { [BLANK]* }* ]+ diff --git a/conformance/semantic/subrow_zero_width_tail/expected.csv b/conformance/semantic/subrow_zero_width_tail/expected.csv new file mode 100644 index 0000000..972ee6c --- /dev/null +++ b/conformance/semantic/subrow_zero_width_tail/expected.csv @@ -0,0 +1,2 @@ +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_tail/input.csv b/conformance/semantic/subrow_zero_width_tail/input.csv new file mode 100644 index 0000000..6b4f5fa --- /dev/null +++ b/conformance/semantic/subrow_zero_width_tail/input.csv @@ -0,0 +1,3 @@ +A,B +x,1 +y,2 diff --git a/conformance/semantic/subrow_zero_width_tail/pattern.rtl b/conformance/semantic/subrow_zero_width_tail/pattern.rtl new file mode 100644 index 0000000..dd6b2da --- /dev/null +++ b/conformance/semantic/subrow_zero_width_tail/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ [VAL: COL->AVP, ROW*->REC] [VAL: COL->AVP] { [BLANK]* } ]+ diff --git a/docs/api.md b/docs/api.md index f0f90dd..209e64e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -26,10 +26,18 @@ rs = ( .with_action_application_strategy(ActionApplicationStrategy.ROW_FIRST) .with_missing_value_handler(lambda attr: "") # str -> str | None .with_anonymous_attribute_template("A%i") # default "$a_%i" + .with_strict_preconditions(False) # default: skipped CONCAT/JOIN actions are reported, not raised .interpret(itm) ) ``` +After `interpret`, `TableInterpreter.diagnostics()` lists the `CONCAT`/`JOIN` actions that +had no effect (as `Diagnostic` objects with `.anchor`, `.operation`, `.message`): a key +mismatch, a named attribute shared by two concatenated records, an empty record product, and +— for actions written on the anchor's own content spec, not inherited — an anchor without a +record (`anchor has no record — REC missing?`) or provided items none of which has a record. +With `with_strict_preconditions(True)` the first such case raises a `RuntimeError` instead. + When the pattern contains no Python callbacks (no `EXT`/custom predicates), `AtpMatcher.match` and `TableInterpreter.interpret` release the GIL — batch processing parallelizes with a plain `ThreadPoolExecutor`, or use @@ -84,7 +92,9 @@ Actions and providers: |---|---| | `ActionSpec.rec(*providers, anchor_pos=None, split_delimiter=None)` | `…->REC`, `REC(n)`, `REC('/')` | | `ActionSpec.avp(provider_or_str)` | `…->AVP`, `'NAME'->AVP` | -| `ActionSpec.join(*providers, key_positions=None)` | `…->JOIN(k…)` | +| `ActionSpec.concat(*providers, key=None)` | `…->CONCAT`, `CONCAT(k…)`, `CONCAT('A')`, `CONCAT(0, 'A')` — fold the provided records into the anchor's record; `key` is an int, a str, an iterable of both, or a `RecordKey` (`key_positions=` is kept for 0.5.x callers) | +| `ActionSpec.join(*providers, key=None)` | `…->JOIN`, `JOIN(k…)`, `JOIN('A')` — the record product (equi-join on `key`) | +| `RecordKey(positions=(), names=())`, `RecordKey.of(…)`, `RecordKey.empty()`; `ActionSpec.key` / `.key_positions` | the key K of a `CONCAT`/`JOIN` action | | `ActionSpec.fill/prefix/suffix(delimiter, *providers)` | `…->FILL("d")` | | `ProviderSpec.val/attr/aux/any(condition, cardinality=1, traversal_order=None)` | `ST*`, `-AV`, `^COL{2}` | | `ProviderSpec.ctx_attr/ctx_val/ctx_aux(text)`, `ctx_avp(name, value)` | `'NAME'`, `@'A'='V'` | @@ -97,7 +107,7 @@ Conditions and predicates: | `ItemFilterConditionSpec.bare(term)`, `.and_(*terms)`, `.or_(*groups)`, shorthands `same_subtable()` … `left_of()` | `(LT & !BLANK)` | | `FilterTerm.*` — spatial (`left_of` … `same_cell`, `col_exact`, `row_offset`, `pos_range`, …) and content (`regex_matched`, `contains`, `blank`, `tagged`, `same_str`, `external`, `custom`) | `spatConstr` / `contConstr` | | `StringExtractor.whitespace_normalized() / trimmed() / upper_case() / lower_case() / substring(b, e) / replaced(rx, rep) / chain(*steps) / custom(desc, fn)` | `=NORM`, `=TRIM`, … | -| `Quantifier.one() / zero_or_one() / one_or_more() / zero_or_more() / exactly(n)` | `?`, `+`, `*`, `{n}` | +| `Quantifier.one() / zero_or_one() / one_or_more() / zero_or_more() / exactly(n)` | `?`, `+`, `*`, `{n}` (`n ≥ 0`; `{1}` ≡ no quantifier, `{0}` ≡ empty match) | Custom (`custom`) and external (`EXT`) predicates take Python callables: cell predicates receive a `Cell`, item filters receive diff --git a/docs/architecture.md b/docs/architecture.md index 550fabb..0f73195 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -87,8 +87,8 @@ If the pattern does not match, `AtpMatcher.match` returns `Optional.empty()`. | Phase | What happens | |---|---| | **1. Initialisation** | Each cell-derived and context-derived item of type VAL/ATTR is entered into the working state with its string value | -| **2. Completion** | Interpretation actions are applied in operation-type order: FILL/PREFIX/SUFFIX → AVP → REC → JOIN; each action uses its providers to retrieve items relative to the anchor and updates the working state | -| **3. Extraction** | The working state is traversed to build the schema (attribute list) and generate records | +| **2. Completion** | Interpretation actions are applied in operation-type order: FILL/PREFIX/SUFFIX → AVP → REC → CONCAT → JOIN (records are folded before they are multiplied); each action uses its providers to retrieve items relative to the anchor and updates the working state. A violated CONCAT/JOIN precondition has no effect and is reported through `TableInterpreter.diagnostics()`. Cell-derived providers scan a spatial index over the items (row-major and column-major orderings with row/column offsets) restricted to a candidate scope derived statically from the filter specification — the anchor's row, column, cell, row range or subtable — so interpretation is linear in the number of cells | +| **3. Extraction** | The live anchors of the working state (joined-away anchors excluded) are traversed to build the schema (attribute list) and generate records — one per item-based record, several per anchor after a JOIN | | **4. Transformation** | Optional post-processing steps are applied: `WhitespaceNormalization`, `FieldSplitting`, `SchemaReordering` | --- @@ -139,7 +139,7 @@ The round-trip property — serialize then compile gives back the original patte | `RowPattern`, `SubrowPattern`, `CellPattern` | `[ ... ]q`, `{ ... }q`, `[ ... ]q` | | `AtomicContentSpec` with tags | `VAL #'tag'` | | `AtomicContentSpec` with extractor | `VAL = TRIM` | -| `ActionSpec` (avp, rec, join, fill, prefix, suffix) | `'NAME'->AVP`, `(prov…)->REC`, etc. | +| `ActionSpec` (avp, rec, concat, join, fill, prefix, suffix) | `'NAME'->AVP`, `(prov…)->REC`, etc. | | `ProviderSpec` with traversal order | leading `-` / `^` / `-^` | | `ProviderSpec` with cardinality | `{n}` / `*` | | `RecordsetTransformation` settings | ``, ``, `` | diff --git a/docs/embedded-rtl.md b/docs/embedded-rtl.md index 48ddefb..2c86631 100644 --- a/docs/embedded-rtl.md +++ b/docs/embedded-rtl.md @@ -108,9 +108,10 @@ compiler. |---|---| | `(…)->REC` / `REC(n)` / `REC('s')` | `rec(…)` / `rec(n, …)` / `rec_split("s", …)` | | `prov->AVP` / `'NAME'->AVP` | `avp(prov)` / `avp("NAME")` | -| `(…)->JOIN` / `JOIN(k)` | `join(…)` / `join(k, …)` | +| `(…)->CONCAT` / `CONCAT(k)` / `CONCAT('A')` / `CONCAT(0, 'A')` | `concat(…)` / `concat(k, …)` / `concat("A", …)` / `concat((0, "A"), …)` (a leading int, str, set/tuple or `RecordKey` is the key) | +| `(…)->JOIN` / `JOIN(k)` / `JOIN('A')` | `join(…)` / `join(k, …)` / `join("A", …)` | | `(…)->FILL('d')`, `PREFIX`, `SUFFIX` | `fill("d", …)`, `prefix(…)`, `suffix(…)` (delimiter optional) | -| `'EUR'` context literal | `lit("EUR")` (VALUE under REC/JOIN, ATTRIBUTE otherwise — as in the compiler) | +| `'EUR'` context literal | `lit("EUR")` (VALUE under REC/CONCAT/JOIN, ATTRIBUTE otherwise — as in the compiler) | | `@'K'='V'` | `ctx_avp("K", "V")` | Provider kinds (VAL/ATTR/UNRESTRICTED) are inferred from the action, exactly as in the RTL diff --git a/docs/index.md b/docs/index.md index 9d5ee20..4e59521 100644 --- a/docs/index.md +++ b/docs/index.md @@ -88,7 +88,7 @@ Requires **Python 3.10+**; binary wheels for Windows, Linux, and macOS. - **RTL** (Regular Table Language) — compact DSL that compiles to ATP; dramatically reduces pattern verbosity. - **ATP → RTL serializer** — round-trip: serialize any `TablePattern` back to an RTL string. - **Content specs** — atomic, delimited, compound, and conditional cell content. -- **Action specs** — `REC`, `AVP`, `JOIN`, `FILL`, `PREFIX`, `SUFFIX` for rich schema construction. +- **Action specs** — `REC`, `AVP`, `CONCAT`, `JOIN`, `FILL`, `PREFIX`, `SUFFIX` for rich schema construction (fold records with `CONCAT`, multiply them with `JOIN`). - **Named fragments** — reuse recurring sub-patterns in RTL with `$name` definitions. - **Post-processing** — whitespace normalization, field splitting, schema reordering. - **150-task benchmark** — Foofah (50), RegTab (60), and Baikal (40) tasks, 1 500 test variants, 100 % pass rate. @@ -109,4 +109,4 @@ Requires **Python 3.10+**; binary wheels for Windows, Linux, and macOS. --- !!! note "Status" - Current release: **0.5.1** (feature parity with jRegTab 0.5.1) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) + Current release: **0.7.1** (feature parity with jRegTab 0.7.1) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) diff --git a/docs/model/atp.md b/docs/model/atp.md index 02d6398..8a5ad8e 100644 --- a/docs/model/atp.md +++ b/docs/model/atp.md @@ -47,7 +47,7 @@ row pattern, its subtable pattern, and the top-level table pattern. |---|---| | `?` | zero or one occurrence | | `1` (default) | exactly one occurrence | -| `{n}` | exactly `n` occurrences (`n ≥ 2`) | +| `{n}` | exactly `n` occurrences (`n ≥ 0`; `{1}` ≡ no quantifier, `{0}` ≡ zero occurrences, i.e. an empty match) | | `+` | one or more occurrences | | `*` | zero or more occurrences | @@ -322,8 +322,8 @@ S_act = (op, ⟨S_prov¹, …, S_provⁿ⟩) S_act = (op, s, ⟨S_prov¹, …, S_provⁿ⟩) ``` -In both forms, `op` is one of the six working-state update operations (`FILL`, -`PREFIX`, `SUFFIX`, `AVP`, `REC`, `JOIN`) and `S_prov¹ … S_provⁿ` are item +In both forms, `op` is one of the seven working-state update operations (`FILL`, +`PREFIX`, `SUFFIX`, `AVP`, `REC`, `CONCAT`, `JOIN`) and `S_prov¹ … S_provⁿ` are item provider specifications whose types must satisfy the consistency constraints for the chosen operation (see [ITM — Interpretation actions](itm.md#interpretation-actions)). @@ -335,8 +335,10 @@ chosen operation (see [ITM — Interpretation actions](itm.md#interpretation-act | `REC('s')` | `ActionSpec.rec(String delim, providers…)` | adds `DelimitedFieldSplit` post-step | | `AVP` | `ActionSpec.avp(provider)` | associates VAL anchor with ATTR item | | `AVP "name"` | `ActionSpec.avp("ATTR_NAME")` | context-derived ATTR constant | - | `JOIN` | `ActionSpec.join(providers…)` | joins records, dedup by named attribute | - | `JOIN(K)` | `ActionSpec.join(Set.of(k…), providers…)` | joins with key positions K dropped | + | `CONCAT` | `ActionSpec.concat(*providers)` | folds the provided records into the anchor's record | + | `CONCAT(K)` | `ActionSpec.concat(*providers, key=…)` — an int, a str, a tuple of both or a `RecordKey` | same, key K not repeated — positions and/or attribute names (was `JOIN(K)` up to 0.5.x) | + | `JOIN` | `ActionSpec.join(*providers)` | record product: one record per (anchor record × provided record) | + | `JOIN(K)` | `ActionSpec.join(*providers, key=…)` | equi-join on the key K (positions and/or attribute names) | | `FILL` | `ActionSpec.fill(delimiter, providers…)` | fills anchor value from providers | | `PREFIX` | `ActionSpec.prefix(delimiter, providers…)` | prepends provider values | | `SUFFIX` | `ActionSpec.suffix(delimiter, providers…)` | appends provider values | diff --git a/docs/model/itm.md b/docs/model/itm.md index 4c5d89a..8739b1c 100644 --- a/docs/model/itm.md +++ b/docs/model/itm.md @@ -227,17 +227,20 @@ into the computation. ### Working state and update operations During interpretation, semantic information is accumulated in a **working state** -`(V, A, val, attr, avp, rec)`: +`(V, A, val, attr, avp, rec, J)`: - `V ⊆ Σ*` — the set of values; - `A ⊆ Σ⁺` — the set of attributes; - `val(ι)` — maps each VAL item to a value in `V`; - `attr(ι)` — maps each ATTR item to an attribute in `A`; - `avp(ι)` — partial map from VAL items to `(attribute, value)` pairs; -- `rec(ι)` — partial map from cell-derived VAL items to sequences of VAL items - (the *item-based records*). +- `rec(ι)` — partial map from cell-derived VAL items to a *non-empty sequence* of item-based + records (each a sequence of VAL items); a single record until a join multiplies it; +- `J ⊆ dom(rec)` — the *joined-away anchors*: items whose records were consumed by a join. They + stay in `rec` (a later join may consume the same records again) but are excluded from recordset + extraction; `dom*(rec) = dom(rec) \ J` are the *live* anchors. -Six **working-state update operations** populate or modify the working state: +Seven **working-state update operations** populate or modify the working state: | Operation | Symbol | Effect | |---|---|---| @@ -246,10 +249,11 @@ Six **working-state update operations** populate or modify the working state: | Suffix | `O_suffix^δ` | Appends provider strings (joined by `δ`) to the anchor's value/attribute | | AVP construction | `O_avp` | Creates an attribute-value pair `(attr(ι₁), val(ι_anch))` for the anchor VAL item using the single ATTR item `ι₁` returned by the provider | | Record construction | `O_rec` | Creates an item-based record with the anchor VAL item as its first element and the provided VAL items as the remaining elements | -| Record join | `O_join^K` | Merges previously created records; key positions `K` are dropped from joined records, duplicate named attributes are removed, and the merged result is stored under the anchor | +| Record concatenation | `O_concat^K` | Folds previously created records into one wide record under the anchor: the key positions `K` (at which all records must agree) are not repeated; the concatenated anchors are removed from `dom(rec)`. A named attribute shared by two records (apart from the key) is a precondition violation: no effect, a diagnostic is recorded | +| Record join | `O_join^K` | The record product: every record of the anchor is combined with every record of the provided anchors — a cross product for `K = ∅`, an equi-join on the key positions `K` otherwise; a shared named attribute is a natural-join condition (kept once if the values agree, the pair is dropped otherwise). The provided anchors are added to `J` | ??? note "API mapping — WorkingState" - **Definition (Working state):** `ws = (V, A, val, attr, avp, rec)`. + **Definition (Working state):** `ws = (V, A, val, attr, avp, rec, J)`. | Formal component | Python | |---|---| @@ -258,10 +262,12 @@ Six **working-state update operations** populate or modify the working state: | `val(ι)` | `WorkingState.val(item)` | | `attr(ι)` | `WorkingState.attr(item)` | | `avp(ι)` | `WorkingState.avp(item)` → `AttributeValuePair(attribute, value)` | - | `rec(ι)` | `WorkingState.rec(item)` → `List` | + | `rec(ι)` | `WorkingState.rec(item)` → the records of the anchor (a list of item sequences), also for joined-away anchors | + | `J` | `WorkingState.all_joined()`, `WorkingState.is_joined(item)` | + | `dom*(rec) = dom(rec) \ J` | `WorkingState.live_anchors()` — the **live** anchors only, in insertion order; this is what recordset extraction sees | | Derived `assoc(ι)` | `WorkingState.assoc(item)` — attribute of `avp(ι)`, or `null` | - Six working-state update operations: + Seven working-state update operations: | Formal operation | Python factory | Anchor type | |---|---|---| @@ -270,14 +276,34 @@ Six **working-state update operations** populate or modify the working state: | `O_suffix^δ` | `WorkingState.applySuffix(anchor, items, delimiter)` | VAL or ATTR | | `O_avp` | `WorkingState.applyAvp(anchor, items)` | VAL | | `O_rec` | `WorkingState.applyRec(anchor, items)` | cell-derived VAL | - | `O_join^K` | `WorkingState.applyJoin(anchor, items, keyPositions)` | cell-derived VAL | + | `O_concat^K` | `WorkingState.apply_concat(anchor, items, key)` | cell-derived VAL | + | `O_join^K` | `WorkingState.apply_join(anchor, items, key)` | cell-derived VAL | + + A violated precondition of `O_concat^K` / `O_join^K` has no effect and is recorded as a + `Diagnostic` (surfaced as `TableInterpreter.diagnostics()`); + `TableInterpreter.with_strict_preconditions(True)` raises a `RuntimeError` instead. + The interpreter also reports, through the same channel, the "not applicable" cases of an + *explicit* `CONCAT`/`JOIN` — one written on the anchor's own content spec: an anchor without a + record (`anchor has no record — REC missing?`) and provided items none of which has a record. + Inherited actions (`actSpecs` of a table/subtable/row/subrow scope) reach anchors that were + never meant to carry records, so for them these cases are routine and stay silent. An anchor + whose record was folded away by an earlier concatenation is routine as well: the working state + keeps such anchors in the set `C` (`is_concatenated`, `all_concatenated`), the counterpart of + `J` for `O_concat`. + + **Implementation note — named keys.** The manuscript defines `K` over positions only + (`K ⊆ ℕ₀`). The implementation additionally accepts key *attribute names* (`RecordKey`: + positions and/or names; RTL `CONCAT(0, 'A')`, `JOIN('Year')`). A name is resolved per record, + through `assoc`, to the position of the item carrying that attribute; `drop_K` and `compat_K` + then apply to the resolved positions, so the semantics of the operations is unchanged — a + named key is a position-independent way of writing the same `K`. Consistency predicates: | Predicate | Python factory | |---|---| - | Basic consistency: `rec(ι)[0] = ι` and `avp(ι) = (a,v) ⟹ val(ι) = v` | `WorkingState.isConsistent()` | - | Recordset-consistency: uniform anchor attribute + distinct per-record attributes | `WorkingState.isRecordsetConsistent()` | + | Basic consistency: `ρ[0] = ι` for every `ρ ∈ rec(ι)`, and `avp(ι) = (a,v) ⟹ val(ι) = v` | `WorkingState.isConsistent()` | + | Recordset-consistency over the live anchors: uniform anchor attribute + distinct attributes in every record | `WorkingState.is_recordset_consistent()` | ### Interpretation actions @@ -299,7 +325,8 @@ satisfy the constraints of the chosen operation (Tab. I in the paper): | String modification | VAL or ATTR (cell-derived) | any | ≥ 1 | | AVP construction | VAL (cell- or context-derived) | ATTR provider | = 1 | | Record construction | VAL (cell-derived) | VAL providers | ≥ 0 | -| Record join | VAL (cell-derived) | VAL providers (cell-derived) | ≥ 0 | +| Record concatenation | VAL (cell-derived) | VAL providers (cell-derived) | ≥ 1 | +| Record join | VAL (cell-derived) | VAL providers (cell-derived) | ≥ 1 | ??? note "API — ActionSpec" **Action spec:** `S_act = (op, ⟨S_prov¹, …, S_provⁿ⟩)` where `op` is a @@ -309,8 +336,10 @@ satisfy the constraints of the chosen operation (Tab. I in the paper): |---|---|---| | `REC` | `ActionSpec.rec(providers…)` | Anchor item → record; providers supply the remaining fields | | `AVP` | `ActionSpec.avp(provider)` | Associates a VAL item (anchor) with an ATTR item from the provider | - | `JOIN` | `ActionSpec.join(providers…)` | Joins item-based records; dedup by named attribute (K=∅) | - | `JOIN(K)` | `ActionSpec.join(Set.of(0), providers…)` | Joins with key positions K dropped (e.g. `JOIN(0)` drops the anchor position) | + | `CONCAT` | `ActionSpec.concat(*providers)` | Folds the provided records into the anchor's record (K=∅) | + | `CONCAT(K)` | `ActionSpec.concat(*providers, key=0)`, `key="A"`, `key=(0, 1, "A")`, `key=RecordKey.of(…)` | Same, key K (positions and/or attribute names) not repeated (e.g. `key=0` drops the anchor position of each provided record) | + | `JOIN` | `ActionSpec.join(*providers)` | Record product: one record per (anchor record × provided record) | + | `JOIN(K)` | `ActionSpec.join(*providers, key=0)`, `key="k"` | Equi-join on the key K (positions and/or attribute names) | | `FILL` | `ActionSpec.fill(delimiter, providers…)` | Fills anchor value using provider values | | `PREFIX` | `ActionSpec.prefix(delimiter, providers…)` | Prepends provider values to the anchor | | `SUFFIX` | `ActionSpec.suffix(delimiter, providers…)` | Appends provider values to the anchor | @@ -327,7 +356,12 @@ satisfy the constraints of the chosen operation (Tab. I in the paper): ## Recordset and schema **Definition (Recordset):** given a schema `S = ⟨a₁, …, aₙ⟩`, a *record* is an -n-tuple `⟨(a₁,v₁), …, (aₙ,vₙ)⟩`; a *recordset* is a finite sequence of records. +n-tuple `⟨(a₁,v₁), …, (aₙ,vₙ)⟩`; a *recordset* is a finite **multiset** of records — as in the +relational model, the order of records is not part of the result, whereas duplicates are. +Implementations materialize a recordset as a sequence (e.g. for CSV export); pyRegTab, like the +reference implementation, emits records in the order in which their anchors were visited during +working state completion and, for an anchor carrying several records after a join, in the +nested-loop order of the join. This order is a documented default, not a guarantee of the model. ??? note "API mapping — Schema, Record, Recordset" | Formal concept | Python class / method | @@ -349,7 +383,7 @@ An initial working state `ws₀` is constructed directly from the semantic layer - each VAL item is assigned its value `val(ι) ∈ V`; - each ATTR item is assigned its attribute `attr(ι) ∈ A`; -- `dom(avp)` and `dom(rec)` are initialised to empty. +- `dom(avp)`, `dom(rec)` and `J` are initialised to empty. ### Phase 2 — Working state completion @@ -358,7 +392,8 @@ The interpretation actions `A` are applied to `ws₀` in a fixed order: 1. String-modification actions (`O_fill`, `O_prefix`, `O_suffix`); 2. AVP-construction actions (`O_avp`); 3. Record-construction actions (`O_rec`); -4. Record-join actions (`O_join`). +4. Record-concatenation actions (`O_concat`) — records are folded … +5. Record-join actions (`O_join`) — … before they are multiplied. Within each phase, actions are applied in *traversal order* over their anchor items. The default strategy visits anchors in row-major order @@ -384,12 +419,13 @@ item-based records: `S = ⟨a₁, a₂, …, aₙ⟩`; unnamed items receive a fresh anonymous attribute for their position. -**Record generation** iterates over `dom(rec)` in the order in which record-construction -actions were applied: +**Record generation** iterates over the live anchors `dom*(rec) = dom(rec) \ J` in the order in +which record-construction actions were applied, and over the records `ρ ∈ rec(ι)` of each anchor +(several after a join): -- For each anchor `ι`, initialise all `n` field values to a *missing value* (via +- For each record `ρ`, initialise all `n` field values to a *missing value* (via an optional user-defined handler `μ`; default: `⊥`). -- For each item `ι'` in `rec(ι)` that has an associated attribute in `S`, fill in +- For each item `ι'` in `ρ` that has an associated attribute in `S`, fill in `val(ι')` at the corresponding position. - Emit the resulting record `⟨(a₁, v₁), …, (aₙ, vₙ)⟩`. diff --git a/docs/rtl-reference.md b/docs/rtl-reference.md index 2a1d85a..0ed50ca 100644 --- a/docs/rtl-reference.md +++ b/docs/rtl-reference.md @@ -106,11 +106,18 @@ cell patterns: | `?` | 0 or 1 | | `*` | 0 or more | | `+` | 1 or more | -| `{n}` | exactly *n* | +| `{n}` | exactly *n* (*n* ≥ 0; `{1}` is the same as no quantifier, `{0}` matches nothing, i.e. an empty match) | The `+` on the outer row pattern above means "one or more data rows"; the `{2}` means "exactly two value cells". +**Empty matches.** An explicit subrow (or subtable) whose children are all optional may +match zero cells (rows) — for instance `{ [BLANK]* }` in a row without blank cells. Such a +subrow matches the empty sequence, exactly like `*` in a regular expression: it never fails +the row, it is tried at the end of the row as well, and a repeated empty match (`{ [BLANK]* }+`, +`{ [BLANK]* }*`, `{ [BLANK]* }{3}`) counts as a single empty iteration rather than looping. +An empty subrow covers no cells and therefore never appears in the interpretable table. + **Inherited action specs** — `[acts]` placed at the table, subtable, row, or subrow level are inherited by all descendant cells. Inherited actions are merged with any local actions on the cell's `contSpec`. Incompatible inherited actions (e.g. `COL->AVP` on an `ATTR` anchor) are @@ -414,18 +421,62 @@ provSpecs -> op | `REC(n)` | `prov->REC(n)` | Same + move the anchor attribute (name with its values) to position *n* | | `REC('s')` | `prov->REC('s')` | Same + split field values by delimiter *s* | | `AVP` | `prov->AVP` | Associate anchor (VAL) with an attribute from the provider (ATTR) | -| `JOIN` | `prov->JOIN` | Join item-based records: all items included, then dedup by named attribute (K=∅) | -| `JOIN(K)` | `prov->JOIN(0)` | Join with key positions K dropped from each joined record before dedup (e.g. `JOIN(0)` drops the anchor position) | +| `CONCAT` | `prov->CONCAT` | Concatenate the provided records to the anchor's record — one wide record, the provided anchors are removed (K=∅: all items included) | +| `CONCAT(K)` | `prov->CONCAT(0)`, `prov->CONCAT(0, 'A')` | Same, with the key K not repeated: dropped from each concatenated record, and all records must agree there (e.g. `CONCAT(0)` drops the anchor of each concatenated record). K lists key **positions** (0-based) and/or key **attribute names** (string literals, resolved per record). A named attribute shared by two records (apart from the key) is an error: the action has no effect and a diagnostic is reported | +| `JOIN` | `prov->JOIN` | Record product: every record of the anchor is combined with every provided record (cross product); the provided anchors are joined-away. A named attribute shared by two records acts as a natural-join condition | +| `JOIN(K)` | `prov->JOIN(0)`, `prov->JOIN('Year')` | Equi-join on the key K (positions and/or attribute names): a record pair is combined only if it agrees at K, the key of the joined record is not repeated | | `FILL('s')` | `prov->FILL('/')` | Fill anchor value forward from provider, separated by *s* | | `PREFIX('s')` | `prov->PREFIX(' ')` | Prepend provider value to anchor, separated by *s* | | `SUFFIX('s')` | `prov->SUFFIX(' ')` | Append provider value to anchor, separated by *s* | +`CONCAT` folds, `JOIN` multiplies: with a single provided record the two coincide, with two or more +they diverge — `CONCAT` yields one wider record, `JOIN` yields one record per provided record. +Up to pyRegTab 0.5.x (jRegTab 0.5.x) the folding operation was spelled `JOIN(K)`; a pattern +written for 0.5.x must replace `JOIN(K)` by `CONCAT(K)`. + +Both operate on *records*: a `CONCAT`/`JOIN` on an anchor that has no `REC` has no effect — the +anchor simply does not reach the recordset — and is reported through +`TableInterpreter.diagnostics()` (`anchor has no record — REC missing?`), as is an explicit +`CONCAT`/`JOIN` none of whose provided items has a record. Add `()->REC` when the anchor's own +value is the whole record (`[(VAL: COL->AVP, ()->REC, RT*->JOIN){';'}]`). Only actions written +on the anchor's own content spec are reported; actions inherited from a row/subrow/subtable/table +level are applied to every cell and skip such anchors silently. + +**Choosing the key positions `K`.** `K` is any number of 0-based positions in the item-based +record — `0` is the anchor, the following positions are the items in the order the `REC` +providers supplied them. At every position in `K` all records being concatenated must agree, and +the item is not repeated in the result; every other position is carried over as is, so a named +attribute that occurs at such a position in two records is a conflict. Put into `K` **every** +position that repeats in each row of a group — the anchor plus all fields that are identical +across the group. With rows `k1 | k11 | a1:A | b1:B | c1` and `k1 | k11 | a1:A | b1:B | c2`, the key +is four positions, `CONCAT(0,1,2,3)`, not two: `A` and `B` repeat exactly like `k1` and `k11` +(`CONCAT(0,1)` would report the shared attribute `A` and leave both rows unfolded). A row whose +`A` differs within the group is then rejected with a diagnostic instead of being folded silently. +`K = ∅` is right only when the records share nothing, not even the anchor (task 069). + +**Naming the key.** A key field that carries a named attribute may be listed by that name +instead of its position: `CONCAT(0,1,'A','B')` is the same key as `CONCAT(0,1,2,3)` for the rows +above. The name is resolved to a position **per record** — the item whose attribute is `A`, wherever +it sits — so the pattern no longer depends on the order of the fields: if the repeated fields sit to +the right of the varying one (`k1 | k11 | c1 | a1:A | b1:B`), the positional key becomes +`CONCAT(0,1,3,4)` while the named key stays `CONCAT(0,1,'A','B')`. Positions and names may be +mixed; a name missing from one of the records, or carried with a different value, is a key +mismatch (no effect, diagnostic). Names are compared exactly (case-sensitive). For `JOIN` a shared +named attribute is already a natural-join condition, so `JOIN('A')` differs from the bare `JOIN` +only in dropping the pairs where one side lacks `A`; the named key matters mostly for `CONCAT`, +where a shared attribute outside the key is a conflict rather than a condition. The canonical +form (serializer) lists positions in ascending order, then names in lexicographic order, +single-quoted: `CONCAT(0, 1, 'A', 'B')`. + Examples by operation: ```rtl [VAL : ST*->REC] // REC, collect whole subtable (Task 01) [VAL : SR->REC(1)]{2} // REC(1), name the record by attribute at position 1 (Task 03) [VAL: 'AIRLINE'->AVP] // AVP with a literal attribute (Illustrative example) +[VAL : RT->REC, BW&STR*->CONCAT(0)] // CONCAT(0): fold the rows below with the same key into one record (Task 16) +[VAL : RT*->REC, BW&STR*->CONCAT(0,'A')] // CONCAT(0,'A'): the same, with the field named A as part of the key (concat_named_key) +[(VAL: COL->AVP, ()->REC, RT*->JOIN){';'}] // JOIN: one record per token × per cell to the right (join_product) [VAL: -AV->PREFIX(', ')] // PREFIX: prepend the value above, separator ", " (Task 116) [BLANK ? VAL#'H': -LT&!BLANK->FILL | …] // FILL: copy the nearest non-blank cell to the left (Task 107) ``` @@ -541,7 +592,7 @@ A quoted string literal supplies a fixed string as an attribute or value: ('AIRLINE')->AVP ``` -The item type is inferred from the action: `->AVP` → ATTR, `->REC` → VAL. +The item type is inferred from the action: `->AVP` → ATTR, `->REC` / `->CONCAT` / `->JOIN` → VAL. --- @@ -554,7 +605,9 @@ The item type is inferred from the action: `->AVP` → ATTR, `->REC` → VAL. | `^COL->AVP` | Associate with an attribute from the same column (column-major) | | `('LABEL')->AVP` | Associate with a fixed string attribute | | `(ST*)->REC` (in parentheses) | Same as `ST*->REC` but explicit grouping | -| `CL->JOIN(0)` | Join (drop anchor) another item from the same cell | +| `CL->CONCAT(0)` | Concatenate (drop anchor) the record of another item from the same cell | +| `BW&STR*->CONCAT(0)` | Fold the rows below with the same key into the anchor's record (group by key) | +| `RT*->JOIN` | One record per cell to the right (record product, e.g. explode × stack) | | `(COL)->FILL('/')` | Fill forward from same-column values, delimiter `/` | | `-AV->PREFIX(', ')` | Prepend the nearest value above, separator ", " | diff --git a/grammar/RTL.g4 b/grammar/RTL.g4 index d9d17c1..bdcbea0 100644 --- a/grammar/RTL.g4 +++ b/grammar/RTL.g4 @@ -1,272 +1,275 @@ -// Regular Table Language (RTL) Grammar -grammar RTL ; - -options { caseInsensitive = true ; } - -// Quantifier -quantifier : zeroOrOne | zeroOrMore | oneOrMore | exactly ; - -zeroOrOne : QUESTION ; -zeroOrMore : MULT ; -oneOrMore : PLUS ; -exactly : LCURLY INT RCURLY ; - -// Named fragment definitions (preamble): $name=[body] or $name={body} -fragmentDef : FRAGMENT_ID ASSIGN LSQUARE cellPatternBody? RSQUARE // cell: $N=[cellBody] - | FRAGMENT_ID ASSIGN LSQUARE rowPatternBody RSQUARE // row: $N=[ [sub]+ ] - | FRAGMENT_ID ASSIGN LCURLY subrowPatternBody RCURLY // subrow: $N={ [cell]+ } - | FRAGMENT_ID ASSIGN LCURLY subtablePatternBody RCURLY // subtable: $N={ [row]+ } - ; - -// Table pattern -tablePattern : fragmentDef* (cellMatchCond QUESTION)? settings? actSpecs? subtablePattern+ ; - -// Optional settings prefix -settings : LANGLE setting (COMMA setting)* RANGLE ; -setting : normSetting | anchSetting | splitSetting ; -normSetting : 'NORM' ; -anchSetting : 'ANCH' LPAREN INT RPAREN ; -splitSetting : 'SPLIT' LPAREN STRING RPAREN ; - -// Subtable pattern: implicit, explicit, or fragment reference -subtablePattern : implSubtablePattern - | explSubtablePattern - | LCURLY FRAGMENT_ID RCURLY quantifier? // subtable fragment ref {$N} - ; - -// Implicit subtable pattern -implSubtablePattern : rowPattern+ ; - -// Explicit subtable pattern -explSubtablePattern : LCURLY subtablePatternBody RCURLY quantifier? ; -subtablePatternBody : (cellMatchCond QUESTION)? (actSpecs)? rowPattern+ ; - -// Row pattern -rowPattern : LSQUARE rowPatternBody RSQUARE quantifier? // regular - | LSQUARE FRAGMENT_ID RSQUARE quantifier? // row fragment ref [$N] - ; -rowPatternBody : (cellMatchCond QUESTION)? (actSpecs)? subrowPattern+ ; - -// Subrow pattern: implicit, explicit, or fragment reference -subrowPattern : implSubrowPattern - | explSubrowPattern - | LCURLY FRAGMENT_ID RCURLY quantifier? // subrow fragment ref {$N} - ; - -// Implicit subrow pattern -implSubrowPattern : cellPattern+ ; - -// Explicit subrow pattern -explSubrowPattern : LCURLY subrowPatternBody RCURLY quantifier? ; -subrowPatternBody : (cellMatchCond QUESTION)? (actSpecs)? cellPattern+ ; - -// Cell pattern -cellPattern : LSQUARE cellPatternBody? RSQUARE quantifier? // regular - | LSQUARE FRAGMENT_ID RSQUARE quantifier? // fragment reference [$name] - ; -cellPatternBody : cellMatchCond QUESTION actSpecs? contSpec - | cellMatchCond - | actSpecs? contSpec - ; - -// Content specification: atomic, delimited, compound, conditional -contSpec : atomContSpec | delimContSpec | compContSpec | condContSpec ; - -// Atomic content specification -atomContSpec : itemDerivDir tags? (ASSIGN strExtr)? (COLON actSpecs)? ; - -// Item derivation directive -itemDerivDir : ATTRIBUTE | VALUE | AUXILIARY | SKIPPED ; -ATTRIBUTE : 'ATTR' ; -VALUE : 'VAL' ; -AUXILIARY : 'AUX' ; -SKIPPED : 'SKIP' | '_' ; - -// User-defined tags -tags : tagItem+ ; -tagItem : HASH STRING ; - -// Item string extractor (supports chains: =REPL("x","").TRIM) -strExtr : strExtrStep ('.' strExtrStep)* ; -strExtrStep : substr | replace | norm | upperCase | lowerCase | trim ; - -// String processing -substr : 'SUBSTR' LPAREN INT COMMA INT RPAREN ; -replace : 'REPL' LPAREN STRING COMMA STRING RPAREN ; -norm : 'NORM' ; -upperCase : 'UC' ; -lowerCase : 'LC' ; -trim : 'TRIM' ; - -// Interpretation action specifications -actSpecs : actSpec (COMMA actSpec)* ; - -// Interpretation action specification -actSpec : provSpecs RIGHT_ARROW op ; -op : fillOp | prefixOp | suffixOp | AVP | recOp | joinOp ; -fillOp : FILL (LPAREN STRING RPAREN)? ; -prefixOp : PREFIX (LPAREN STRING RPAREN)? ; -suffixOp : SUFFIX (LPAREN STRING RPAREN)? ; -recOp : REC (LPAREN (INT | STRING) RPAREN)? ; -joinOp : JOIN (LPAREN INT (COMMA INT)* RPAREN)? ; -FILL : 'FILL' ; -PREFIX : 'PREFIX' ; -SUFFIX : 'SUFFIX' ; -AVP : 'AVP' ; -REC : 'REC' ; -JOIN : 'JOIN' ; - -provSpecs : provSpec | (LPAREN provSpec (COMMA provSpec)* RPAREN) | LPAREN RPAREN ; - -// Delimited content specification -delimContSpec : LPAREN atomContSpec RPAREN LCURLY separator RCURLY; - -separator : STRING ; // Separator - -// Compound content specification -compContSpec : openDelim? compSeg (separator compSeg)* closeDelim? ; -compSeg : atomContSpec | delimContSpec ; - -openDelim : STRING ; // Opening delimiter -closeDelim : STRING ; // Closing delimiter - -// Conditional content specification -condContSpec : cellMatchCond QUESTION (xContSpec VBAR xContSpec) ; -xContSpec : atomContSpec | delimContSpec | compContSpec ; - -// Cell match condition -cellMatchCond : cellMatchConstr ; -cellMatchConstr : regex | blank | contains | ext ; - -// Item provider specification -provSpec : tblProvSpec | ctxProvSpec | ctxAvpSpec ; - -// Cell derived item provider specification -// Single bare form: spatConstr only (avoids ambiguity with ctxProvSpec STRING). -// Bare & -conjunction starting with spatConstr is also unambiguous (keywords, not STRING). -// Disjunctions (|) and content-only constraints still require parentheses. -tblProvSpec : traversalOrderMark? (spatConstr | LPAREN constraints RPAREN | bareConjConstraints) cardinality? ; - -// Bare & -conjunction of constraints starting with a spatConstr keyword -bareConjConstraints : spatConstr (AMP baseConstr)+ ; - -// Traversal order mark (absence = ROW_MAJOR) -traversalOrderMark : reverseRowMajor | columnMajor | reverseColumnMajor ; -reverseRowMajor : MINUS ; -columnMajor : CARET ; -reverseColumnMajor : MINUS CARET ; - -// Context derived item provider specification -ctxProvSpec : STRING ; - -// Constant attribute-value pair provider: @'ATTR'='VALUE' -ctxAvpSpec : AT STRING ASSIGN STRING ; - -// Cardinality k: {n} = at most n; * = UNBOUNDED (0..*); absent = at most 1 (default) -cardinality : LCURLY INT RCURLY | MULT ; - -// Constraints (| has lower precedence than &; parentheses for explicit grouping) -constraints : orGroup (VBAR orGroup)* ; -orGroup : baseConstr (AMP baseConstr)* ; -baseConstr : constr | LPAREN constraints RPAREN ; -constr : spatConstr | contConstr ; - -// Spatial constraints -spatConstr : LEFT_OF | RIGHT_OF | ABOVE | BELOW | SAME_ROW | SAME_COLUMN - | SAME_SUBROW | SAME_SUBCOLUMN | SAME_SUBTABLE | NOT_SAME_CELL | SAME_CELL - | col | row | pos ; - -LEFT_OF : 'LT' ; // sameSubrow(a) && col < col(a) -RIGHT_OF : 'RT' ; // sameSubrow(a) && col > col(a) -ABOVE : 'AV' ; // sameSubcol(a) && row < row(a) -BELOW : 'BW' ; // sameSubcol(a) && row > row(a) -SAME_ROW : 'ROW' ; // sameRow(a) && !sameCell(a) -SAME_COLUMN : 'COL' ; // sameCol(a) && !sameCell(a) -SAME_SUBROW : 'SR' ; // sameSubrow(a) && !sameCell(a) -SAME_SUBCOLUMN : 'SC' ; // sameSubcol(a) && !sameCell(a) -SAME_SUBTABLE : 'ST' ; // sameSubtable(a) && !sameCell(a) -NOT_SAME_CELL : 'NCL' ; // !sameCell(a) -SAME_CELL : 'CL' ; // sameCell(a) - -// Positional constraints -row : 'R' (range | offset | INT) ; -col : 'C' (range | offset | INT) ; -pos : 'P' (range | offset | INT) ; - -range : start DOUBLE_PERIOD end? ; -start : offset | INT ; -end : offset | INT ; - -offset : (MINUS INT) | (PLUS INT) ; - -// Content constraints -contConstr : regex | blank | tag | sameStr | contains | ext ; - -// External Java binding: EXT('name') — resolved against Bindings at compile time -ext : EXT LPAREN STRING RPAREN ; -EXT : 'EXT' ; - -// Contains constraint -contains : EXCLAMATION? TILDA STRING ; -TILDA : '~' ; - -tag : EXCLAMATION? tagItem ; - -sameStr : STR ; -STR : 'STR' ; - -regex : EXCLAMATION? STRING ; - -blank : EXCLAMATION? 'BLANK' ; - -PLUS : '+' ; -MINUS : '-' ; -CARET : '^' ; -MULT : '*' ; -AMP : '&' ; - -LPAREN : '(' ; -RPAREN : ')' ; -LCURLY : '{' ; -RCURLY : '}' ; -LSQUARE : '[' ; -RSQUARE : ']' ; -LANGLE : '<' ; -RANGLE : '>' ; - -COLON : ':' ; -COMMA : ',' ; -QUESTION : '?' ; -VBAR : '|' ; -EXCLAMATION : '!' ; - -DOUBLE_PERIOD : '..' ; -ASSIGN : '=' ; - -RIGHT_ARROW : '->' ; - -HASH : '#' ; -AT : '@' ; - -// Fragment identifier: $name (single token avoids keyword conflicts with caseInsensitive=true) -FRAGMENT_ID : '$' [A-Z][A-Z0-9_]* ; - -INT : [0-9]+ ; - -STRING - : '"' (ESC | '""' | ~["])* '"' - | '\'' (ESC | '\'\'' | ~['])* '\'' - | '“' (ESC | .)*? ('”' | '″') // smart quotes - ; - -fragment ESC - : '`\'' // backtick single-quote - | '`"' // backtick double-quote - ; - -WS : [ \r\t \n]+ -> channel(HIDDEN) ; - -ZWNBSP : []+ -> channel(HIDDEN) ; // Remove UTF8 BOM character - -LineComment - : '//' ~[\r\n]* -> channel(HIDDEN) ; +// Regular Table Language (RTL) Grammar +grammar RTL ; + +options { caseInsensitive = true ; } + +// Quantifier +quantifier : zeroOrOne | zeroOrMore | oneOrMore | exactly ; + +zeroOrOne : QUESTION ; +zeroOrMore : MULT ; +oneOrMore : PLUS ; +exactly : LCURLY INT RCURLY ; + +// Named fragment definitions (preamble): $name=[body] or $name={body} +fragmentDef : FRAGMENT_ID ASSIGN LSQUARE cellPatternBody? RSQUARE // cell: $N=[cellBody] + | FRAGMENT_ID ASSIGN LSQUARE rowPatternBody RSQUARE // row: $N=[ [sub]+ ] + | FRAGMENT_ID ASSIGN LCURLY subrowPatternBody RCURLY // subrow: $N={ [cell]+ } + | FRAGMENT_ID ASSIGN LCURLY subtablePatternBody RCURLY // subtable: $N={ [row]+ } + ; + +// Table pattern +tablePattern : fragmentDef* (cellMatchCond QUESTION)? settings? actSpecs? subtablePattern+ ; + +// Optional settings prefix +settings : LANGLE setting (COMMA setting)* RANGLE ; +setting : normSetting | anchSetting | splitSetting ; +normSetting : 'NORM' ; +anchSetting : 'ANCH' LPAREN INT RPAREN ; +splitSetting : 'SPLIT' LPAREN STRING RPAREN ; + +// Subtable pattern: implicit, explicit, or fragment reference +subtablePattern : implSubtablePattern + | explSubtablePattern + | LCURLY FRAGMENT_ID RCURLY quantifier? // subtable fragment ref {$N} + ; + +// Implicit subtable pattern +implSubtablePattern : rowPattern+ ; + +// Explicit subtable pattern +explSubtablePattern : LCURLY subtablePatternBody RCURLY quantifier? ; +subtablePatternBody : (cellMatchCond QUESTION)? (actSpecs)? rowPattern+ ; + +// Row pattern +rowPattern : LSQUARE rowPatternBody RSQUARE quantifier? // regular + | LSQUARE FRAGMENT_ID RSQUARE quantifier? // row fragment ref [$N] + ; +rowPatternBody : (cellMatchCond QUESTION)? (actSpecs)? subrowPattern+ ; + +// Subrow pattern: implicit, explicit, or fragment reference +subrowPattern : implSubrowPattern + | explSubrowPattern + | LCURLY FRAGMENT_ID RCURLY quantifier? // subrow fragment ref {$N} + ; + +// Implicit subrow pattern +implSubrowPattern : cellPattern+ ; + +// Explicit subrow pattern +explSubrowPattern : LCURLY subrowPatternBody RCURLY quantifier? ; +subrowPatternBody : (cellMatchCond QUESTION)? (actSpecs)? cellPattern+ ; + +// Cell pattern +cellPattern : LSQUARE cellPatternBody? RSQUARE quantifier? // regular + | LSQUARE FRAGMENT_ID RSQUARE quantifier? // fragment reference [$name] + ; +cellPatternBody : cellMatchCond QUESTION actSpecs? contSpec + | cellMatchCond + | actSpecs? contSpec + ; + +// Content specification: atomic, delimited, compound, conditional +contSpec : atomContSpec | delimContSpec | compContSpec | condContSpec ; + +// Atomic content specification +atomContSpec : itemDerivDir tags? (ASSIGN strExtr)? (COLON actSpecs)? ; + +// Item derivation directive +itemDerivDir : ATTRIBUTE | VALUE | AUXILIARY | SKIPPED ; +ATTRIBUTE : 'ATTR' ; +VALUE : 'VAL' ; +AUXILIARY : 'AUX' ; +SKIPPED : 'SKIP' | '_' ; + +// User-defined tags +tags : tagItem+ ; +tagItem : HASH STRING ; + +// Item string extractor (supports chains: =REPL("x","").TRIM) +strExtr : strExtrStep ('.' strExtrStep)* ; +strExtrStep : substr | replace | norm | upperCase | lowerCase | trim ; + +// String processing +substr : 'SUBSTR' LPAREN INT COMMA INT RPAREN ; +replace : 'REPL' LPAREN STRING COMMA STRING RPAREN ; +norm : 'NORM' ; +upperCase : 'UC' ; +lowerCase : 'LC' ; +trim : 'TRIM' ; + +// Interpretation action specifications +actSpecs : actSpec (COMMA actSpec)* ; + +// Interpretation action specification +actSpec : provSpecs RIGHT_ARROW op ; +op : fillOp | prefixOp | suffixOp | AVP | recOp | concatOp | joinOp ; +fillOp : FILL (LPAREN STRING RPAREN)? ; +prefixOp : PREFIX (LPAREN STRING RPAREN)? ; +suffixOp : SUFFIX (LPAREN STRING RPAREN)? ; +recOp : REC (LPAREN (INT | STRING) RPAREN)? ; +concatOp : CONCAT (LPAREN keyRef (COMMA keyRef)* RPAREN)? ; +joinOp : JOIN (LPAREN keyRef (COMMA keyRef)* RPAREN)? ; +keyRef : INT | STRING ; // key position (0-based) or key attribute name +FILL : 'FILL' ; +PREFIX : 'PREFIX' ; +SUFFIX : 'SUFFIX' ; +AVP : 'AVP' ; +REC : 'REC' ; +CONCAT : 'CONCAT' ; +JOIN : 'JOIN' ; + +provSpecs : provSpec | (LPAREN provSpec (COMMA provSpec)* RPAREN) | LPAREN RPAREN ; + +// Delimited content specification +delimContSpec : LPAREN atomContSpec RPAREN LCURLY separator RCURLY; + +separator : STRING ; // Separator + +// Compound content specification +compContSpec : openDelim? compSeg (separator compSeg)* closeDelim? ; +compSeg : atomContSpec | delimContSpec ; + +openDelim : STRING ; // Opening delimiter +closeDelim : STRING ; // Closing delimiter + +// Conditional content specification +condContSpec : cellMatchCond QUESTION (xContSpec VBAR xContSpec) ; +xContSpec : atomContSpec | delimContSpec | compContSpec ; + +// Cell match condition +cellMatchCond : cellMatchConstr ; +cellMatchConstr : regex | blank | contains | ext ; + +// Item provider specification +provSpec : tblProvSpec | ctxProvSpec | ctxAvpSpec ; + +// Cell derived item provider specification +// Single bare form: spatConstr only (avoids ambiguity with ctxProvSpec STRING). +// Bare & -conjunction starting with spatConstr is also unambiguous (keywords, not STRING). +// Disjunctions (|) and content-only constraints still require parentheses. +tblProvSpec : traversalOrderMark? (spatConstr | LPAREN constraints RPAREN | bareConjConstraints) cardinality? ; + +// Bare & -conjunction of constraints starting with a spatConstr keyword +bareConjConstraints : spatConstr (AMP baseConstr)+ ; + +// Traversal order mark (absence = ROW_MAJOR) +traversalOrderMark : reverseRowMajor | columnMajor | reverseColumnMajor ; +reverseRowMajor : MINUS ; +columnMajor : CARET ; +reverseColumnMajor : MINUS CARET ; + +// Context derived item provider specification +ctxProvSpec : STRING ; + +// Constant attribute-value pair provider: @'ATTR'='VALUE' +ctxAvpSpec : AT STRING ASSIGN STRING ; + +// Cardinality k: {n} = at most n; * = UNBOUNDED (0..*); absent = at most 1 (default) +cardinality : LCURLY INT RCURLY | MULT ; + +// Constraints (| has lower precedence than &; parentheses for explicit grouping) +constraints : orGroup (VBAR orGroup)* ; +orGroup : baseConstr (AMP baseConstr)* ; +baseConstr : constr | LPAREN constraints RPAREN ; +constr : spatConstr | contConstr ; + +// Spatial constraints +spatConstr : LEFT_OF | RIGHT_OF | ABOVE | BELOW | SAME_ROW | SAME_COLUMN + | SAME_SUBROW | SAME_SUBCOLUMN | SAME_SUBTABLE | NOT_SAME_CELL | SAME_CELL + | col | row | pos ; + +LEFT_OF : 'LT' ; // sameSubrow(a) && col < col(a) +RIGHT_OF : 'RT' ; // sameSubrow(a) && col > col(a) +ABOVE : 'AV' ; // sameSubcol(a) && row < row(a) +BELOW : 'BW' ; // sameSubcol(a) && row > row(a) +SAME_ROW : 'ROW' ; // sameRow(a) && !sameCell(a) +SAME_COLUMN : 'COL' ; // sameCol(a) && !sameCell(a) +SAME_SUBROW : 'SR' ; // sameSubrow(a) && !sameCell(a) +SAME_SUBCOLUMN : 'SC' ; // sameSubcol(a) && !sameCell(a) +SAME_SUBTABLE : 'ST' ; // sameSubtable(a) && !sameCell(a) +NOT_SAME_CELL : 'NCL' ; // !sameCell(a) +SAME_CELL : 'CL' ; // sameCell(a) + +// Positional constraints +row : 'R' (range | offset | INT) ; +col : 'C' (range | offset | INT) ; +pos : 'P' (range | offset | INT) ; + +range : start DOUBLE_PERIOD end? ; +start : offset | INT ; +end : offset | INT ; + +offset : (MINUS INT) | (PLUS INT) ; + +// Content constraints +contConstr : regex | blank | tag | sameStr | contains | ext ; + +// External Java binding: EXT('name') — resolved against Bindings at compile time +ext : EXT LPAREN STRING RPAREN ; +EXT : 'EXT' ; + +// Contains constraint +contains : EXCLAMATION? TILDA STRING ; +TILDA : '~' ; + +tag : EXCLAMATION? tagItem ; + +sameStr : STR ; +STR : 'STR' ; + +regex : EXCLAMATION? STRING ; + +blank : EXCLAMATION? 'BLANK' ; + +PLUS : '+' ; +MINUS : '-' ; +CARET : '^' ; +MULT : '*' ; +AMP : '&' ; + +LPAREN : '(' ; +RPAREN : ')' ; +LCURLY : '{' ; +RCURLY : '}' ; +LSQUARE : '[' ; +RSQUARE : ']' ; +LANGLE : '<' ; +RANGLE : '>' ; + +COLON : ':' ; +COMMA : ',' ; +QUESTION : '?' ; +VBAR : '|' ; +EXCLAMATION : '!' ; + +DOUBLE_PERIOD : '..' ; +ASSIGN : '=' ; + +RIGHT_ARROW : '->' ; + +HASH : '#' ; +AT : '@' ; + +// Fragment identifier: $name (single token avoids keyword conflicts with caseInsensitive=true) +FRAGMENT_ID : '$' [A-Z][A-Z0-9_]* ; + +INT : [0-9]+ ; + +STRING + : '"' (ESC | '""' | ~["])* '"' + | '\'' (ESC | '\'\'' | ~['])* '\'' + | '“' (ESC | .)*? ('”' | '″') // smart quotes + ; + +fragment ESC + : '`\'' // backtick single-quote + | '`"' // backtick double-quote + ; + +WS : [ \r\t \n]+ -> channel(HIDDEN) ; + +ZWNBSP : []+ -> channel(HIDDEN) ; // Remove UTF8 BOM character + +LineComment + : '//' ~[\r\n]* -> channel(HIDDEN) ; diff --git a/grammar/UPSTREAM b/grammar/UPSTREAM index 2207657..1642ad3 100644 --- a/grammar/UPSTREAM +++ b/grammar/UPSTREAM @@ -1,4 +1,4 @@ -commit: 035ff1a139e885e4cea85aa66a33e89a6b30f8c9 -tag: v0.5.0 +commit: f0093339139f828353db31ad822dd73f68c72eab +tag: v0.7.1 path: src/main/antlr4/ru/icc/regtab/rtl/RTL.g4 -sha256: 4fffbdf3f2dcb13935b8f062b5c6c55321de08cf130a4a28fa0845f5b48d68c0 +sha256: 81387334831f495a468c3673a43a92392be08bf4e73cda95a0e9ef183fe75714 diff --git a/plans/INDEX.md b/plans/INDEX.md index 24bb9c4..d8c10f7 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -3,6 +3,11 @@ Указатель по каталогу `plans/`: один план — одна крупная работа, самый свежий сверху. Статус и результат каждой работы — в шапке и в разделе «Результат» самого плана. +- [JREGTAB_0_7_1_PARITY.md](JREGTAB_0_7_1_PARITY.md) — паритет с jRegTab 0.7.1: + `CONCAT(K)` (свёртка) и `JOIN(K)` (произведение записей), именованный ключ, диагностика + интерпретатора, индекс провайдеров (линейно по числу ячеек), сабровы нулевой ширины, + `{0}`/`{1}`; CLI-раннер `python -m pyregtab.runner`, побайтно совпадающий с Java-раннером + regtab-eval-on-atbench на 244 решениях. **РЕАЛИЗОВАН** (2026-09-08). - [ANCH_MOVES_ATTRIBUTE.md](ANCH_MOVES_ATTRIBUTE.md) — `ANCH(n)`/`REC(n)` переставляет сам атрибут (имя вместе со значениями), а не только значения; синк двух семантических conformance-кейсов и паритет с jRegTab 0.5.1. **РЕАЛИЗОВАН** (2026-08-26). diff --git a/plans/JREGTAB_0_7_1_PARITY.md b/plans/JREGTAB_0_7_1_PARITY.md new file mode 100644 index 0000000..928fef7 --- /dev/null +++ b/plans/JREGTAB_0_7_1_PARITY.md @@ -0,0 +1,155 @@ +# План: паритет с jRegTab 0.7.1 и замена jRegTab в regtab-eval-on-atbench + +**Статус:** РЕАЛИЗОВАН (2026-09-08; результаты и отклонения от плана — в §7) +**Дата:** 2026-09-08 +**Upstream:** `d:\YandexDisk\code2\jregtab` @ v0.7.1 (`f009333`); pyRegTab до этого плана — +паритет с jRegTab 0.5.1. +**Характер:** ломающее изменение семантики (`JOIN` → произведение записей, прежняя +свёртка — `CONCAT`), новый синтаксис (`CONCAT`, именованный ключ, `{0}`/`{1}`), +диагностика интерпретатора, оптимизация провайдеров без изменения результата, +исправление матчера. Версия 0.7.1 (номер синхронизирован с jRegTab). + +--- + +## 1. Контекст + +jRegTab с 0.5.1 до 0.7.1 получил шесть работ (`jregtab/plans/INDEX.md`), pyRegTab отстал по +всем. Потребитель — `regtab-eval-on-atbench`: 244 RTL-решения на ATBench, раннер +`tools/regtab-runner/RtlRunner.java` на `ru.icc.regtab:regtab:0.7.1`; три решения используют +новый `JOIN` (произведение), страта subtitle зависит от сабровов нулевой ширины, крупные +таблицы (до 1,19 млн ячеек) — от индекса провайдеров 0.5.2. Цель: pyRegTab исполняет все 244 +решения с побайтно тем же `output.csv`, что и Java-раннер, и проходит новый conformance-корпус. + +| jRegTab | план апстрима | что переносится | +|---|---|---| +| 0.5.2 | `PERF_LARGE_TABLES.md` | пространственный индекс над items, область кандидатов из спецификации фильтра, ранний выход по k | +| 0.5.3 | `PERF_MEMORY_LAYOUT.md` | Java-специфично (лямбды `IntRange`, `IdentityHashMap`); в Rust-ядре уже компактно — не переносится | +| 0.6.0 | `ITM_JOIN_PRODUCT.md` | `CONCAT(K)` = прежний `JOIN(K)` без dedup и с предусловиями; `JOIN(K)` = произведение записей; `rec` многозначная, множество `J`; `Diagnostic`, строгий режим | +| 0.7.0 | `RTL_NAMED_RECORD_KEY.md` | `RecordKey` (позиции и/или имена), `CONCAT(0,'A')`, канонический вид `CONCAT(0, 1, 'A', 'B')` | +| 0.7.0 | `DIAG_ANCHOR_WITHOUT_RECORD.md` | «anchor has no record — REC missing?», «none of the provided items has a record…», флаг `inherited` у действия, множество `C` | +| 0.7.1 | `ZERO_WIDTH_SUBROW.md` | сабров/субтаблица нулевой ширины — пустое совпадение; `{0}`/`{1}`; ошибка `{n}` как диагностика компиляции | +| 0.5.0 | `CONFORMANCE_SEMANTIC_SECTION.md` | раннер уже есть; синк 11 новых семантических кейсов, 2 negative, 2 curated positive | + +## 2. Установленные факты + +- Семантика провайдера в [src/semantics.rs](../src/semantics.rs) — полный перебор + `sem.cell_items` + сортировка на каждый вызов: квадратично по числу ячеек, как в jRegTab 0.5.1. + `SemanticsCore` неизменяем после построения, поэтому индекс строится один раз в `interpret` + и передаётся в `provide`; области кандидатов (`CandidateScope`) выводятся из `FilterCond` + один раз при инстанцировании провайдера (`matcher::to_provider_inst`). +- Порядок линеаризации: `(row, col)` по τ, внутри ячейки — `index` по возрастанию (в том + числе для обратных обходов) — как `ItemLinearization.comparator`. Стабильная сортировка + сохраняет порядок создания при равенстве. +- `WorkingState.rec: IndexMap>` — одна запись на якорь; `apply_join` + делает свёртку с dedup (0.5.x). +- Матчер: `while (stack.len() < max) && i < n` и `cells[next - 1]` — те же два дефекта, что + в Java 0.7.0 (`ZERO_WIDTH_SUBROW.md`). `Quantifier::exactly` требует `n >= 2`. +- `tests/atp_patterns.py` генерируется `tools/translate_atp.py` из Java-тестов апстрима; + 12 задач с `JOIN` переехали на `CONCAT`, `task_098` — на ключ `{0,1,2,3}`. +- Conformance: `diff -r` показывает 12 изменённых пар positive (`JOIN`→`CONCAT`), 2 новые + пары, 2 negative, 11 semantic-кейсов, README, VERSION. Фикстуры задач апстрима не менялись. +- Java есть локально (`JAVA_HOME=C:\Java\jdk-25.0.2`), jar-раннер atbench собран — можно + сравнить выходы напрямую на всех 244 решениях. + +## 3. Ядро (Rust) + +1. `spec.rs`: `OperationType::Concat`; `RecordKey { positions, names }` с валидацией + (позиции ≥ 0, имена непустые); `ActionSpec.key: RecordKey` вместо `key_positions`; + проверки `ActionSpec::new` для `Concat` как для `Join`; `Quantifier::exactly(n)` при `n ≥ 0`. +2. `rtl/`: ключевое слово `CONCAT`; `keyRef : INT | STRING` для `CONCAT(...)`/`JOIN(...)`; + пустое имя — ошибка компиляции с позицией; `{n}` вне диапазона `int` — ошибка с позицией; + сериализатор — позиции по возрастанию, затем имена лексикографически в одинарных кавычках. +3. `semantics.rs`: `rec: IndexMap>>`, множества `J` и `C`, + `Diagnostic`, `report` (строгий режим — `Err`), `apply_concat` (предусловия (i)–(iii), + no effect + диагностика), `apply_join` (произведение, `compat_K`, `agree`, `dedup`, + отложенное потребление), `live_anchors()`; проверки согласованности по живым якорям и + каждой записи. `ItemIndex` + `CandidateScope` — порт `CellDerivedItemIndex`, + `CandidateScope`, `CandidateScopes`. +4. `interp.rs`: пять корзин `str → avp → rec → concat → join`; `check_records` для явных + действий; `InterpreterCfg.strict_preconditions`; `interpret` возвращает и диагностику; + схема и записи — по тройкам (якорь, запись, позиция) над живыми якорями. +5. `matcher.rs`: снятие `i < n`, пустая итерация (`next == i`) → `min = 0`, break; пустые + сабровы/субтаблицы не регистрируются в ITM; `ActionInst.inherited`; `scope` в + `ProviderInst::Cell`. + +## 4. Python-слой + +- `RecordKey`, `Diagnostic`, `OperationType.CONCAT`; `ActionSpec.concat(...)`, + `ActionSpec.join(...)` с `key=` (int | str | RecordKey | iterable), `key_positions=` сохранён; + `ActionSpec.key` / `.key_positions`; `TableInterpreter.with_strict_preconditions(bool)`, + `.diagnostics()`; `Quantifier.exactly(0|1)`. +- `pyregtab.dsl`: `concat(...)`, `join(...)` с ведущим ключом (int / str / set / RecordKey). +- CLI-раннер `python -m pyregtab.runner ` — контракт `RtlRunner.java` из + regtab-eval-on-atbench: `./input.csv` → `./output.csv`, заголовок всегда, RFC 4180 с + многострочными ячейками, неровные строки дополняются, `None` → `""`, коды 0/1/2; + диагностики интерпретатора — в stderr. + +## 5. Тесты + +- Rust: порт `WorkingStateConcatTest`/`WorkingStateJoinTest`, рандомизированная + эквивалентность нового `provide` полному перебору (аналог + `CellDerivedItemProviderEquivalenceTest`), сабровы нулевой ширины. +- Python: порт `TableInterpreterMultiRecordTest` (`tests/test_join_concat.py`), кванторы + `{0}`/`{1}`, DSL `concat`; `tests/atp_patterns.py` регенерируется переводчиком; + `test_dsl.py` — `JOIN(0)` → `CONCAT(0)`; task-раннер и раннер semantic-секции требуют + пустых `diagnostics()` (как в апстриме). +- Внешняя проверка: все 244 решения regtab-eval-on-atbench через Java-раннер и через + `python -m pyregtab.runner`, побайтное сравнение `output.csv`; время на крупных входах. + +## 6. Документация и версия + +`docs/rtl-reference.md`, `docs/model/itm.md`, `docs/model/atp.md`, `docs/api.md`, +`docs/architecture.md`, `docs/embedded-rtl.md`, `docs/index.md`, `README.md` — порт правок +апстрима в Python-терминах; `grammar/RTL.g4` + `grammar/UPSTREAM`, `conformance/` + +`conformance/UPSTREAM` — v0.7.1; версия 0.7.1 в `Cargo.toml`, `pyproject.toml`, +`python/pyregtab/__init__.py`; CI: differential-job на v0.7.1. + +## 7. Результат + +``` +cargo test --no-default-features 45 passed (было 21; +10 WorkingState concat, +8 join, + +1 рандомизированная эквивалентность + индексированного provide полному перебору + (> 10 000 вызовов на 25 случайных таблицах), + +5 сабровы нулевой ширины) +cargo test (с фичей python) 45 passed +cargo clippy --all-targets -D warnings — чисто в обеих конфигурациях +pytest tests -q 1981 passed (было 1925: +11 semantic-кейсов ×2, + +2 negative, +2 positive ×2, +28 test_join_concat.py) +mkdocs build --strict чисто +tools/check_grammar_sync.py OK (пин v0.7.1, sha256 81387334…) +diff -r -x UPSTREAM ../jregtab/conformance conformance — расхождений нет +``` + +**Внешняя проверка (regtab-eval-on-atbench, 244 решения с данными):** Java-раннер на +`ru.icc.regtab:regtab:0.7.1` и `python -m pyregtab.runner` дают **побайтно одинаковый +`output.csv` во всех 244 кейсах** (скрипт `atbench_diff.py` в scratchpad сессии; итог +`Counter({'same': 244})`). Суммарное время 244 прогонов: Java 71 с (включая старт JVM), +pyRegTab 46 с. Самый крупный вход `stack_test11` (84 888 × 14 = 1,19 млн ячеек, identity-паттерн): +Java 7,0 с, pyRegTab 7,5 с (парсинг CSV 1,0 с, матчинг 0,7 с, интерпретация 5,2 с, запись 0,6 с); +до индекса провайдеров этот кейс в pyRegTab был квадратичным (как в jRegTab 0.5.1). + +**Отклонения от плана и решения по ходу:** + +1. `RecordKey` в Python: фабрики `positions(…)`/`names(…)` из Java конфликтуют с одноимёнными + свойствами, поэтому оставлены свойства `positions`/`names`, конструктор + `RecordKey(positions=(), names=())`, `RecordKey.of(positions=None, names=None)` и `empty()`. + `ActionSpec.concat/join(*providers, key=…)` принимает int, str, `RecordKey` или итерируемое + из позиций и имён; `key_positions=` и одноимённый getter сохранены для кода под 0.5.x. +2. `ActionSpec.key_positions` (Rust) переименован в `key: RecordKey`; `ActionSpec::key_positions()` + оставлен как аксессор. +3. Пустые сабровы/субтаблицы в `SyntaxMatch` не регистрируются вовсе (в Java они остаются в + `MatchResult` как пустые интервалы, но в ITM не попадают); наблюдаемое поведение то же. +4. Целочисленный литерал вне диапазона `int` отклоняется лексером с позицией + («Invalid integer literal 99999999999: out of int range»), а не только в кванторе. +5. Строгий режим — `RuntimeError` (Java: `IllegalStateException`) с текстом + ` skipped at CellDerivedItem[…]: `. +6. Дополнительно к плану: `Recordset.to_csv(..., quote_all=False, newline=" +")` — формат + раннера пишется ядром (запись 1,19 млн записей 4,5 с → 0,6 с); `WorkingState` на `FxHash` + (`rustc-hash`), проверки согласованности и генерация записей без аллокаций на запись + (интерпретация stack_test11 7,0 с → 5,2 с). +7. `PERF_MEMORY_LAYOUT` (0.5.3) не переносился: Rust-ядро и так хранит ячейки/items плоскими + структурами. +8. Regtab-eval-on-atbench не менялся: замена `java -jar regtab-runner.jar ` на + `python -m pyregtab.runner ` в `harness/run.py` — решение того проекта. diff --git a/pyproject.toml b/pyproject.toml index 4b3ff0b..5fc6780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "pyregtab" -version = "0.5.1" +version = "0.7.1" description = "pyRegTab: pattern-based extraction of recordsets from tables (RTL / ATP / ITM)" readme = "README.md" license = { text = "MIT" } diff --git a/python/pyregtab/__init__.py b/python/pyregtab/__init__.py index 38f3178..2f84a93 100644 --- a/python/pyregtab/__init__.py +++ b/python/pyregtab/__init__.py @@ -42,6 +42,7 @@ ItemFilterConditionSpec, StringExtractor, ProviderSpec, + RecordKey, ActionSpec, AtomicContentSpec, DelimitedContentSpec, @@ -55,6 +56,7 @@ # atp / interpret AtpMatcher, TableInterpreter, + Diagnostic, SchemaConstructionStrategy, ActionApplicationStrategy, WhitespaceNormalization, @@ -73,7 +75,7 @@ from pyregtab import dsl -__version__ = "0.5.1" +__version__ = "0.7.1" __all__ = [ "TableSyntax", "Cell", "Row", "Subrow", "Subtable", "GridPosition", @@ -83,10 +85,12 @@ "ItemDerivationDirective", "OperationType", "TraversalOrder", "CellDerivedProviderKind", "ContextDerivedProviderKind", "Quantifier", "CellPredicate", "CellMatchCondition", "FilterTerm", - "ItemFilterConditionSpec", "StringExtractor", "ProviderSpec", "ActionSpec", + "ItemFilterConditionSpec", "StringExtractor", "ProviderSpec", "RecordKey", + "ActionSpec", "AtomicContentSpec", "DelimitedContentSpec", "CompoundContentSpec", "ConditionalContentSpec", "CellPattern", "SubrowPattern", "RowPattern", "SubtablePattern", "TablePattern", "AtpMatcher", "TableInterpreter", + "Diagnostic", "SchemaConstructionStrategy", "ActionApplicationStrategy", "WhitespaceNormalization", "AnchorAttributeAtPosition", "DelimitedFieldSplit", "FieldSplitting", "SchemaReordering", "Bindings", diff --git a/python/pyregtab/__main__.py b/python/pyregtab/__main__.py new file mode 100644 index 0000000..89b439e --- /dev/null +++ b/python/pyregtab/__main__.py @@ -0,0 +1,7 @@ +"""``python -m pyregtab `` — alias of ``python -m pyregtab.runner``.""" + +import sys + +from pyregtab.runner import main + +sys.exit(main()) diff --git a/python/pyregtab/_core.pyi b/python/pyregtab/_core.pyi index dc49d44..5811c8c 100644 --- a/python/pyregtab/_core.pyi +++ b/python/pyregtab/_core.pyi @@ -42,6 +42,7 @@ class OperationType(Enum): SUFFIX = ... AVP = ... REC = ... + CONCAT = ... JOIN = ... class TraversalOrder(Enum): @@ -212,6 +213,9 @@ class Recordset: *, sep: str = ",", missing: str = "", + quote_all: bool = False, + newline: str = " +", ) -> Optional[str]: ... # ---------------------------------------------------------------- spec @@ -424,9 +428,31 @@ class ProviderSpec: def ctx_avp(attr_name: str, value: str) -> ProviderSpec: ... def is_context_literal(self) -> bool: ... +class RecordKey: + """The key K of CONCAT(K) / JOIN(K): 0-based positions in the item-based + record (0 is the anchor) and/or attribute names, resolved per record.""" + positions: set[int] + names: set[str] + def __init__( + self, positions: Optional[Iterable[int]] = None, names: Optional[Iterable[str]] = None + ) -> None: ... + @staticmethod + def empty() -> RecordKey: ... + @staticmethod + def of( + positions: Optional[Iterable[int]] = None, names: Optional[Iterable[str]] = None + ) -> RecordKey: ... + def is_empty(self) -> bool: ... + +# A key reference: an int position, a str attribute name, a RecordKey, or an +# iterable mixing positions and names. +KeyRef = Union[int, str, RecordKey, Iterable[Union[int, str]], None] + class ActionSpec: operation_type: OperationType inherited: bool + key: RecordKey + key_positions: set[int] def __init__( self, operation_type: OperationType, @@ -436,6 +462,7 @@ class ActionSpec: split_delimiter: Optional[str] = None, key_positions: Optional[set[int]] = None, inherited: bool = False, + key: KeyRef = None, ) -> None: ... @staticmethod def rec( @@ -449,9 +476,17 @@ class ActionSpec: provider: Union[ProviderSpec, ItemFilterConditionSpec, str] ) -> ActionSpec: ... @staticmethod + def concat( + *providers: Union[ProviderSpec, ItemFilterConditionSpec], + key: KeyRef = None, + key_positions: KeyRef = None, + cardinality: int = 1, + ) -> ActionSpec: ... + @staticmethod def join( *providers: Union[ProviderSpec, ItemFilterConditionSpec], - key_positions: Union[int, Iterable[int], None] = None, + key: KeyRef = None, + key_positions: KeyRef = None, cardinality: int = 1, ) -> ActionSpec: ... @staticmethod @@ -687,7 +722,16 @@ class TableInterpreter: self, transformations: Sequence[RecordsetTransformation] ) -> TableInterpreter: ... def with_anonymous_attribute_template(self, template: str) -> TableInterpreter: ... + def with_strict_preconditions(self, strict: bool) -> TableInterpreter: ... def interpret(self, table: InterpretableTable) -> Recordset: ... + def diagnostics(self) -> list[Diagnostic]: ... + +class Diagnostic: + """A CONCAT/JOIN action that had no effect during the most recent + interpretation: a violated precondition or a missing record.""" + anchor: CellDerivedItem + operation: str + message: str # ---------------------------------------------------------------- rtl diff --git a/python/pyregtab/dsl.py b/python/pyregtab/dsl.py index 7943d18..1cdfc4b 100644 --- a/python/pyregtab/dsl.py +++ b/python/pyregtab/dsl.py @@ -44,6 +44,7 @@ ItemFilterConditionSpec, ProviderSpec, Quantifier, + RecordKey, RowPattern, StringExtractor, SubrowPattern, @@ -81,7 +82,8 @@ # context providers "lit", "ctx_avp", # actions - "rec", "rec_split", "avp", "join", "fill", "prefix", "suffix", + "rec", "rec_split", "avp", "concat", "join", "fill", "prefix", "suffix", + "RecordKey", # builder / helper types "Prov", "Acts", ] @@ -818,7 +820,7 @@ def __init__(self, attribute: str, value: str): def lit(text: str) -> _Ctx: - """Context literal (RTL ``'EUR'``): VALUE under REC/JOIN, ATTRIBUTE otherwise.""" + """Context literal (RTL ``'EUR'``): VALUE under REC/CONCAT/JOIN, ATTRIBUTE otherwise.""" return _Ctx(text) @@ -833,7 +835,7 @@ def ctx_avp(attribute: str, value: str) -> _CtxAvp: def _kind_for(op: str) -> str: - if op in ("rec", "join"): + if op in ("rec", "concat", "join"): return "val" if op == "avp": return "attr" @@ -848,7 +850,7 @@ def _resolve(providers: tuple, op: str) -> list: out.append(arg._spec(kind)) elif isinstance(arg, _Ctx): out.append( - ProviderSpec.ctx_val(arg.text) if op in ("rec", "join") + ProviderSpec.ctx_val(arg.text) if op in ("rec", "concat", "join") else ProviderSpec.ctx_attr(arg.text) ) elif isinstance(arg, _CtxAvp): @@ -879,13 +881,31 @@ def avp(provider: Union[Prov, str]) -> ActionSpec: return ActionSpec.avp(provider._spec("attr")) -def join(*args: Union[int, set, _ProvArg]) -> ActionSpec: - """RTL ``(...)->JOIN``; a leading int or set is the key position(s) ``JOIN(k)``.""" - key_positions = None - if args and isinstance(args[0], (int, set)): - key_positions = args[0] - args = args[1:] - return ActionSpec.join(*_resolve(args, "join"), key_positions=key_positions) +_KeyArg = Union[int, str, set, frozenset, tuple, list, RecordKey] + + +def _split_key(args): + """A leading key reference — int position, str attribute name, set/tuple/list + of both, or ``RecordKey`` — is the key K of ``CONCAT(K)`` / ``JOIN(K)``.""" + if args and isinstance(args[0], (int, str, set, frozenset, tuple, list, RecordKey)): + return args[0], args[1:] + return None, args + + +def concat(*args: Union[_KeyArg, _ProvArg]) -> ActionSpec: + """RTL ``(...)->CONCAT`` — fold the provided records into the anchor's record; + a leading int/str/set/``RecordKey`` is the key ``CONCAT(k)`` / ``CONCAT('A')`` / + ``CONCAT(0, 'A')``.""" + key, args = _split_key(args) + return ActionSpec.concat(*_resolve(args, "concat"), key=key) + + +def join(*args: Union[_KeyArg, _ProvArg]) -> ActionSpec: + """RTL ``(...)->JOIN`` — the record product (cross product with the provided + records); a leading int/str/set/``RecordKey`` is the equi-join key ``JOIN(k)`` / + ``JOIN('A')``.""" + key, args = _split_key(args) + return ActionSpec.join(*_resolve(args, "join"), key=key) def fill(*args: Union[str, _ProvArg]) -> ActionSpec: diff --git a/python/pyregtab/runner.py b/python/pyregtab/runner.py new file mode 100644 index 0000000..fd5c02f --- /dev/null +++ b/python/pyregtab/runner.py @@ -0,0 +1,136 @@ +"""CLI runner of RTL solutions — the pyRegTab counterpart of the +``regtab-runner`` (``RtlRunner.java``) used by regtab-eval-on-atbench, with the +same I/O contract so that one replaces the other in a harness: + + python -m pyregtab.runner + +reads ``./input.csv`` (RFC 4180: quoted fields may contain commas, doubled +quotes and line breaks; empty lines are skipped; ragged rows are padded with +empty cells to the widest row), applies the RTL pattern (``RtlCompiler.compile`` +→ ``AtpMatcher.match`` → ``TableInterpreter`` with ``RECORD_FIRST`` → +``pattern.transform``) and writes ``./output.csv``: the schema as the header +row (always), one row per record, every field double-quoted, a missing value +as the empty string, UTF-8, LF line endings. + +Exit codes: 0 — success, 1 — the pattern did not match, 2 — invalid usage; +an RTL compile error raises (non-zero exit, message on stderr). Interpreter +diagnostics (skipped CONCAT/JOIN actions) are printed to stderr and do not +change the exit code. ``--input`` / ``--output`` override the file names; +``--strict`` turns diagnostics into errors. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from pyregtab import ( + AtpMatcher, + Recordset, + RtlCompiler, + SchemaConstructionStrategy, + TableInterpreter, + TableSyntax, +) + + +def parse_csv(text: str) -> list[list[str]]: + """RFC 4180 parser mirroring ``RtlRunner.parseCsv``: a quoted field may + contain commas, doubled quotes and line breaks; CRLF, CR and LF all end a + row; empty lines are skipped (a line holding only ``""`` is a row with one + empty field).""" + rows: list[list[str]] = [] + fields: list[str] = [] + cur: list[str] = [] + in_quotes = False + quoted = False # a quote was seen on this line: "" is a field, not an empty line + i, n = 0, len(text) + while i < n: + ch = text[i] + if in_quotes: + if ch != '"': + cur.append(ch) + elif i + 1 < n and text[i + 1] == '"': + cur.append('"') + i += 1 + else: + in_quotes = False + elif ch == '"': + in_quotes = True + quoted = True + elif ch == ",": + fields.append("".join(cur)) + cur = [] + elif ch in "\r\n": + if ch == "\r" and i + 1 < n and text[i + 1] == "\n": + i += 1 # CRLF is one line break + if fields or cur or quoted: + fields.append("".join(cur)) + rows.append(fields) + fields = [] + cur = [] + quoted = False + else: + cur.append(ch) + i += 1 + if fields or cur or quoted: + fields.append("".join(cur)) + rows.append(fields) + return rows + + +def load_table(path: Path) -> TableSyntax: + rows = parse_csv(path.read_text(encoding="utf-8")) + num_rows = len(rows) + num_cols = max((len(r) for r in rows), default=0) + syntax = TableSyntax(num_rows, num_cols) + for r, row in enumerate(rows): + for c in range(num_cols): + syntax.cell(r, c).set_text(row[c] if c < len(row) else "") + return syntax + + +def write_csv(path: Path, rs: Recordset) -> None: + """The regtab-runner output format: the schema as the header row, every + field double-quoted (quotes doubled), a missing value as the empty + string, UTF-8, LF line endings.""" + rs.to_csv(path, quote_all=True, newline="\n") + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog="python -m pyregtab.runner", + description="Apply an RTL pattern to input.csv and write output.csv (regtab-runner contract).", + ) + ap.add_argument("pattern", help="path to the .rtl solution") + ap.add_argument("--input", default="input.csv", help="input table (default: ./input.csv)") + ap.add_argument("--output", default="output.csv", help="output recordset (default: ./output.csv)") + ap.add_argument("--strict", action="store_true", + help="fail on a skipped CONCAT/JOIN action instead of reporting it") + try: + args = ap.parse_args(argv) + except SystemExit as e: + return 2 if e.code else 0 + + rtl = Path(args.pattern).read_bytes().decode("utf-8") + syntax = load_table(Path(args.input)) + pattern = RtlCompiler.compile(rtl) + itm = AtpMatcher.match(pattern, syntax) + if itm is None: + print("RTL pattern did not match input", file=sys.stderr) + return 1 + interpreter = ( + TableInterpreter() + .with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + .with_strict_preconditions(args.strict) + ) + rs = pattern.transform(interpreter.interpret(itm)) + for d in interpreter.diagnostics(): + print(f"diagnostic: {d}", file=sys.stderr) + write_csv(Path(args.output), rs) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/interp.rs b/src/interp.rs index 6b25c89..38efce2 100644 --- a/src/interp.rs +++ b/src/interp.rs @@ -1,7 +1,13 @@ //! Port of `ru.icc.regtab.interpret.TableInterpreter`: 4 interpretation phases. +//! +//! Working state completion applies the actions in operation-type order +//! `FILL/PREFIX/SUFFIX → AVP → REC → CONCAT → JOIN`: records are folded by +//! concatenation before they are multiplied by joins. Preconditions violated +//! by `CONCAT`/`JOIN` leave the working state unchanged and are reported +//! through the returned [`Diagnostic`]s (or fail under strict preconditions). use crate::recordset::{RecordCore, RecordsetCore, Schema}; -use crate::semantics::{ActionInst, ItemId, OpInst, SemanticsCore, WorkingState}; +use crate::semantics::{ActionInst, Diagnostic, ItemId, ItemIndex, OpInst, SemanticsCore, WorkingState}; use crate::spec::{EvalEnv, ItemType, PyFunc, Transformation}; use crate::syntax::SyntaxCore; use crate::util::CoreResult; @@ -27,6 +33,9 @@ pub struct InterpreterCfg { pub missing_value_handler: Option, pub transformations: Vec, pub anonymous_attribute_template: String, + /// A violated `CONCAT`/`JOIN` precondition fails the interpretation + /// instead of having no effect (Java: `withStrictPreconditions(true)`). + pub strict_preconditions: bool, } impl Default for InterpreterCfg { @@ -37,20 +46,28 @@ impl Default for InterpreterCfg { missing_value_handler: None, transformations: Vec::new(), anonymous_attribute_template: "$a_%i".to_string(), + strict_preconditions: false, } } } +/// The recordset of an interpretation together with the diagnostics of its +/// working state completion (empty when no action was skipped). +pub struct Interpretation { + pub recordset: RecordsetCore, + pub diagnostics: Vec, +} + pub fn interpret( cfg: &InterpreterCfg, syntax: &SyntaxCore, sem: &SemanticsCore, py_table: Option<&crate::spec::PyTableHandle>, -) -> CoreResult { +) -> CoreResult { let env = EvalEnv { syntax, py_table }; // Phase 1: working state initialization - let mut ws = WorkingState::default(); + let mut ws = WorkingState::new(cfg.strict_preconditions); for (i, item) in sem.cell_items.iter().enumerate() { match item.ty { ItemType::Value => { @@ -76,6 +93,7 @@ pub fn interpret( // Phase 2: working state completion complete_working_state(cfg, &mut ws, sem, &env)?; + let diagnostics = ws.take_diagnostics(); // Phase 3: recordset extraction let recordset = extract_recordset(cfg, &mut ws, sem)?; @@ -85,7 +103,7 @@ pub fn interpret( for t in &cfg.transformations { rs = t.with_template(&cfg.anonymous_attribute_template).apply(rs)?; } - Ok(rs) + Ok(Interpretation { recordset: rs, diagnostics }) } fn anchor_pos(sem: &SemanticsCore, action: &ActionInst) -> Option<(usize, usize)> { @@ -98,7 +116,7 @@ fn anchor_pos(sem: &SemanticsCore, action: &ActionInst) -> Option<(usize, usize) } } -fn sort_actions(cfg: &InterpreterCfg, sem: &SemanticsCore, actions: &mut Vec<&ActionInst>) { +fn sort_actions(cfg: &InterpreterCfg, sem: &SemanticsCore, actions: &mut [&ActionInst]) { actions.sort_by(|a, b| { let pa = anchor_pos(sem, a); let pb = anchor_pos(sem, b); @@ -123,6 +141,7 @@ fn complete_working_state( let mut str_actions: Vec<&ActionInst> = Vec::new(); let mut avp_actions: Vec<&ActionInst> = Vec::new(); let mut rec_actions: Vec<&ActionInst> = Vec::new(); + let mut concat_actions: Vec<&ActionInst> = Vec::new(); let mut join_actions: Vec<&ActionInst> = Vec::new(); for action in &sem.actions { @@ -130,6 +149,7 @@ fn complete_working_state( OpInst::Fill(_) | OpInst::Prefix(_) | OpInst::Suffix(_) => str_actions.push(action), OpInst::Avp => avp_actions.push(action), OpInst::Rec => rec_actions.push(action), + OpInst::Concat(_) => concat_actions.push(action), OpInst::Join(_) => join_actions.push(action), } } @@ -137,11 +157,15 @@ fn complete_working_state( sort_actions(cfg, sem, &mut str_actions); sort_actions(cfg, sem, &mut avp_actions); sort_actions(cfg, sem, &mut rec_actions); + sort_actions(cfg, sem, &mut concat_actions); sort_actions(cfg, sem, &mut join_actions); - for group in [str_actions, avp_actions, rec_actions, join_actions] { + // One spatial index per interpretation, shared by all providers. + let index = ItemIndex::build(sem, env.syntax); + + for group in [str_actions, avp_actions, rec_actions, concat_actions, join_actions] { for action in group { - apply_action(ws, sem, env, action)?; + apply_action(ws, sem, env, &index, action)?; } } Ok(()) @@ -151,12 +175,13 @@ fn apply_action( ws: &mut WorkingState, sem: &SemanticsCore, env: &EvalEnv, + index: &ItemIndex, action: &ActionInst, ) -> CoreResult<()> { let anchor = action.anchor; let mut items: Vec = Vec::new(); for provider in &action.providers { - items.extend(provider.provide(anchor, sem, env)?); + items.extend(provider.provide(anchor, sem, env, index)?); } match &action.op { OpInst::Fill(d) => ws.apply_fill(sem, anchor, &items, d), @@ -171,16 +196,64 @@ fn apply_action( } } OpInst::Rec => ws.apply_rec(sem, anchor, &items), - OpInst::Join(kp) => { - if !items.is_empty() { - ws.apply_join(anchor, &items, kp) - } else { - Ok(()) + OpInst::Concat(key) => { + if items.is_empty() { + return Ok(()); } + check_records(ws, sem, action, &items, "CONCAT")?; + ws.apply_concat(sem, anchor, &items, key) + } + OpInst::Join(key) => { + if items.is_empty() { + return Ok(()); + } + check_records(ws, sem, action, &items, "JOIN")?; + ws.apply_join(sem, anchor, &items, key) } } } +/// Makes the silent "not applicable" cases of `CONCAT`/`JOIN` visible for +/// *explicit* actions: an anchor without a record, or provided items none of +/// which has a record — both usually a forgotten `REC`. Inherited actions +/// reach anchors that were never meant to carry records, so for them these +/// cases are routine and not reported. An anchor whose record was folded away +/// by an earlier `CONCAT` (ι ∈ C) is routine as well: its own `CONCAT` is +/// applied after the one that consumed it. +fn check_records( + ws: &mut WorkingState, + sem: &SemanticsCore, + action: &ActionInst, + items: &[ItemId], + operation: &str, +) -> CoreResult<()> { + if action.inherited { + return Ok(()); + } + let ItemId::Cell(anchor) = action.anchor else { + return Ok(()); + }; + if !ws.has_rec(anchor) { + if !ws.is_concatenated(anchor) { + ws.report(sem, anchor, operation, "anchor has no record — REC missing?")?; + } + return Ok(()); + } + for &item in items { + if let ItemId::Cell(c) = item { + if c != anchor && (ws.has_rec(c) || ws.is_concatenated(c)) { + return Ok(()); + } + } + } + ws.report( + sem, + anchor, + operation, + "none of the provided items has a record — REC missing on the provider side?", + ) +} + fn anonymous_attribute(cfg: &InterpreterCfg, index: usize) -> String { cfg.anonymous_attribute_template.replace("%i", &index.to_string()) } @@ -198,8 +271,47 @@ fn extract_recordset( Ok(RecordsetCore { schema, records }) } +/// Visit order of `(anchor, record, position)` triples for schema construction +/// (port of `SchemaConstructionStrategy.buildVisitOrder`): the records of an +/// anchor are always visited in their sequence order. +fn visit_order( + strategy: SchemaStrategy, + anchors: &[usize], + ws: &WorkingState, +) -> Vec<(usize, usize, usize)> { + let mut out = Vec::new(); + match strategy { + SchemaStrategy::RecordFirst => { + for (a, &anchor) in anchors.iter().enumerate() { + for (r, seq) in ws.rec(anchor).into_iter().flatten().enumerate() { + for i in 1..seq.len() { + out.push((a, r, i)); + } + } + } + } + SchemaStrategy::PositionFirst => { + let mut max_len = 0; + for &anchor in anchors { + for seq in ws.rec(anchor).into_iter().flatten() { + max_len = max_len.max(seq.len()); + } + } + for i in 1..max_len { + for (a, &anchor) in anchors.iter().enumerate() { + let n = ws.rec(anchor).map(|rs| rs.len()).unwrap_or(0); + for r in 0..n { + out.push((a, r, i)); + } + } + } + } + } + out +} + fn construct_schema(cfg: &InterpreterCfg, ws: &mut WorkingState) -> CoreResult { - let anchors: Vec = ws.rec.keys().copied().collect(); + let anchors: Vec = ws.live_anchors(); let mut schema_attrs: Vec = Vec::new(); let mut anon_map: HashMap = HashMap::new(); @@ -225,42 +337,15 @@ fn construct_schema(cfg: &InterpreterCfg, ws: &mut WorkingState) -> CoreResult = match cfg.strategy { - SchemaStrategy::RecordFirst => { - let mut out = Vec::new(); - for (a, &anchor) in anchors.iter().enumerate() { - let len = ws.rec.get(&anchor).map(|s| s.len()).unwrap_or(0); - for i in 1..len { - out.push((a, i)); - } - } - out - } - SchemaStrategy::PositionFirst => { - let mut max_len = 0; - for &anchor in &anchors { - max_len = max_len.max(ws.rec.get(&anchor).map(|s| s.len()).unwrap_or(0)); - } - let mut out = Vec::new(); - for i in 1..max_len { - for a in 0..anchors.len() { - out.push((a, i)); - } - } - out - } - }; - + let triples = visit_order(cfg.strategy, &anchors, ws); let mut in_schema: Vec = schema_attrs.clone(); - for (a, pos_idx) in pairs { + for (a, rec_idx, pos_idx) in triples { let anchor = anchors[a]; - let sequence = ws.rec.get(&anchor).cloned().unwrap_or_default(); - if pos_idx >= sequence.len() { - continue; - } - let item = sequence[pos_idx]; + let item = match ws.rec(anchor).and_then(|rs| rs.get(rec_idx)) { + Some(seq) if pos_idx < seq.len() => seq[pos_idx], + _ => continue, + }; match ws.assoc(item).map(|s| s.to_string()) { Some(attr) => { if !in_schema.contains(&attr) { @@ -293,20 +378,37 @@ fn generate_records( schema: &Schema, ) -> CoreResult> { let n = schema.attributes.len(); - let mut records = Vec::with_capacity(ws.rec.len()); - for sequence in ws.rec.values() { - let mut values: Vec> = Vec::with_capacity(n); - for attr in &schema.attributes { - values.push(handle_missing(cfg, attr)?); + let anchors = ws.live_anchors(); + let index: HashMap<&str, usize> = schema + .attributes + .iter() + .enumerate() + .map(|(i, a)| (a.as_str(), i)) + .collect(); + // The missing values of a record depend on the schema only. + let missing: Vec> = match &cfg.missing_value_handler { + None => vec![None; n], + Some(_) => { + let mut m = Vec::with_capacity(n); + for attr in &schema.attributes { + m.push(handle_missing(cfg, attr)?); + } + m } - for &item in sequence { - if let Some(a) = ws.assoc(item) { - if let Some(idx) = schema.index_of(a) { - values[idx] = ws.val.get(&item).cloned(); + }; + let mut records = Vec::with_capacity(anchors.len()); + for anchor in anchors { + for sequence in ws.rec(anchor).into_iter().flatten() { + let mut values = missing.clone(); + for &item in sequence { + if let Some(a) = ws.assoc(item) { + if let Some(&idx) = index.get(a) { + values[idx] = ws.val.get(&item).cloned(); + } } } + records.push(RecordCore { values }); } - records.push(RecordCore { values }); } Ok(records) } diff --git a/src/lib.rs b/src/lib.rs index 8b949ed..70be7e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,6 +55,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -70,6 +71,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/matcher.rs b/src/matcher.rs index ce145da..5dd57e6 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -1,7 +1,9 @@ //! Port of `ru.icc.regtab.atp.match` + `AtpMatcher`: //! backtracking syntactic matching and semantic layer construction. -use crate::semantics::{ActionInst, CellItem, CtxItem, ItemId, OpInst, ProviderInst, SemanticsCore}; +use crate::semantics::{ + ActionInst, CandidateScope, CellItem, CtxItem, ItemId, OpInst, ProviderInst, SemanticsCore, +}; use crate::spec::*; use crate::syntax::SyntaxCore; use crate::util::{split_literal, CoreErr, CoreResult}; @@ -46,28 +48,36 @@ type Outcome = Option; /// Port of the generic `matchPatterns` (backtracking over a pattern list). fn match_patterns

( patterns: &[P], - n_elements: usize, element_index: usize, state: &mut MatchState, quantifier_of: &dyn Fn(&P) -> Quantifier, dispatch: &mut dyn FnMut(&P, usize, &mut MatchState) -> CoreResult, ) -> CoreResult { let mut i = element_index; - let n = n_elements; for j in 0..patterns.len() { let pattern = &patterns[j]; let q = quantifier_of(pattern); - let min = q.min(); + let mut min = q.min(); let max = q.max(); let mut stack: Vec<(usize, Snapshot)> = Vec::new(); - while (stack.len() as i64) < max && i < n { + // No "i < n" guard: a pattern that can match empty (a subrow/subtable + // whose children are all optional) must be tried at the end of the + // sequence too; cell and row dispatchers fail on their own when i >= n. + while (stack.len() as i64) < max { let saved = state.snapshot(); let dispatched = dispatch(pattern, i, state)?; match dispatched { Some(next) => { stack.push((i, saved)); + if next == i { + // Empty iteration: as in regex engines, an empty match is + // never repeated and satisfies any remaining lower bound + // ((a*)+ and (a*){3} match ""). + min = 0; + break; + } i = next; } None => { @@ -85,7 +95,6 @@ fn match_patterns

( loop { let next = match_patterns( &patterns[j + 1..], - n_elements, i, state, quantifier_of, @@ -220,7 +229,6 @@ fn dispatch_subrow( let saved = state.snapshot(); let inner = match_patterns( &pattern.cell_patterns, - cells.len(), cell_index, state, &|p: &Arc| p.quantifier, @@ -234,11 +242,15 @@ fn dispatch_subrow( state.restore(saved); return Ok(None); } - state.subrows.push(( - row_index, - cells[cell_index].1, - cells[next - 1].1, - )); + // A zero-width subrow (next == cell_index) matches the empty sequence: it + // covers no cells and is never materialized in the interpretable table. + if next > cell_index { + state.subrows.push(( + row_index, + cells[cell_index].1, + cells[next - 1].1, + )); + } Ok(Some(next)) } @@ -259,7 +271,6 @@ fn dispatch_row( let saved = state.snapshot(); let inner = match_patterns( &pattern.subrow_patterns, - cells.len(), 0, state, &|p: &Arc| p.quantifier, @@ -284,7 +295,6 @@ fn dispatch_subtable( let saved = state.snapshot(); let inner = match_patterns( &pattern.row_patterns, - syntax.num_rows, row_index, state, &|p: &Arc| p.quantifier, @@ -298,7 +308,11 @@ fn dispatch_subtable( state.restore(saved); return Ok(None); } - state.subtables.push((row_index, next - 1)); + // A zero-height subtable (all row patterns optional) matches the empty + // sequence and introduces no subtable boundary. + if next > row_index { + state.subtables.push((row_index, next - 1)); + } Ok(Some(next)) } @@ -321,7 +335,6 @@ pub fn syntax_match( let mut state = MatchState::default(); let outcome = match_patterns( &atp.subtable_patterns, - syntax.num_rows, 0, &mut state, &|p: &Arc| p.quantifier, @@ -548,16 +561,19 @@ fn to_provider_inst( kind: lit.kind(), }); } + let cond = spec + .filter_condition + .clone() + .ok_or_else(|| SemErr::Other("filterCondition".into()))?; + let scope = CandidateScope::of_cond(&cond); Ok(ProviderInst::Cell { - cond: spec - .filter_condition - .clone() - .ok_or_else(|| SemErr::Other("filterCondition".into()))?, + cond, order: spec.traversal_order, cardinality: spec.cardinality, kind: spec.target_item_kind.unwrap_or(CellKind::Unrestricted), exclude_anchor: true, lenient, + scope, }) } @@ -573,13 +589,14 @@ fn instantiate_action( OperationType::Suffix => OpInst::Suffix(delim), OperationType::Avp => OpInst::Avp, OperationType::Rec => OpInst::Rec, - OperationType::Join => OpInst::Join(action_spec.key_positions.clone()), + OperationType::Concat => OpInst::Concat(action_spec.key.clone()), + OperationType::Join => OpInst::Join(action_spec.key.clone()), }; let mut providers = Vec::with_capacity(action_spec.providers.len()); for ps in &action_spec.providers { providers.push(to_provider_inst(ps, sem, action_spec.inherited)?); } - Ok(ActionInst { anchor, providers, op }) + Ok(ActionInst { anchor, providers, op, inherited: action_spec.inherited }) } pub fn construct_semantics( @@ -808,3 +825,92 @@ mod tests { } } } + +/// Zero-width subrows and subtables (port of the `SyntaxMatcherTest` cases +/// added in jRegTab 0.7.1): a subrow whose cell patterns all matched zero +/// cells matches the empty sequence, is tried at the end of the row too, is +/// never repeated by `+`/`*`/`{n}` and does not appear in the syntax layer. +#[cfg(test)] +mod zero_width_tests { + use super::*; + use crate::rtl::{compile, BindingsCore}; + + fn syntax() -> SyntaxCore { + let mut s = SyntaxCore::new(3, 2).unwrap(); + for (r, c, t) in [(0, 0, "A"), (0, 1, "B"), (1, 0, "x"), (1, 1, "1"), (2, 0, "y"), (2, 1, "2")] { + s.cell_mut(r, c).set_text(t.to_string()); + } + s + } + + fn matches(rtl: &str) -> (bool, SyntaxCore) { + let mut s = syntax(); + let pattern = compile(rtl, &BindingsCore::default()).expect("compile"); + let sem = match_atp(&pattern, &mut s, Vec::new()).expect("match"); + (sem.is_some(), s) + } + + #[test] + fn empty_subrow_mid_row_and_tail() { + for body in [ + "[VAL] { [BLANK]* } [VAL]", + "[VAL] [VAL] { [BLANK]* }", + "{ [BLANK]* } [VAL] [VAL]", + ] { + let (ok, s) = matches(&format!("[ [ATTR]+ ]\n[ {body} ]+")); + assert!(ok, "{body}"); + // the empty subrow covers no cells: the non-empty subrows still + // partition every row without gaps or overlaps + for row in &s.rows { + let mut next = 0; + for sr in &row.subrows { + assert_eq!(sr.col_start, next, "{body}"); + assert!(sr.col_end >= sr.col_start, "{body}"); + next = sr.col_end + 1; + } + assert_eq!(next, s.num_cols, "{body}"); + } + } + // the explicit empty subrow splits the implicit cells into two subrows… + let (_, s) = matches("[ [ATTR]+ ]\n[ [VAL] { [BLANK]* } [VAL] ]+"); + assert_eq!(s.rows[1].subrows.len(), 2); + // …while a trailing one leaves the single implicit subrow alone + let (_, s) = matches("[ [ATTR]+ ]\n[ [VAL] [VAL] { [BLANK]* } ]+"); + assert_eq!(s.rows[1].subrows.len(), 1); + } + + #[test] + fn empty_subrow_repeated_does_not_loop_and_satisfies_the_lower_bound() { + for q in ["+", "*", "{1}", "{0}", "{3}"] { + let (ok, _) = matches(&format!("[ [ATTR]+ ]\n[ [VAL] {{ [BLANK]* }}{q} [VAL] {{ [BLANK]* }}{q} ]+")); + assert!(ok, "quantifier {q}"); + } + } + + #[test] + fn non_empty_subrows_still_partition_the_row() { + let (ok, s) = matches("[ [ATTR]+ ]\n[ { [VAL] } { [VAL] } ]+"); + assert!(ok); + assert_eq!(s.rows[1].subrows.len(), 2); + } + + #[test] + fn empty_subtable_all_rows_optional() { + let (ok, s) = matches("{ [ [BLANK]+ ]* } [ [ATTR]+ ] [ [VAL] [VAL] ]+"); + assert!(ok); + // the empty subtable adds no boundary: only the implicit subtable + // holding the header and data row patterns remains + assert_eq!(s.subtables.len(), 1); + assert_eq!((s.subtables[0].row_start, s.subtables[0].row_end), (0, 2)); + } + + #[test] + fn exactly_zero_consumes_nothing() { + let (ok, _) = matches("[ [ATTR]+ ]\n[ [VAL]{0} [VAL] [VAL] ]+"); + assert!(ok); + let (ok, _) = matches("[ [ATTR]+ ]\n[ [VAL]{1} [VAL]{1} ]+"); + assert!(ok); + let (ok, _) = matches("[ [ATTR]+ ]\n[ [VAL]{0} ]+"); + assert!(!ok, "{{0}} matches nothing, so the row is not covered"); + } +} diff --git a/src/py.rs b/src/py.rs index fb49883..15ce0d0 100644 --- a/src/py.rs +++ b/src/py.rs @@ -837,38 +837,42 @@ impl PyRecordset { /// CSV export (RFC 4180 quoting). Returns the CSV text when `path` is /// None, otherwise writes it to `path` (UTF-8) and returns None. /// Missing values are rendered as `missing` (default: empty field). - #[pyo3(signature = (path=None, *, sep=",", missing=""))] + /// `quote_all` double-quotes every field (the regtab-runner format); + /// `newline` is the line terminator (default CRLF, as `csv.writer`). + #[pyo3(signature = (path=None, *, sep=",", missing="", quote_all=false, newline="\r\n"))] fn to_csv( &self, path: Option, sep: &str, missing: &str, + quote_all: bool, + newline: &str, ) -> PyResult> { - fn field(s: &str, sep: &str) -> String { - if s.contains(sep) || s.contains('"') || s.contains('\n') || s.contains('\r') { - format!("\"{}\"", s.replace('"', "\"\"")) + fn field(out: &mut String, s: &str, sep: &str, quote_all: bool) { + if quote_all || s.contains(sep) || s.contains('"') || s.contains('\n') || s.contains('\r') { + out.push('"'); + out.push_str(&s.replace('"', "\"\"")); + out.push('"'); } else { - s.to_string() + out.push_str(s); } } let mut csv = String::new(); - let header: Vec = self - .core - .schema - .attributes - .iter() - .map(|a| field(a, sep)) - .collect(); - csv.push_str(&header.join(sep)); - csv.push_str("\r\n"); + for (i, a) in self.core.schema.attributes.iter().enumerate() { + if i > 0 { + csv.push_str(sep); + } + field(&mut csv, a, sep, quote_all); + } + csv.push_str(newline); for r in &self.core.records { - let row: Vec = r - .values - .iter() - .map(|v| field(v.as_deref().unwrap_or(missing), sep)) - .collect(); - csv.push_str(&row.join(sep)); - csv.push_str("\r\n"); + for (i, v) in r.values.iter().enumerate() { + if i > 0 { + csv.push_str(sep); + } + field(&mut csv, v.as_deref().unwrap_or(missing), sep, quote_all); + } + csv.push_str(newline); } match path { None => Ok(Some(csv)), @@ -1527,6 +1531,111 @@ impl PyProviderSpec { } } +/// The key K of `CONCAT(K)` / `JOIN(K)`: 0-based positions in the item-based +/// record and/or attribute names, resolved per record at apply time. +#[pyclass(name = "RecordKey", frozen, eq)] +#[derive(Clone, PartialEq)] +pub struct PyRecordKey { + pub core: sp::RecordKey, +} + +/// Builds a `RecordKey` from the accepted Python spellings: `None` (K = ∅), +/// an `int` (one position), a `str` (one attribute name), a `RecordKey`, or an +/// iterable mixing positions and names. +fn extract_record_key(key: Option<&Bound<'_, PyAny>>) -> PyResult { + let Some(key) = key else { + return Ok(sp::RecordKey::empty()); + }; + if key.is_none() { + return Ok(sp::RecordKey::empty()); + } + if let Ok(k) = key.extract::() { + return Ok(k.core); + } + let mut positions = BTreeSet::new(); + let mut names = BTreeSet::new(); + let mut add = |item: &Bound<'_, PyAny>| -> PyResult<()> { + if let Ok(s) = item.extract::() { + names.insert(s); + } else if let Ok(n) = item.extract::() { + positions.insert(n); + } else { + return Err(PyValueError::new_err( + "key references must be int positions or str attribute names", + )); + } + Ok(()) + }; + if key.extract::().is_ok() || key.extract::().is_ok() { + add(key)?; + } else { + for item in key.try_iter()? { + add(&item?)?; + } + } + sp::RecordKey::new(positions, names).map_err(core_err) +} + +#[pymethods] +impl PyRecordKey { + /// `RecordKey(positions=(), names=())` — any iterables of ints and strs. + #[new] + #[pyo3(signature = (positions=None, names=None))] + fn new(positions: Option<&Bound<'_, PyAny>>, names: Option<&Bound<'_, PyAny>>) -> PyResult { + let mut ps: BTreeSet = BTreeSet::new(); + if let Some(positions) = positions { + for p in positions.try_iter()? { + ps.insert(p?.extract::()?); + } + } + let mut ns: BTreeSet = BTreeSet::new(); + if let Some(names) = names { + for n in names.try_iter()? { + ns.insert(n?.extract::()?); + } + } + Ok(PyRecordKey { core: sp::RecordKey::new(ps, ns).map_err(core_err)? }) + } + /// K = ∅. + #[staticmethod] + fn empty() -> Self { + PyRecordKey { core: sp::RecordKey::empty() } + } + /// `RecordKey.of(positions, names)` — the Java factories `positions(…)` + /// / `names(…)` / `of(…)` collapse into this one (in Python the accessors + /// `positions` / `names` are properties). + #[staticmethod] + #[pyo3(signature = (positions=None, names=None))] + fn of(positions: Option<&Bound<'_, PyAny>>, names: Option<&Bound<'_, PyAny>>) -> PyResult { + Self::new(positions, names) + } + #[getter] + fn positions(&self) -> BTreeSet { + self.core.positions.clone() + } + #[getter] + fn names(&self) -> BTreeSet { + self.core.names.clone() + } + fn is_empty(&self) -> bool { + self.core.is_empty() + } + fn __repr__(&self) -> String { + format!( + "RecordKey(positions={:?}, names={:?})", + self.core.positions.iter().collect::>(), + self.core.names.iter().collect::>() + ) + } + fn __hash__(&self) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + self.core.positions.hash(&mut h); + self.core.names.hash(&mut h); + h.finish() + } +} + #[pyclass(name = "ActionSpec", frozen)] #[derive(Clone)] pub struct PyActionSpec { @@ -1564,8 +1673,12 @@ fn providers_from_args( #[pymethods] impl PyActionSpec { + /// `key_positions` (positions only) is kept for callers written against + /// 0.5.x; `key` — a `RecordKey`, an int, a str or an iterable of both — + /// takes precedence when given. #[new] - #[pyo3(signature = (operation_type, delimiter=None, providers=vec![], anchor_pos=None, split_delimiter=None, key_positions=None, inherited=false))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (operation_type, delimiter=None, providers=vec![], anchor_pos=None, split_delimiter=None, key_positions=None, inherited=false, key=None))] fn new( operation_type: sp::OperationType, delimiter: Option, @@ -1574,7 +1687,12 @@ impl PyActionSpec { split_delimiter: Option, key_positions: Option>, inherited: bool, + key: Option<&Bound<'_, PyAny>>, ) -> PyResult { + let key = match key { + Some(k) => extract_record_key(Some(k))?, + None => sp::RecordKey::positions(key_positions.unwrap_or_default()).map_err(core_err)?, + }; Ok(PyActionSpec { core: sp::ActionSpec::new( operation_type, @@ -1582,7 +1700,7 @@ impl PyActionSpec { providers.into_iter().map(|p| p.core).collect(), anchor_pos, split_delimiter, - key_positions.unwrap_or_default(), + key, inherited, ) .map_err(core_err)?, @@ -1604,7 +1722,7 @@ impl PyActionSpec { providers_from_args(providers, cardinality)?, anchor_pos, split_delimiter, - BTreeSet::new(), + sp::RecordKey::empty(), false, ) .map_err(core_err)?, @@ -1637,42 +1755,38 @@ impl PyActionSpec { vec![p], None, None, - BTreeSet::new(), + sp::RecordKey::empty(), false, ) .map_err(core_err)?, }) } + /// `CONCAT(K)`: fold the provided records into the anchor's record. `key` + /// (or the legacy `key_positions`) is an int, a str, a `RecordKey` or an + /// iterable of positions and attribute names. + #[staticmethod] + #[pyo3(signature = (*providers, key=None, key_positions=None, cardinality=1))] + fn concat( + providers: &Bound<'_, PyTuple>, + key: Option<&Bound<'_, PyAny>>, + key_positions: Option<&Bound<'_, PyAny>>, + cardinality: i64, + ) -> PyResult { + Self::record_op(sp::OperationType::Concat, providers, key.or(key_positions), cardinality) + } + + /// `JOIN(K)`: the record product — a cross product for K = ∅, an equi-join + /// on the key K otherwise. #[staticmethod] - #[pyo3(signature = (*providers, key_positions=None, cardinality=1))] + #[pyo3(signature = (*providers, key=None, key_positions=None, cardinality=1))] fn join( providers: &Bound<'_, PyTuple>, + key: Option<&Bound<'_, PyAny>>, key_positions: Option<&Bound<'_, PyAny>>, cardinality: i64, ) -> PyResult { - let keys: BTreeSet = match key_positions { - None => BTreeSet::new(), - Some(kp) => { - if let Ok(one) = kp.extract::() { - BTreeSet::from([one]) - } else { - kp.extract::>()? - } - } - }; - Ok(PyActionSpec { - core: sp::ActionSpec::new( - sp::OperationType::Join, - None, - providers_from_args(providers, cardinality)?, - None, - None, - keys, - false, - ) - .map_err(core_err)?, - }) + Self::record_op(sp::OperationType::Join, providers, key.or(key_positions), cardinality) } #[staticmethod] @@ -1711,12 +1825,42 @@ impl PyActionSpec { fn inherited(&self) -> bool { self.core.inherited } + /// The key K of a `CONCAT`/`JOIN` action (empty for other operations). + #[getter] + fn key(&self) -> PyRecordKey { + PyRecordKey { core: self.core.key.clone() } + } + /// The positional part of the key (kept for callers written against 0.5.x). + #[getter] + fn key_positions(&self) -> BTreeSet { + self.core.key.positions.clone() + } fn as_inherited(&self) -> Self { PyActionSpec { core: self.core.as_inherited() } } } impl PyActionSpec { + fn record_op( + op: sp::OperationType, + providers: &Bound<'_, PyTuple>, + key: Option<&Bound<'_, PyAny>>, + cardinality: i64, + ) -> PyResult { + Ok(PyActionSpec { + core: sp::ActionSpec::new( + op, + None, + providers_from_args(providers, cardinality)?, + None, + None, + extract_record_key(key)?, + false, + ) + .map_err(core_err)?, + }) + } + fn str_op( op: sp::OperationType, delimiter: String, @@ -1745,8 +1889,16 @@ impl PyActionSpec { } } Ok(PyActionSpec { - core: sp::ActionSpec::new(op, Some(delimiter), out, None, None, BTreeSet::new(), false) - .map_err(core_err)?, + core: sp::ActionSpec::new( + op, + Some(delimiter), + out, + None, + None, + sp::RecordKey::empty(), + false, + ) + .map_err(core_err)?, }) } } @@ -2846,6 +2998,55 @@ impl PyAtpMatcher { } } +/// A `CONCAT`/`JOIN` action that had no effect during working state +/// completion: a violated precondition or a missing record (port of +/// `ru.icc.regtab.itm.semantics.Diagnostic`). +#[pyclass(name = "Diagnostic")] +pub struct PyDiagnostic { + pub anchor: PyCellDerivedItem, + pub operation: String, + pub message: String, +} + +#[pymethods] +impl PyDiagnostic { + /// The anchor item of the action that was skipped. + #[getter] + fn anchor(&self, py: Python<'_>) -> PyCellDerivedItem { + PyCellDerivedItem { + s: self.anchor.s.clone(), + tags: self.anchor.tags.clone(), + index: self.anchor.index, + ty: self.anchor.ty, + row: self.anchor.row, + col: self.anchor.col, + span: self.anchor.span, + table: self.anchor.table.as_ref().map(|t| t.clone_ref(py)), + } + } + /// The operation name (`CONCAT`, `JOIN`, …). + #[getter] + fn operation(&self) -> String { + self.operation.clone() + } + /// What precondition was violated and why. + #[getter] + fn message(&self) -> String { + self.message.clone() + } + fn __str__(&self) -> String { + format!( + "{} skipped at {}: {}", + self.operation, + self.anchor.__repr__(), + self.message + ) + } + fn __repr__(&self) -> String { + format!("Diagnostic({})", self.__str__()) + } +} + #[pyclass(name = "TableInterpreter")] pub struct PyTableInterpreter { strategy: SchemaStrategy, @@ -2853,6 +3054,10 @@ pub struct PyTableInterpreter { missing: Option, transformations: Vec, template: String, + strict_preconditions: bool, + /// Diagnostics of the most recent `interpret` call, with the table they + /// refer to (needed to materialize the anchor items). + last: Option<(Py, Arc, Vec)>, } #[pymethods] @@ -2865,9 +3070,57 @@ impl PyTableInterpreter { missing: None, transformations: Vec::new(), template: "$a_%i".to_string(), + strict_preconditions: false, + last: None, } } + /// Strict preconditions: a `CONCAT` / `JOIN` action whose precondition is + /// violated (e.g. a named attribute shared by two records being + /// concatenated, or a missing record) raises a `RuntimeError` instead of + /// having no effect. Default: `False` — by the formal model the operation + /// has no effect, and the violation is reported through `diagnostics()`. + fn with_strict_preconditions<'py>( + mut slf: PyRefMut<'py, Self>, + strict: bool, + ) -> PyRefMut<'py, Self> { + slf.strict_preconditions = strict; + slf + } + + /// Diagnostics of the most recent `interpret(...)` call: every `CONCAT` / + /// `JOIN` action that had no effect — a violated precondition (key + /// mismatch, a named attribute shared by two concatenated records, an + /// empty record product), or, for actions written on the anchor's own + /// content spec (not inherited), an anchor without a record or provided + /// items none of which has a record (`REC` missing). Empty if nothing was + /// skipped (or before the first call). + fn diagnostics(&self, py: Python<'_>) -> Vec { + let Some((table, sem, diags)) = &self.last else { + return Vec::new(); + }; + diags + .iter() + .map(|d| { + let it = &sem.cell_items[d.anchor]; + PyDiagnostic { + anchor: PyCellDerivedItem { + s: it.s.clone(), + tags: it.tags.clone(), + index: it.index, + ty: it.ty, + row: it.row, + col: it.col, + span: it.span, + table: Some(table.clone_ref(py)), + }, + operation: d.operation.clone(), + message: d.message.clone(), + } + }) + .collect() + } + fn with_strategy<'py>( mut slf: PyRefMut<'py, Self>, strategy: SchemaStrategy, @@ -2913,28 +3166,30 @@ impl PyTableInterpreter { Ok(slf) } - fn interpret(&self, py: Python<'_>, table: &PyInterpretableTable) -> PyResult { + fn interpret(&mut self, py: Python<'_>, table: &PyInterpretableTable) -> PyResult { let cfg = InterpreterCfg { strategy: self.strategy, action_strategy: self.action_strategy, missing_value_handler: self.missing.clone(), transformations: self.transformations.clone(), anonymous_attribute_template: self.template.clone(), + strict_preconditions: self.strict_preconditions, }; - if self.missing.is_none() && !table.sem.has_py_callbacks() { + self.last = None; + let out = if self.missing.is_none() && !table.sem.has_py_callbacks() { // Fast path: pure-native interpretation with the GIL released. let core = table.table.bind(py).borrow().core.clone(); let sem = table.sem.clone(); - let rs = py - .allow_threads(move || crate::interp::interpret(&cfg, &core, &sem, None)) - .map_err(core_err)?; - return Ok(PyRecordset { core: rs }); - } - let table_any: Py = table.table.clone_ref(py).into_any(); - let s = table.table.bind(py).borrow(); - let rs = crate::interp::interpret(&cfg, &s.core, &table.sem, Some(&table_any)) - .map_err(core_err)?; - Ok(PyRecordset { core: rs }) + py.allow_threads(move || crate::interp::interpret(&cfg, &core, &sem, None)) + .map_err(core_err)? + } else { + let table_any: Py = table.table.clone_ref(py).into_any(); + let s = table.table.bind(py).borrow(); + crate::interp::interpret(&cfg, &s.core, &table.sem, Some(&table_any)) + .map_err(core_err)? + }; + self.last = Some((table.table.clone_ref(py), table.sem.clone(), out.diagnostics)); + Ok(PyRecordset { core: out.recordset }) } } diff --git a/src/rtl/ast.rs b/src/rtl/ast.rs index 8e5848f..c92655c 100644 --- a/src/rtl/ast.rs +++ b/src/rtl/ast.rs @@ -120,11 +120,19 @@ pub enum PProvSpec { CtxAvp(String, String), } +/// `keyRef : INT | STRING` — a key position (0-based) or a key attribute name. +#[derive(Clone, Debug)] +pub enum PKeyRef { + Pos(i64), + Name { name: String, line: i64, col: i64 }, +} + #[derive(Clone, Debug)] pub enum POp { Avp, Rec { anchor: Option, split: Option }, - Join(Vec), + Concat(Vec), + Join(Vec), Fill(Option), Prefix(Option), Suffix(Option), diff --git a/src/rtl/build.rs b/src/rtl/build.rs index a33d41e..99b5ecf 100644 --- a/src/rtl/build.rs +++ b/src/rtl/build.rs @@ -396,51 +396,46 @@ impl<'a> ATPBuilder<'a> { for p in &ast.providers { providers.push(self.build_prov_spec(p, &ast.op)?); } - let (op, delim, anchor_pos, split_delim, keys): ( + let (op, delim, anchor_pos, split_delim, key): ( OperationType, Option, Option, Option, - BTreeSet, + RecordKey, ) = match &ast.op { - POp::Avp => (OperationType::Avp, None, None, None, BTreeSet::new()), + POp::Avp => (OperationType::Avp, None, None, None, RecordKey::empty()), POp::Rec { anchor, split } => ( OperationType::Rec, None, *anchor, split.clone(), - BTreeSet::new(), - ), - POp::Join(keys) => ( - OperationType::Join, - None, - None, - None, - keys.iter().copied().collect(), + RecordKey::empty(), ), + POp::Concat(refs) => (OperationType::Concat, None, None, None, build_key(refs)?), + POp::Join(refs) => (OperationType::Join, None, None, None, build_key(refs)?), POp::Fill(d) => ( OperationType::Fill, Some(d.clone().unwrap_or_default()), None, None, - BTreeSet::new(), + RecordKey::empty(), ), POp::Prefix(d) => ( OperationType::Prefix, Some(d.clone().unwrap_or_default()), None, None, - BTreeSet::new(), + RecordKey::empty(), ), POp::Suffix(d) => ( OperationType::Suffix, Some(d.clone().unwrap_or_default()), None, None, - BTreeSet::new(), + RecordKey::empty(), ), }; - ActionSpec::new(op, delim, providers, anchor_pos, split_delim, keys, false) + ActionSpec::new(op, delim, providers, anchor_pos, split_delim, key, false) .map_err(RtlErr::from) } @@ -451,7 +446,7 @@ impl<'a> ATPBuilder<'a> { Ok(ProviderSpec::ctx_avp(name.clone(), value.clone())) } PProvSpec::Ctx(literal) => { - let ty = if matches!(op, POp::Rec { .. } | POp::Join(_)) { + let ty = if matches!(op, POp::Rec { .. } | POp::Concat(_) | POp::Join(_)) { ItemType::Value } else { ItemType::Attribute @@ -471,7 +466,7 @@ impl<'a> ATPBuilder<'a> { }; let condition = self.build_provider_condition(&ast.body)?; let kind = match op { - POp::Rec { .. } | POp::Join(_) => CellKind::Val, + POp::Rec { .. } | POp::Concat(_) | POp::Join(_) => CellKind::Val, POp::Avp => CellKind::Attr, _ => CellKind::Unrestricted, }; @@ -758,6 +753,32 @@ impl<'a> ATPBuilder<'a> { } } +/// `RecordKey` of a `CONCAT(...)` / `JOIN(...)`: `INT` → position, `STRING` → +/// attribute name; an empty (blank) name is a compile error with its position +/// (port of `ATPBuilder.buildKey`). +fn build_key(refs: &[PKeyRef]) -> Result { + let mut positions = BTreeSet::new(); + let mut names = BTreeSet::new(); + for r in refs { + match r { + PKeyRef::Pos(n) => { + positions.insert(*n); + } + PKeyRef::Name { name, line, col } => { + if name.trim().is_empty() { + return Err(RtlErr::at( + format!("Key attribute name must not be empty: '{name}'"), + *line, + *col, + )); + } + names.insert(name.clone()); + } + } + } + RecordKey::new(positions, names).map_err(RtlErr::from) +} + fn bound_value(b: &PBound) -> i64 { match b { PBound::Offset(d) => *d, diff --git a/src/rtl/lexer.rs b/src/rtl/lexer.rs index 3fb53b9..d2474fe 100644 --- a/src/rtl/lexer.rs +++ b/src/rtl/lexer.rs @@ -45,6 +45,7 @@ pub enum Tok { KwSuffix, KwAvp, KwRec, + KwConcat, KwJoin, KwLt, KwRt, @@ -92,6 +93,7 @@ fn keyword(word: &str) -> Option { "SUFFIX" => Tok::KwSuffix, "AVP" => Tok::KwAvp, "REC" => Tok::KwRec, + "CONCAT" => Tok::KwConcat, "JOIN" => Tok::KwJoin, "LT" => Tok::KwLt, "RT" => Tok::KwRt, @@ -326,15 +328,27 @@ pub fn lex(src: &str) -> Result, RtlErr> { Tok::FragmentId(name) } '0'..='9' => { + // INT : [0-9]+ — the value is kept as i64; a literal beyond the + // range of a Java int (the reference implementation's INT) is + // reported where it is used (quantifier, cardinality, position). let mut n: i64 = 0; + let mut digits = String::new(); while let Some(c) = lx.peek() { if let Some(d) = c.to_digit(10) { - n = n * 10 + d as i64; + n = n.saturating_mul(10).saturating_add(d as i64); + digits.push(c); lx.bump(); } else { break; } } + if n > i32::MAX as i64 { + return Err(RtlErr::at( + format!("Invalid integer literal {digits}: out of int range"), + line, + col, + )); + } Tok::Int(n) } '"' | '\'' => { diff --git a/src/rtl/parser.rs b/src/rtl/parser.rs index cf2580a..33282e5 100644 --- a/src/rtl/parser.rs +++ b/src/rtl/parser.rs @@ -745,17 +745,13 @@ impl Parser { } Ok(POp::Rec { anchor, split }) } + Tok::KwConcat => { + self.bump(); + Ok(POp::Concat(self.key_refs()?)) + } Tok::KwJoin => { self.bump(); - let mut keys = Vec::new(); - if self.eat(Tok::LParen) { - keys.push(self.expect_int()?); - while self.eat(Tok::Comma) { - keys.push(self.expect_int()?); - } - self.expect(Tok::RParen, "')'")?; - } - Ok(POp::Join(keys)) + Ok(POp::Join(self.key_refs()?)) } Tok::KwFill => { self.bump(); @@ -773,6 +769,35 @@ impl Parser { } } + /// `(LPAREN keyRef (COMMA keyRef)* RPAREN)?` of `concatOp` / `joinOp`. + fn key_refs(&mut self) -> Result, RtlErr> { + let mut keys = Vec::new(); + if self.eat(Tok::LParen) { + keys.push(self.key_ref()?); + while self.eat(Tok::Comma) { + keys.push(self.key_ref()?); + } + self.expect(Tok::RParen, "')'")?; + } + Ok(keys) + } + + /// `keyRef : INT | STRING`. + fn key_ref(&mut self) -> Result { + let t = self.cur().clone(); + match t.tok { + Tok::Int(n) => { + self.bump(); + Ok(PKeyRef::Pos(n)) + } + Tok::Str(s) => { + self.bump(); + Ok(PKeyRef::Name { name: s, line: t.line, col: t.col }) + } + other => self.err(format!("expected INT or STRING key reference, found {other:?}")), + } + } + fn op_string_arg(&mut self) -> Result, RtlErr> { if self.eat(Tok::LParen) { let s = self.expect_string()?; diff --git a/src/rtl/serialize.rs b/src/rtl/serialize.rs index a7396b2..1de887d 100644 --- a/src/rtl/serialize.rs +++ b/src/rtl/serialize.rs @@ -295,20 +295,25 @@ fn serialize_op(a: &ActionSpec) -> CoreResult { "REC".to_string() } } - OperationType::Join => { - if a.key_positions.is_empty() { - "JOIN".to_string() - } else { - let args: Vec = a.key_positions.iter().map(|k| k.to_string()).collect(); - format!("JOIN({})", args.join(", ")) - } - } + OperationType::Concat => op_with_key("CONCAT", &a.key), + OperationType::Join => op_with_key("JOIN", &a.key), OperationType::Fill => op_with_delim("FILL", a), OperationType::Prefix => op_with_delim("PREFIX", a), OperationType::Suffix => op_with_delim("SUFFIX", a), }) } +/// Canonical form of the key K: positions in ascending order, then names in +/// lexicographic order, single-quoted (`CONCAT(0, 1, 'A', 'B')`). +fn op_with_key(name: &str, key: &RecordKey) -> String { + if key.is_empty() { + return name.to_string(); + } + let mut args: Vec = key.positions.iter().map(|k| k.to_string()).collect(); + args.extend(key.names.iter().map(|a| format!("'{}'", esc_sq(a)))); + format!("{name}({})", args.join(", ")) +} + fn op_with_delim(name: &str, a: &ActionSpec) -> String { match &a.delimiter { Some(d) if !d.is_empty() => format!("{name}(\"{}\")", esc_dq(d)), diff --git a/src/semantics.rs b/src/semantics.rs index 4d02ef5..23fbf7a 100644 --- a/src/semantics.rs +++ b/src/semantics.rs @@ -1,10 +1,27 @@ //! Port of `ru.icc.regtab.itm.semantics`: items, providers, working state. //! Java identity-keyed maps become `ItemId`-keyed insertion-ordered maps. +//! +//! Cell-derived providers do not scan the whole item set: a spatial index over +//! the items ([`ItemIndex`], port of `CellDerivedItemIndex`) and a candidate +//! scope derived statically from the filter specification ([`CandidateScope`], +//! port of `CandidateScope` / `CandidateScopes`) restrict the scan to the +//! anchor's row, column, cell, row range or subtable. The filter κ is still +//! applied to every candidate of the scope and the scan stops after k matches, +//! so the result of Υ^{J,k}_{τ,κ} is unchanged (pinned by the randomized +//! equivalence test at the end of this file). -use crate::spec::{CellKind, CtxKind, EvalEnv, FilterCond, ItemType, TraversalOrder, UNBOUNDED}; +use crate::spec::{ + CellKind, CtxKind, EvalEnv, FilterCond, FilterTerm, ItemType, RecordKey, TraversalOrder, + UNBOUNDED, +}; use crate::util::CoreResult; use indexmap::IndexMap; -use std::collections::BTreeSet; +use rustc_hash::{FxBuildHasher, FxHashSet}; +use std::collections::{HashMap, HashSet}; + +/// Insertion-ordered map with a fast non-cryptographic hasher: the working +/// state performs a handful of lookups per cell of the table. +pub type FastIndexMap = IndexMap; #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum ItemId { @@ -27,6 +44,16 @@ pub struct CellItem { pub span: (usize, usize), } +impl CellItem { + /// `CellDerivedItem.toString()` — used in diagnostics. + pub fn describe(&self) -> String { + format!( + "CellDerivedItem[str=\"{}\", index={}, cell=({}, {}), type={:?}]", + self.s, self.index, self.row, self.col, self.ty + ) + } +} + /// Context-derived item. #[derive(Clone, Debug)] pub struct CtxItem { @@ -41,6 +68,9 @@ pub struct ActionInst { pub anchor: ItemId, pub providers: Vec, pub op: OpInst, + /// `true` if the action was inherited from the `actSpecs` of a + /// table/subtable/row/subrow scope; affects diagnostics only. + pub inherited: bool, } #[derive(Clone, Debug)] @@ -50,7 +80,8 @@ pub enum OpInst { Suffix(String), Avp, Rec, - Join(BTreeSet), + Concat(RecordKey), + Join(RecordKey), } #[derive(Clone, Debug)] @@ -63,6 +94,8 @@ pub enum ProviderInst { kind: CellKind, exclude_anchor: bool, lenient: bool, + /// Static over-approximation of the support of `cond`. + scope: CandidateScope, }, /// Context-derived provider over indices into `SemanticsCore::ctx_items`. Ctx { items: Vec, kind: CtxKind }, @@ -102,6 +135,452 @@ impl SemanticsCore { } } +// ---------------------------------------------------------------- CandidateScope + +/// Inclusive interval of row or column indices; open ends use [`Interval::OPEN_LO`] / +/// [`Interval::OPEN_HI`]. When `relative` is set, both bounds are offsets from the +/// anchor's index. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Interval { + pub lo: i64, + pub hi: i64, + pub relative: bool, +} + +impl Interval { + pub const OPEN_LO: i64 = i64::MIN; + pub const OPEN_HI: i64 = i64::MAX; + + pub const fn absolute(lo: i64, hi: i64) -> Interval { + Interval { lo, hi, relative: false } + } + pub const fn absolute_point(index: i64) -> Interval { + Interval::absolute(index, index) + } + pub const fn relative(lo: i64, hi: i64) -> Interval { + Interval { lo, hi, relative: true } + } + pub const fn relative_point(delta: i64) -> Interval { + Interval::relative(delta, delta) + } + + pub fn is_point(&self) -> bool { + self.lo == self.hi + } + + /// Resolved lower bound for the anchor index (unclamped). + fn resolve_lo(&self, anchor_index: i64) -> i64 { + if self.lo == Self::OPEN_LO { + i64::MIN + } else if self.relative { + anchor_index.saturating_add(self.lo) + } else { + self.lo + } + } + + /// Resolved upper bound for the anchor index (unclamped). + fn resolve_hi(&self, anchor_index: i64) -> i64 { + if self.hi == Self::OPEN_HI { + i64::MAX + } else if self.relative { + anchor_index.saturating_add(self.hi) + } else { + self.hi + } + } + + /// Conjunction with another interval: exact intersection when both are of + /// the same kind, otherwise the more specific one (a point wins; ties keep + /// `self`). The result always contains the intersection of the two. + pub fn and(&self, other: Option) -> Interval { + let Some(other) = other else { return *self }; + if self.relative == other.relative { + return Interval { + lo: self.lo.max(other.lo), + hi: self.hi.min(other.hi), + relative: self.relative, + }; + } + if self.is_point() { + return *self; + } + if other.is_point() { + return other; + } + *self + } +} + +/// Candidate scope — a static over-approximation of the support of an item +/// filter κ: a row interval × column interval (each optional, absolute or +/// relative to the anchor) plus a `same_subtable` flag. It never changes the +/// result: κ is still applied to every candidate of the scope, so any scope +/// that contains the support of κ is correct, and [`CandidateScope::ALL`] is +/// always correct. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct CandidateScope { + pub rows: Option, + pub cols: Option, + pub same_subtable: bool, +} + +impl CandidateScope { + /// No restriction: the whole target set. + pub const ALL: CandidateScope = CandidateScope { rows: None, cols: None, same_subtable: false }; + + pub const fn rows(rows: Interval) -> CandidateScope { + CandidateScope { rows: Some(rows), cols: None, same_subtable: false } + } + pub const fn cols(cols: Interval) -> CandidateScope { + CandidateScope { rows: None, cols: Some(cols), same_subtable: false } + } + pub const fn subtable() -> CandidateScope { + CandidateScope { rows: None, cols: None, same_subtable: true } + } + + /// Conjunction of two scopes: each component only narrows. + pub fn and(&self, other: &CandidateScope) -> CandidateScope { + CandidateScope { + rows: match self.rows { + None => other.rows, + Some(r) => Some(r.and(other.rows)), + }, + cols: match self.cols { + None => other.cols, + Some(c) => Some(c.and(other.cols)), + }, + same_subtable: self.same_subtable || other.same_subtable, + } + } + + pub fn is_all(&self) -> bool { + self.rows.is_none() && self.cols.is_none() && !self.same_subtable + } + + /// Port of `CandidateScopes.of(ItemFilterConditionSpec)`: a conjunction is + /// the intersection of its terms' scopes, a disjunction or an opaque + /// condition yields [`CandidateScope::ALL`]. + pub fn of_cond(cond: &FilterCond) -> CandidateScope { + match cond { + FilterCond::Bare(t) => CandidateScope::of_term(t), + FilterCond::And(terms) => { + let mut scope = CandidateScope::ALL; + for t in terms { + scope = scope.and(&CandidateScope::of_term(t)); + } + scope + } + FilterCond::Or(_) | FilterCond::Custom { .. } => CandidateScope::ALL, + } + } + + /// Port of `CandidateScopes.of(FilterTerm)`: for every term the scope + /// contains every candidate for which the term can hold. + pub fn of_term(term: &FilterTerm) -> CandidateScope { + fn open_hi(hi: i64) -> i64 { + if hi == UNBOUNDED { + Interval::OPEN_HI + } else { + hi + } + } + match term { + // c.sameSubrow(a) && col < col(a): same row, columns to the left + FilterTerm::LeftOf => CandidateScope { + rows: Some(Interval::relative_point(0)), + cols: Some(Interval::relative(Interval::OPEN_LO, -1)), + same_subtable: false, + }, + FilterTerm::RightOf => CandidateScope { + rows: Some(Interval::relative_point(0)), + cols: Some(Interval::relative(1, Interval::OPEN_HI)), + same_subtable: false, + }, + // c.sameSubcol(a) && row < row(a): same subtable, same column, rows above + FilterTerm::Above => CandidateScope { + rows: Some(Interval::relative(Interval::OPEN_LO, -1)), + cols: Some(Interval::relative_point(0)), + same_subtable: true, + }, + FilterTerm::Below => CandidateScope { + rows: Some(Interval::relative(1, Interval::OPEN_HI)), + cols: Some(Interval::relative_point(0)), + same_subtable: true, + }, + FilterTerm::SameSubrow => CandidateScope::rows(Interval::relative_point(0)), + FilterTerm::SameSubcol => CandidateScope { + rows: None, + cols: Some(Interval::relative_point(0)), + same_subtable: true, + }, + FilterTerm::SameSubtable => CandidateScope::subtable(), + FilterTerm::SameRow => CandidateScope::rows(Interval::relative_point(0)), + FilterTerm::SameCol => CandidateScope::cols(Interval::relative_point(0)), + FilterTerm::SameCell => CandidateScope { + rows: Some(Interval::relative_point(0)), + cols: Some(Interval::relative_point(0)), + same_subtable: false, + }, + FilterTerm::ColExact(n) => CandidateScope::cols(Interval::absolute_point(*n)), + FilterTerm::ColOffset(d) => CandidateScope::cols(Interval::relative_point(*d)), + FilterTerm::ColRange(from, to) => { + CandidateScope::cols(Interval::relative(*from, open_hi(*to))) + } + FilterTerm::ColAbsoluteRange(lo, hi) => { + CandidateScope::cols(Interval::absolute(*lo, open_hi(*hi))) + } + FilterTerm::RowExact(n) => CandidateScope::rows(Interval::absolute_point(*n)), + FilterTerm::RowOffset(d) => CandidateScope::rows(Interval::relative_point(*d)), + FilterTerm::RowAbsoluteRange(lo, hi) => { + CandidateScope::rows(Interval::absolute(*lo, open_hi(*hi))) + } + // position, content, sameStr, external / custom predicates and NCL + // do not restrict the position + _ => CandidateScope::ALL, + } + } + + fn clamp_lo(v: i64, n: usize) -> usize { + if v < 0 { + 0 + } else if v > n as i64 { + n + } else { + v as usize + } + } + + /// Clamped to `[-1, n - 1]`, returned as `i64`. + fn clamp_hi(v: i64, n: usize) -> i64 { + if v < -1 { + -1 + } else if v > n as i64 - 1 { + n as i64 - 1 + } else { + v + } + } + + /// Resolved row bounds `(lo, hi)` for the anchor — `lo ∈ [0, num_rows]`, + /// `hi ∈ [-1, num_rows - 1]`. + fn row_bounds(&self, anchor_row: usize, num_rows: usize) -> (usize, i64) { + match self.rows { + None => (0, num_rows as i64 - 1), + Some(r) => ( + Self::clamp_lo(r.resolve_lo(anchor_row as i64), num_rows), + Self::clamp_hi(r.resolve_hi(anchor_row as i64), num_rows), + ), + } + } + + fn col_bounds(&self, anchor_col: usize, num_cols: usize) -> (usize, i64) { + match self.cols { + None => (0, num_cols as i64 - 1), + Some(c) => ( + Self::clamp_lo(c.resolve_lo(anchor_col as i64), num_cols), + Self::clamp_hi(c.resolve_hi(anchor_col as i64), num_cols), + ), + } + } +} + +// ---------------------------------------------------------------- ItemIndex + +/// Spatial index over the cell-derived items of one semantics layer (port of +/// `CellDerivedItemIndex`): two orderings of the items — row-major and +/// column-major, exactly as produced by the linearization Ω_τ — together with +/// row / column offsets, so that a [`CandidateScope`] resolves to a contiguous +/// slice of one of them: a row, a column, a cell (binary search inside a row), +/// a row range (a subtable is a row range), or the whole set. Every slice is +/// already in traversal order, so a provider needs neither a copy of the item +/// set nor a sort. +#[derive(Debug)] +pub struct ItemIndex { + num_rows: usize, + num_cols: usize, + /// Item indices sorted by (row, col, index), stable. + row_major: Vec, + row_start: Vec, + /// Item indices sorted by (col, row, index), stable. + col_major: Vec, + col_start: Vec, + /// Column-major ordering of the items of each subtable (by subtable index), + /// built only when some provider needs it. + subtable_col_major: Vec>, +} + +/// A contiguous slice of an ordered array, to be traversed forward or, when +/// `backward` is set, from the end — cell by cell, keeping the ascending index +/// order inside each cell (this is exactly the order of Ω_τ for the reverse +/// traversal orders). +pub struct Range<'a> { + pub items: &'a [usize], + pub backward: bool, +} + +impl ItemIndex { + /// Builds the index; `syntax` supplies the subtable partition used by + /// subtable-scoped column-major providers. + pub fn build(sem: &SemanticsCore, syntax: &crate::syntax::SyntaxCore) -> ItemIndex { + let items = &sem.cell_items; + let n = items.len(); + let num_rows = items.iter().map(|it| it.row + 1).max().unwrap_or(0); + let num_cols = items.iter().map(|it| it.col + 1).max().unwrap_or(0); + + let mut row_major: Vec = (0..n).collect(); + row_major.sort_by_key(|&i| (items[i].row, items[i].col, items[i].index)); + let mut col_major: Vec = (0..n).collect(); + col_major.sort_by_key(|&i| (items[i].col, items[i].row, items[i].index)); + + let mut row_start = vec![0usize; num_rows + 1]; + let mut idx = 0; + for (r, start) in row_start.iter_mut().enumerate().take(num_rows) { + *start = idx; + while idx < n && items[row_major[idx]].row == r { + idx += 1; + } + } + row_start[num_rows] = n; + + let mut col_start = vec![0usize; num_cols + 1]; + idx = 0; + for (c, start) in col_start.iter_mut().enumerate().take(num_cols) { + *start = idx; + while idx < n && items[col_major[idx]].col == c { + idx += 1; + } + } + col_start[num_cols] = n; + + let needs_subtable_col_major = sem.actions.iter().any(|a| { + a.providers.iter().any(|p| match p { + ProviderInst::Cell { order, scope, .. } => { + scope.same_subtable + && matches!( + order, + TraversalOrder::ColumnMajor | TraversalOrder::ReverseColumnMajor + ) + } + ProviderInst::Ctx { .. } => false, + }) + }); + let mut subtable_col_major = Vec::new(); + if needs_subtable_col_major { + for st in &syntax.subtables { + let lo = st.row_start.min(num_rows); + let hi = (st.row_end + 1).min(num_rows); + let mut a: Vec = if lo < hi { + row_major[row_start[lo]..row_start[hi]].to_vec() + } else { + Vec::new() + }; + a.sort_by_key(|&i| (items[i].col, items[i].row, items[i].index)); + subtable_col_major.push(a); + } + } + + ItemIndex { num_rows, num_cols, row_major, row_start, col_major, col_start, subtable_col_major } + } + + /// Resolves the scope against the anchor and returns the candidate slice in + /// traversal order τ. The slice is a superset of the items lying in + /// `scope(anchor)`. + pub fn lookup( + &self, + scope: &CandidateScope, + anchor: &CellItem, + order: TraversalOrder, + env: &EvalEnv, + sem: &SemanticsCore, + ) -> Range<'_> { + let backward = matches!( + order, + TraversalOrder::ReverseRowMajor | TraversalOrder::ReverseColumnMajor + ); + let column_major = matches!( + order, + TraversalOrder::ColumnMajor | TraversalOrder::ReverseColumnMajor + ); + let empty = Range { items: &self.row_major[0..0], backward }; + if self.row_major.is_empty() { + return empty; + } + if scope.is_all() { + let a = if column_major { &self.col_major } else { &self.row_major }; + return Range { items: a, backward }; + } + + let (mut r_lo, mut r_hi) = scope.row_bounds(anchor.row, self.num_rows); + let (c_lo, c_hi) = scope.col_bounds(anchor.col, self.num_cols); + let subtable = env.syntax.subtable_of_row(anchor.row); + if scope.same_subtable { + let Some(st_idx) = subtable else { return empty }; + let st = &env.syntax.subtables[st_idx]; + r_lo = r_lo.max(st.row_start); + r_hi = r_hi.min(st.row_end as i64); + } + if r_lo as i64 > r_hi || c_lo as i64 > c_hi { + return empty; + } + let (r_hi, c_hi) = (r_hi as usize, c_hi as usize); + let items = &sem.cell_items; + + if r_lo == r_hi { + // one row: row-major slice sorted by (col, index); narrow by columns + let (mut from, mut to) = (self.row_start[r_lo], self.row_start[r_lo + 1]); + if scope.cols.is_some() { + from = lower_bound(&self.row_major, from, to, |i| items[i].col < c_lo); + to = lower_bound(&self.row_major, from, to, |i| items[i].col <= c_hi); + } + return Range { items: &self.row_major[from..to], backward }; + } + if c_lo == c_hi { + // one column: column-major slice sorted by (row, index); narrow by rows + let (mut from, mut to) = (self.col_start[c_lo], self.col_start[c_lo + 1]); + if scope.rows.is_some() || scope.same_subtable { + from = lower_bound(&self.col_major, from, to, |i| items[i].row < r_lo); + to = lower_bound(&self.col_major, from, to, |i| items[i].row <= r_hi); + } + return Range { items: &self.col_major[from..to], backward }; + } + if !column_major { + // row range (a subtable is a row range) in row-major order + return Range { + items: &self.row_major[self.row_start[r_lo]..self.row_start[r_hi + 1]], + backward, + }; + } + if scope.same_subtable { + let a = &self.subtable_col_major[subtable.expect("checked above")]; + let from = lower_bound(a, 0, a.len(), |i| items[i].col < c_lo); + let to = lower_bound(a, from, a.len(), |i| items[i].col <= c_hi); + return Range { items: &a[from..to], backward }; + } + Range { + items: &self.col_major[self.col_start[c_lo]..self.col_start[c_hi + 1]], + backward, + } + } +} + +/// First position in `a[from..to)` for which `less` is false (the array is +/// partitioned by `less`). +fn lower_bound(a: &[usize], from: usize, to: usize, less: impl Fn(usize) -> bool) -> usize { + let (mut lo, mut hi) = (from, to); + while lo < hi { + let mid = (lo + hi) / 2; + if less(a[mid]) { + lo = mid + 1; + } else { + hi = mid; + } + } + lo +} + +// ---------------------------------------------------------------- providers + impl ProviderInst { /// Port of `ItemProvider.provide(anchor)`. pub fn provide( @@ -109,12 +588,13 @@ impl ProviderInst { anchor: ItemId, sem: &SemanticsCore, env: &EvalEnv, + index: &ItemIndex, ) -> CoreResult> { match self { ProviderInst::Ctx { items, kind } => { self.provide_ctx(anchor, sem, items, *kind) } - ProviderInst::Cell { cond, order, cardinality, kind, exclude_anchor, lenient } => { + ProviderInst::Cell { cond, order, cardinality, kind, exclude_anchor, lenient, scope } => { let ItemId::Cell(anchor_idx) = anchor else { return Err("CellDerivedItemProvider requires a cell-derived anchor".into()); }; @@ -136,50 +616,59 @@ impl ProviderInst { ) .into()); } - // Candidates: all cell items, minus anchor, restricted by kind. - let mut filtered: Vec = Vec::new(); - for (i, cand) in sem.cell_items.iter().enumerate() { + let mut result: Vec = Vec::new(); + if *cardinality == 0 { + return Ok(result); + } + let range = index.lookup(scope, anch, *order, env, sem); + let a = range.items; + // J \ {anchor}, the kind restriction on J and κ; stops once k + // items are collected. + let accept = |i: usize, result: &mut Vec| -> CoreResult { if *exclude_anchor && i == anchor_idx { - continue; + return Ok(false); } + let cand = &sem.cell_items[i]; let type_ok = match kind { CellKind::Unrestricted | CellKind::Aux => true, CellKind::Val => cand.ty == ItemType::Value, CellKind::Attr => cand.ty == ItemType::Attribute, }; - if !type_ok { - continue; + if !type_ok || !cond.eval(anch, cand, env)? { + return Ok(false); } - if cond.eval(anch, cand, env)? { - filtered.push(i); + result.push(ItemId::Cell(i)); + Ok(*cardinality != UNBOUNDED && result.len() as i64 >= *cardinality) + }; + if !range.backward { + for &i in a { + if accept(i, &mut result)? { + return Ok(result); + } } + return Ok(result); } - // Linearization Ω_τ. - filtered.sort_by(|&x, &y| { - let a = &sem.cell_items[x]; - let b = &sem.cell_items[y]; - if a.row == b.row && a.col == b.col { - return a.index.cmp(&b.index); + // Reverse traversal: cells in reverse order, items inside a cell + // still by ascending index. + let mut i = a.len(); + while i > 0 { + let last = i - 1; + let (r, c) = (sem.cell_items[a[last]].row, sem.cell_items[a[last]].col); + let mut j = last; + while j > 0 + && sem.cell_items[a[j - 1]].row == r + && sem.cell_items[a[j - 1]].col == c + { + j -= 1; } - match order { - TraversalOrder::RowMajor => { - a.row.cmp(&b.row).then(a.col.cmp(&b.col)) - } - TraversalOrder::ReverseRowMajor => { - b.row.cmp(&a.row).then(b.col.cmp(&a.col)) - } - TraversalOrder::ColumnMajor => { - a.col.cmp(&b.col).then(a.row.cmp(&b.row)) - } - TraversalOrder::ReverseColumnMajor => { - b.col.cmp(&a.col).then(b.row.cmp(&a.row)) + for &k in &a[j..=last] { + if accept(k, &mut result)? { + return Ok(result); } } - }); - if *cardinality != UNBOUNDED && filtered.len() as i64 > *cardinality { - filtered.truncate(*cardinality as usize); + i = j; } - Ok(filtered.into_iter().map(ItemId::Cell).collect()) + Ok(result) } } } @@ -235,22 +724,149 @@ impl ProviderInst { } } +// ---------------------------------------------------------------- Diagnostic + +/// A diagnostic recorded during working state completion when an operation is +/// not applicable to its anchor (a violated precondition, a missing record). +/// By the formal model the operation then has no effect; the diagnostic makes +/// the silent no-op visible to the pattern author. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Diagnostic { + /// Index of the anchor into `SemanticsCore::cell_items`. + pub anchor: usize, + /// The operation name (`CONCAT`, `JOIN`, …). + pub operation: String, + /// What precondition was violated and why. + pub message: String, +} + +impl Diagnostic { + /// `Diagnostic.toString()`: ` skipped at : `. + pub fn describe(&self, sem: &SemanticsCore) -> String { + format!( + "{} skipped at {}: {}", + self.operation, + sem.cell_items[self.anchor].describe(), + self.message + ) + } +} + // ---------------------------------------------------------------- WorkingState -/// Port of `WorkingState`: insertion-ordered maps keyed by item identity. +/// Port of `WorkingState`: `ws = (V, A, val, attr, avp, rec, J)`, insertion- +/// ordered maps keyed by item identity. +/// +/// `rec` maps an anchor (cell-item index) to a *non-empty sequence* of +/// item-based records: a single record after `O_rec` / `O_concat`, several +/// after `O_join` (the record product). `J` — the *joined-away anchors* — are +/// items whose records have been consumed by a join; they stay in `rec` (a +/// later join may consume the same records again, irrespective of action +/// order) but are excluded from recordset extraction. `C` — the +/// *concatenated-away anchors* — are items whose records were folded into +/// another anchor's record by a concatenation and left `dom(rec)`; the set is +/// kept so that a later action on such an anchor can be told apart from an +/// anchor that never had a record. #[derive(Default, Debug)] pub struct WorkingState { - pub val: IndexMap, - pub attr: IndexMap, - pub avp: IndexMap, - pub rec: IndexMap>, // keyed by cell-item index + pub val: FastIndexMap, + pub attr: FastIndexMap, + pub avp: FastIndexMap, + /// Keyed by cell-item index; insertion order defines the order of records. + /// Concatenated-away anchors keep their (stale) entry and are masked by `C`. + rec: FastIndexMap>>, + /// J: joined-away anchors. + joined: FxHashSet, + /// C: concatenated-away anchors (removed from `dom(rec)`). + concatenated: FxHashSet, + /// Preconditions violated during completion; the operations had no effect. + diagnostics: Vec, + /// If set, a violated precondition raises instead of being recorded. + strict_preconditions: bool, } impl WorkingState { + pub fn new(strict_preconditions: bool) -> WorkingState { + WorkingState { strict_preconditions, ..Default::default() } + } + pub fn assoc(&self, item: ItemId) -> Option<&str> { self.avp.get(&item).map(|(a, _)| a.as_str()) } + // --- rec accessors --- + + /// ι ∈ dom(rec). + pub fn has_rec(&self, anchor: usize) -> bool { + self.rec.contains_key(&anchor) && !self.concatenated.contains(&anchor) + } + + /// rec(ι): the records of the anchor, also for joined-away anchors; + /// `None` if ι ∉ dom(rec). + pub fn rec(&self, anchor: usize) -> Option<&Vec>> { + if self.concatenated.contains(&anchor) { + return None; + } + self.rec.get(&anchor) + } + + /// ι ∈ J. + pub fn is_joined(&self, anchor: usize) -> bool { + self.joined.contains(&anchor) + } + + /// ι ∈ C. + pub fn is_concatenated(&self, anchor: usize) -> bool { + self.concatenated.contains(&anchor) + } + + /// The *live* anchors `dom(rec) \ J` in insertion order — exactly what + /// recordset extraction sees. + pub fn live_anchors(&self) -> Vec { + self.rec + .keys() + .copied() + .filter(|a| !self.joined.contains(a) && !self.concatenated.contains(a)) + .collect() + } + + pub fn all_joined(&self) -> &FxHashSet { + &self.joined + } + + pub fn all_concatenated(&self) -> &FxHashSet { + &self.concatenated + } + + pub fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } + + pub fn take_diagnostics(&mut self) -> Vec { + std::mem::take(&mut self.diagnostics) + } + + /// Records that an operation is not applicable to its anchor and has no + /// effect: adds a [`Diagnostic`]; under strict preconditions returns an + /// error with the same text instead (Java: `IllegalStateException`). + pub fn report( + &mut self, + sem: &SemanticsCore, + anchor: usize, + operation: &str, + message: impl Into, + ) -> CoreResult<()> { + let d = Diagnostic { anchor, operation: operation.to_string(), message: message.into() }; + let text = d.describe(sem); + self.diagnostics.push(d); + if self.strict_preconditions { + return Err(text.into()); + } + Ok(()) + } + + // --- string helpers --- + fn get_val_or_attr(&self, sem: &SemanticsCore, anchor: ItemId) -> CoreResult { match sem.item_type(anchor) { ItemType::Value => self @@ -293,6 +909,8 @@ impl WorkingState { .join(delimiter) } + // --- O_fill / O_prefix / O_suffix --- + pub fn apply_fill( &mut self, sem: &SemanticsCore, @@ -334,6 +952,8 @@ impl WorkingState { self.set_val_or_attr(sem, anchor, format!("{current}{delimiter}{suffix}")) } + // --- O_avp --- + pub fn apply_avp(&mut self, anchor: ItemId, items: &[ItemId]) -> CoreResult<()> { if self.avp.contains_key(&anchor) { return Ok(()); @@ -356,6 +976,8 @@ impl WorkingState { Ok(()) } + // --- O_rec: rec(anchor) := <> --- + pub fn apply_rec(&mut self, sem: &SemanticsCore, anchor: ItemId, items: &[ItemId]) -> CoreResult<()> { let ItemId::Cell(anchor_idx) = anchor else { return Err("O_rec requires a cell-derived anchor".into()); @@ -378,65 +1000,301 @@ impl WorkingState { } sequence.push(item); } - self.rec.insert(anchor_idx, sequence); + self.rec.insert(anchor_idx, vec![sequence]); + Ok(()) + } + + /// The provided cell-derived items other than the anchor that have a + /// record, without duplicates, in provider order. + fn others_with_records(&self, anchor_idx: usize, items: &[ItemId]) -> Vec { + let mut others: Vec = Vec::new(); + for &item in items { + if let ItemId::Cell(ci) = item { + if ci != anchor_idx && self.has_rec(ci) && !others.contains(&ci) { + others.push(ci); + } + } + } + others + } + + // --- O_concat^K: rec(anchor) := ; rec.remove(i_k) --- + + /// Concatenates the records of the provided anchors to the anchor's record + /// (one wide record) and removes them from dom(rec). Applicable iff (i) the + /// anchor and at least one provided item have records, (ii) all records + /// agree on the key K (key positions and/or key attribute names), and (iii) + /// apart from the key no named attribute occurs in more than one of the + /// concatenated records; otherwise the operation has no effect and a + /// [`Diagnostic`] is recorded. + pub fn apply_concat( + &mut self, + sem: &SemanticsCore, + anchor: ItemId, + items: &[ItemId], + key: &RecordKey, + ) -> CoreResult<()> { + let ItemId::Cell(anchor_idx) = anchor else { + return Err("O_concat requires a cell-derived anchor".into()); + }; + let Some(anchor_recs) = self.rec(anchor_idx) else { + return Ok(()); + }; + if items.is_empty() { + return Ok(()); + } + let anchor_rec = anchor_recs[0].clone(); + let others = self.others_with_records(anchor_idx, items); + if others.is_empty() { + return Ok(()); // (i) + } + let mut result = anchor_rec.clone(); + for &other in &others { + let other_rec = &self.rec[&other][0]; + if let Some(problem) = self.key_mismatch(&anchor_rec, other_rec, key) { + return self.report(sem, anchor_idx, "CONCAT", problem); // (ii) + } + result.extend(self.drop_k(other_rec, key)); + } + if let Some(duplicate) = self.duplicate_attribute(&result) { + return self.report( + sem, + anchor_idx, + "CONCAT", + format!( + "named attribute '{duplicate}' occurs in more than one of the concatenated records" + ), + ); // (iii) + } + self.rec.insert(anchor_idx, vec![result]); + for other in others { + self.joined.remove(&other); + self.concatenated.insert(other); + } Ok(()) } + // --- O_join^K: rec(anchor) := ; J := J ∪ {i_k} --- + + /// Multiplies every record of the anchor by every record of the provided + /// anchors (a cross product for K = ∅, an equi-join on the key K otherwise; + /// a named attribute shared by two records acts as a natural-join + /// condition) and marks the provided anchors as joined-away. Pairs whose + /// keys differ or whose shared attributes disagree are dropped; if no pair + /// survives, the anchor keeps its records (left outer join). pub fn apply_join( &mut self, + sem: &SemanticsCore, anchor: ItemId, items: &[ItemId], - key_positions: &BTreeSet, + key: &RecordKey, ) -> CoreResult<()> { let ItemId::Cell(anchor_idx) = anchor else { return Err("O_join requires a cell-derived anchor".into()); }; - let Some(anchor_rec) = self.rec.get(&anchor_idx).cloned() else { + let Some(anchor_recs) = self.rec(anchor_idx) else { return Ok(()); }; if items.is_empty() { return Ok(()); } - let mut result = anchor_rec; - for &item in items { - let ItemId::Cell(ci) = item else { continue }; - let Some(other_rec) = self.rec.get(&ci).cloned() else { + let anchor_recs = anchor_recs.clone(); + let others = self.others_with_records(anchor_idx, items); + if others.is_empty() { + return Ok(()); + } + let mut joined_recs: Vec> = Vec::new(); + for &other in &others { + joined_recs.extend(self.rec[&other].iter().cloned()); + } + + let mut result: Vec> = Vec::new(); + let mut dropped = 0usize; + for rho in &anchor_recs { + for rho2 in &joined_recs { + if self.key_mismatch(rho, rho2, key).is_some() || !self.agree(rho, rho2) { + dropped += 1; + continue; + } + let mut combined = rho.clone(); + combined.extend(self.drop_k(rho2, key)); + result.push(self.dedup(&combined)); + } + } + if result.is_empty() { + self.report( + sem, + anchor_idx, + "JOIN", + format!( + "none of the {dropped} record pairs satisfies the key/attribute conditions; the anchor keeps its records" + ), + )?; + } else { + self.rec.insert(anchor_idx, result); + } + self.joined.extend(others); + Ok(()) + } + + /// drop_K(ρ̄): the sequence with the key items removed — the items at the + /// key positions (0-based) and the items carrying a key attribute name; + /// the latter are resolved per record. + fn drop_k(&self, sequence: &[ItemId], key: &RecordKey) -> Vec { + if key.is_empty() { + return sequence.to_vec(); + } + let mut result = Vec::with_capacity(sequence.len()); + for (i, &item) in sequence.iter().enumerate() { + if key.positions.contains(&(i as i64)) { continue; - }; - for (pos, it) in other_rec.iter().enumerate() { - if !key_positions.contains(&(pos as i64)) { - result.push(*it); + } + if let Some(a) = self.assoc(item) { + if key.names.contains(a) { + continue; } } - self.rec.shift_remove(&ci); + result.push(item); } - // dedup by named attribute: keep first occurrence. - let mut seen: Vec = Vec::new(); - let mut deduped = Vec::with_capacity(result.len()); - for it in result { - match self.assoc(it) { + result + } + + /// The index of the first item of the record carrying the named attribute. + fn index_of_attribute(&self, sequence: &[ItemId], attribute: &str) -> Option { + sequence.iter().position(|&it| self.assoc(it) == Some(attribute)) + } + + /// dedup(ρ̄): keeps the first occurrence of each named attribute; items + /// without avp are always kept. + fn dedup(&self, sequence: &[ItemId]) -> Vec { + let mut seen: HashSet<&str> = HashSet::new(); + let mut result = Vec::with_capacity(sequence.len()); + for &item in sequence { + match self.assoc(item) { Some(a) => { - if !seen.iter().any(|x| x == a) { - seen.push(a.to_string()); - deduped.push(it); + if seen.insert(a) { + result.push(item); } } - None => deduped.push(it), + None => result.push(item), } } - self.rec.insert(anchor_idx, deduped); - Ok(()) + result + } + + fn pair_text(pair: &(String, String)) -> String { + format!("AttributeValuePair[attribute={}, value={}]", pair.0, pair.1) + } + + /// compat_K(ρ, ρ'): `None` if for every key position k both records are + /// long enough and the items at k are compatible (both with the same + /// attribute-value pair, or both unnamed with the same value), and for + /// every key attribute name both records carry the name with the same + /// attribute-value pair; otherwise a description of the first mismatch. + fn key_mismatch(&self, rho: &[ItemId], rho2: &[ItemId], key: &RecordKey) -> Option { + for &k in &key.positions { + let k = k as usize; + if k >= rho.len() || k >= rho2.len() { + return Some(format!("key position {k} is beyond the end of a record")); + } + let (a, b) = (rho[k], rho2[k]); + match (self.avp.get(&a), self.avp.get(&b)) { + (Some(pa), Some(pb)) => { + if pa != pb { + return Some(format!( + "key position {k} differs: {} vs {}", + Self::pair_text(pa), + Self::pair_text(pb) + )); + } + } + (None, None) => { + if self.val.get(&a) != self.val.get(&b) { + return Some(format!( + "key position {k} differs: '{}' vs '{}'", + self.val.get(&a).map(|s| s.as_str()).unwrap_or("null"), + self.val.get(&b).map(|s| s.as_str()).unwrap_or("null") + )); + } + } + _ => return Some(format!("key position {k} mixes a named and an unnamed item")), + } + } + for name in &key.names { + let (i, j) = ( + self.index_of_attribute(rho, name), + self.index_of_attribute(rho2, name), + ); + let (Some(i), Some(j)) = (i, j) else { + return Some(format!("key attribute '{name}' is missing in a record")); + }; + let (pa, pb) = (&self.avp[&rho[i]], &self.avp[&rho2[j]]); + if pa != pb { + return Some(format!( + "key attribute '{name}' differs: {} vs {}", + Self::pair_text(pa), + Self::pair_text(pb) + )); + } + } + None + } + + /// agree(ρ, ρ'): every named attribute appearing in both records carries + /// the same value. + fn agree(&self, rho: &[ItemId], rho2: &[ItemId]) -> bool { + let mut named: HashMap<&str, &str> = HashMap::new(); + for &item in rho { + if let Some((a, v)) = self.avp.get(&item) { + named.entry(a.as_str()).or_insert(v.as_str()); + } + } + for &item in rho2 { + if let Some((a, v)) = self.avp.get(&item) { + if let Some(&existing) = named.get(a.as_str()) { + if existing != v { + return false; + } + } + } + } + true + } + + /// The first named attribute occurring twice in the sequence. + fn duplicate_attribute(&self, sequence: &[ItemId]) -> Option { + let mut seen: Vec<&str> = Vec::with_capacity(sequence.len()); + self.duplicate_attribute_in(sequence, &mut seen).map(|a| a.to_string()) + } + + /// [`Self::duplicate_attribute`] with a caller-provided scratch buffer + /// (records are short, so a linear scan beats a hash set and the buffer + /// is reused across the millions of records of a large table). + fn duplicate_attribute_in<'a>(&'a self, sequence: &[ItemId], seen: &mut Vec<&'a str>) -> Option<&'a str> { + seen.clear(); + for &item in sequence { + if let Some(a) = self.assoc(item) { + if seen.contains(&a) { + return Some(a); + } + seen.push(a); + } + } + None } pub fn set_avp(&mut self, item: ItemId, attribute: String, value: String) { self.avp.insert(item, (attribute, value)); } - // --- consistency checks --- + // --- consistency checks (over the live anchors) --- pub fn is_anchor_attribute_uniform(&self) -> bool { let mut common: Option<&str> = None; - for &anchor_idx in self.rec.keys() { + for anchor_idx in self.rec.keys().copied() { + if self.joined.contains(&anchor_idx) || self.concatenated.contains(&anchor_idx) { + continue; + } if let Some(a) = self.assoc(ItemId::Cell(anchor_idx)) { match common { None => common = Some(a), @@ -452,14 +1310,14 @@ impl WorkingState { } pub fn is_record_attributes_distinct(&self) -> bool { - for sequence in self.rec.values() { - let mut seen: Vec<&str> = Vec::new(); - for &item in sequence { - if let Some(a) = self.assoc(item) { - if seen.contains(&a) { - return false; - } - seen.push(a); + let mut seen: Vec<&str> = Vec::new(); + for (anchor_idx, records) in &self.rec { + if self.joined.contains(anchor_idx) || self.concatenated.contains(anchor_idx) { + continue; + } + for sequence in records { + if self.duplicate_attribute_in(sequence, &mut seen).is_some() { + return false; } } } @@ -470,3 +1328,675 @@ impl WorkingState { self.is_anchor_attribute_uniform() && self.is_record_attributes_distinct() } } + +// ---------------------------------------------------------------- unit tests + +#[cfg(test)] +mod tests { + use super::*; + use crate::syntax::SyntaxCore; + + // ------------------------------------------------ working state fixtures + + /// A semantics layer of VALUE items placed at the given cells (index 0 + /// unless a `(text, row, col, index)` quadruple says otherwise). + fn sem_of(items: &[(&str, usize, usize, usize)]) -> SemanticsCore { + SemanticsCore { + cell_items: items + .iter() + .map(|&(s, r, c, i)| CellItem { + s: s.to_string(), + tags: Vec::new(), + index: i, + row: r, + col: c, + ty: ItemType::Value, + span: (0, s.len()), + }) + .collect(), + ctx_items: Vec::new(), + actions: Vec::new(), + } + } + + fn init(sem: &SemanticsCore, strict: bool) -> WorkingState { + let mut ws = WorkingState::new(strict); + for (i, it) in sem.cell_items.iter().enumerate() { + ws.val.insert(ItemId::Cell(i), it.s.clone()); + } + ws + } + + fn name(ws: &mut WorkingState, sem: &SemanticsCore, item: usize, attribute: &str) { + ws.set_avp(ItemId::Cell(item), attribute.to_string(), sem.cell_items[item].s.clone()); + } + + fn cells(ids: &[usize]) -> Vec { + ids.iter().map(|&i| ItemId::Cell(i)).collect() + } + + fn key_pos(k: &[i64]) -> RecordKey { + RecordKey::positions(k.iter().copied()).unwrap() + } + + fn key_names(a: &[&str]) -> RecordKey { + RecordKey::names(a.iter().map(|s| s.to_string())).unwrap() + } + + // ------------------------------------------------ O_concat (port of WorkingStateConcatTest) + + #[test] + fn concat_folds_records_with_distinct_attributes_key_not_repeated() { + // T-1 | D16 rec(0) = + // T-1 | 001 rec(2) = + let sem = sem_of(&[("T-1", 1, 0, 0), ("D16", 1, 1, 0), ("T-1", 2, 0, 0), ("001", 2, 1, 0)]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 0, "ID"); + name(&mut ws, &sem, 1, "REF_TP"); + name(&mut ws, &sem, 2, "ID"); + name(&mut ws, &sem, 3, "REF_SN"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1, 3])])); + assert!(!ws.has_rec(2), "the concatenated anchor is removed from dom(rec)"); + assert!(ws.is_concatenated(2)); + assert_eq!(ws.live_anchors(), vec![0]); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn concat_folds_unnamed_records_task016_shape() { + // book | 5 ; book | 6 ; book | 7 -> book,5,6,7 + let sem = sem_of(&[ + ("book", 1, 0, 0), ("5", 1, 1, 0), + ("book", 2, 0, 0), ("6", 2, 1, 0), + ("book", 3, 0, 0), ("7", 3, 1, 0), + ]); + let mut ws = init(&sem, false); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(4), &cells(&[5])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2, 4]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1, 3, 5])])); + assert!(!ws.has_rec(2) && !ws.has_rec(4)); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn concat_shared_named_attribute_no_effect_and_diagnostic() { + // A | 5 rec = + // A | 7 rec = -- Qty occurs in both: not a key, a conflict + let sem = sem_of(&[("A", 1, 0, 0), ("5", 1, 1, 0), ("A", 2, 0, 0), ("7", 2, 1, 0)]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 0, "ID"); + name(&mut ws, &sem, 1, "Qty"); + name(&mut ws, &sem, 2, "ID"); + name(&mut ws, &sem, 3, "Qty"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1])]), "anchor record unchanged"); + assert_eq!(ws.rec(2), Some(&vec![cells(&[2, 3])]), "the other record is kept: both survive"); + assert_eq!(ws.live_anchors().len(), 2); + assert_eq!(ws.diagnostics().len(), 1); + let d = &ws.diagnostics()[0]; + assert_eq!((d.anchor, d.operation.as_str()), (0, "CONCAT")); + assert!(d.message.contains("Qty"), "{}", d.message); + } + + #[test] + fn concat_key_position_mismatch_no_effect_and_diagnostic() { + let sem = sem_of(&[("X", 1, 0, 0), ("5", 1, 1, 0), ("Y", 2, 0, 0), ("7", 2, 1, 0)]); + let mut ws = init(&sem, false); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1])])); + assert!(ws.has_rec(2)); + assert_eq!(ws.diagnostics().len(), 1); + assert!(ws.diagnostics()[0].message.contains("key position 0")); + } + + #[test] + fn concat_strict_preconditions_fail_with_the_same_message() { + let sem = sem_of(&[("A", 1, 0, 0), ("5", 1, 1, 0), ("A", 2, 0, 0), ("7", 2, 1, 0)]); + let mut ws = init(&sem, true); + name(&mut ws, &sem, 0, "ID"); + name(&mut ws, &sem, 1, "Qty"); + name(&mut ws, &sem, 2, "ID"); + name(&mut ws, &sem, 3, "Qty"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + let err = ws + .apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])) + .unwrap_err(); + let text = err.text(); + assert!(text.contains("Qty") && text.starts_with("CONCAT skipped at"), "{text}"); + assert_eq!(ws.diagnostics().len(), 1, "the diagnostic is recorded before failing"); + } + + #[test] + fn concat_items_without_records_are_ignored() { + let sem = sem_of(&[("book", 1, 0, 0), ("5", 1, 1, 0), ("book", 2, 0, 0)]); + let mut ws = init(&sem, false); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + let before = ws.rec(0).cloned(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0).cloned(), before); + assert!(ws.diagnostics().is_empty(), "no record to concatenate is not a violation"); + } + + #[test] + fn concat_named_key_resolved_per_record_at_different_positions() { + // rec(0) = A at position 2 + // rec(3) = A at position 1 + let sem = sem_of(&[ + ("k", 1, 0, 0), ("c1", 1, 1, 0), ("a1", 1, 2, 0), + ("k", 2, 0, 0), ("a1", 2, 1, 0), ("c2", 2, 2, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 2, "A"); + name(&mut ws, &sem, 4, "A"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1, 2])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(3), &cells(&[4, 5])).unwrap(); + + let key = RecordKey::new([0].into(), ["A".to_string()].into()).unwrap(); + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[3]), &key).unwrap(); + + assert_eq!( + ws.rec(0), + Some(&vec![cells(&[0, 1, 2, 5])]), + "the anchor keeps its key; A is dropped from the concatenated record at its own position" + ); + assert!(!ws.has_rec(3)); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn concat_named_key_missing_in_a_record_no_effect_and_diagnostic() { + let sem = sem_of(&[("k", 1, 0, 0), ("a1", 1, 1, 0), ("k", 2, 0, 0), ("c2", 2, 1, 0)]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 1, "A"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key_names(&["A"])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1])])); + assert!(ws.has_rec(2), "no effect: both records remain"); + assert_eq!(ws.diagnostics().len(), 1); + assert!(ws.diagnostics()[0].message.contains("key attribute 'A' is missing")); + } + + #[test] + fn concat_named_key_values_differ_no_effect_and_diagnostic() { + let sem = sem_of(&[("k", 1, 0, 0), ("a1", 1, 1, 0), ("k", 2, 0, 0), ("a2", 2, 1, 0)]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 1, "A"); + name(&mut ws, &sem, 3, "A"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + let key = RecordKey::new([0].into(), ["A".to_string()].into()).unwrap(); + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[2]), &key).unwrap(); + + assert!(ws.has_rec(2)); + assert_eq!(ws.diagnostics().len(), 1); + assert!(ws.diagnostics()[0].message.contains("key attribute 'A' differs")); + } + + #[test] + fn concat_mixed_key_is_equivalent_to_the_positional_key_task098_shape() { + // k1 | k11 | a1:A | b1:B | c1 CONCAT(0,1,2,3) == CONCAT(0,1,'A','B') + // k1 | k11 | a1:A | b1:B | c2 + let mixed = RecordKey::new([0, 1].into(), ["A".to_string(), "B".to_string()].into()).unwrap(); + for key in [key_pos(&[0, 1, 2, 3]), mixed] { + let sem = sem_of(&[ + ("k1", 1, 0, 0), ("k11", 1, 1, 0), ("a1", 1, 2, 0), ("b1", 1, 3, 0), ("c1", 1, 4, 0), + ("k1", 2, 0, 0), ("k11", 2, 1, 0), ("a1", 2, 2, 0), ("b1", 2, 3, 0), ("c2", 2, 4, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 2, "A"); + name(&mut ws, &sem, 3, "B"); + name(&mut ws, &sem, 7, "A"); + name(&mut ws, &sem, 8, "B"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1, 2, 3, 4])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(5), &cells(&[6, 7, 8, 9])).unwrap(); + + ws.apply_concat(&sem, ItemId::Cell(0), &cells(&[5]), &key).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1, 2, 3, 4, 9])])); + assert!(ws.diagnostics().is_empty()); + } + } + + // ------------------------------------------------ O_join (port of WorkingStateJoinTest) + + // id | x | y tokens a (0), b (1) of one cell; rec(2) = , rec(3) = + // a;b | 1 | 2 + fn explode_stack() -> (SemanticsCore, WorkingState) { + let sem = sem_of(&[ + ("a", 1, 0, 0), ("b", 1, 0, 1), + ("1", 1, 1, 0), ("2", 1, 2, 0), + ("x", 0, 1, 0), ("y", 0, 2, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 0, "id"); + name(&mut ws, &sem, 1, "id"); + name(&mut ws, &sem, 2, "value"); + name(&mut ws, &sem, 3, "value"); + name(&mut ws, &sem, 4, "var"); + name(&mut ws, &sem, 5, "var"); + ws.apply_rec(&sem, ItemId::Cell(0), &[]).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(1), &[]).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[4])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(3), &cells(&[5])).unwrap(); + (sem, ws) + } + + #[test] + fn join_cross_product_one_record_per_joined_record_in_nested_loop_order() { + let (sem, mut ws) = explode_stack(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 3]), &RecordKey::empty()).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 2, 4]), cells(&[0, 3, 5])])); + assert!(ws.is_joined(2) && ws.is_joined(3)); + assert_eq!(ws.all_joined(), &FxHashSet::from_iter([2, 3])); + assert!(ws.rec(2).is_some(), "a joined-away anchor keeps its records"); + assert_eq!(ws.live_anchors(), vec![0, 1], "live anchors only"); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn join_lazy_consumption_second_anchor_joins_the_same_records() { + let (sem, mut ws) = explode_stack(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 3]), &RecordKey::empty()).unwrap(); + ws.apply_join(&sem, ItemId::Cell(1), &cells(&[2, 3]), &RecordKey::empty()).unwrap(); + + assert_eq!(ws.rec(0).unwrap().len(), 2); + assert_eq!(ws.rec(1), Some(&vec![cells(&[1, 2, 4]), cells(&[1, 3, 5])])); + assert!( + ws.is_recordset_consistent(), + "the 'value' attribute of the joined-away anchors does not break uniformity" + ); + } + + #[test] + fn join_composition_second_join_multiplies_further() { + let (mut sem, mut ws) = explode_stack(); + for (s, r, c) in [("p", 2, 3), ("q", 2, 4)] { + sem.cell_items.push(CellItem { + s: s.to_string(), + tags: Vec::new(), + index: 0, + row: r, + col: c, + ty: ItemType::Value, + span: (0, 1), + }); + } + ws.val.insert(ItemId::Cell(6), "p".into()); + ws.val.insert(ItemId::Cell(7), "q".into()); + name(&mut ws, &sem, 6, "w"); + name(&mut ws, &sem, 7, "w"); + ws.apply_rec(&sem, ItemId::Cell(6), &[]).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(7), &[]).unwrap(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 3]), &RecordKey::empty()).unwrap(); + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[6, 7]), &RecordKey::empty()).unwrap(); + + assert_eq!( + ws.rec(0), + Some(&vec![ + cells(&[0, 2, 4, 6]), cells(&[0, 2, 4, 7]), + cells(&[0, 3, 5, 6]), cells(&[0, 3, 5, 7]), + ]) + ); + } + + #[test] + fn join_equi_join_on_key_position_drops_mismatched_pairs_and_the_joined_key() { + // rec(p1) = rec(q1) = rec(q2) = + let sem = sem_of(&[ + ("X", 1, 0, 0), ("5", 1, 1, 0), + ("X", 1, 2, 0), ("kg", 1, 3, 0), + ("Z", 2, 2, 0), ("pc", 2, 3, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 1, "Qty"); + name(&mut ws, &sem, 3, "Unit"); + name(&mut ws, &sem, 5, "Unit"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(4), &cells(&[5])).unwrap(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 4]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1, 3])])); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn join_no_surviving_pair_anchor_keeps_its_records_left_outer() { + let sem = sem_of(&[("Y", 1, 0, 0), ("8", 1, 1, 0), ("X", 1, 2, 0), ("kg", 1, 3, 0)]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 1, "Qty"); + name(&mut ws, &sem, 3, "Unit"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2]), &key_pos(&[0])).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0, 1])])); + assert!(ws.is_joined(2), "J is still extended"); + assert_eq!(ws.diagnostics().len(), 1); + assert_eq!(ws.diagnostics()[0].operation, "JOIN"); + } + + #[test] + fn join_shared_named_attribute_is_a_natural_join_condition() { + // rec(r1) = ; rec(s1) = ; rec(s2) = + let sem = sem_of(&[ + ("S1", 1, 0, 0), ("2024", 1, 1, 0), + ("2024", 2, 0, 0), ("10", 2, 1, 0), + ("2025", 3, 0, 0), ("12", 3, 1, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 0, "Store"); + name(&mut ws, &sem, 1, "Year"); + name(&mut ws, &sem, 2, "Year"); + name(&mut ws, &sem, 3, "Sales"); + name(&mut ws, &sem, 4, "Year"); + name(&mut ws, &sem, 5, "Sales"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(4), &cells(&[5])).unwrap(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 4]), &RecordKey::empty()).unwrap(); + + let records = ws.rec(0).unwrap(); + assert_eq!(records.len(), 1, "the 2025 pair disagrees on Year and is dropped"); + assert_eq!(records[0], cells(&[0, 1, 3]), "Year occurs once (dedup)"); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn join_items_without_records_are_ignored() { + let (mut sem, mut ws) = explode_stack(); + sem.cell_items.push(CellItem { + s: "z".into(), + tags: Vec::new(), + index: 0, + row: 3, + col: 0, + ty: ItemType::Value, + span: (0, 1), + }); + ws.val.insert(ItemId::Cell(6), "z".into()); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[6]), &RecordKey::empty()).unwrap(); + + assert_eq!(ws.rec(0), Some(&vec![cells(&[0])])); + assert!(ws.all_joined().is_empty()); + assert!(ws.diagnostics().is_empty()); + } + + #[test] + fn join_named_key_pair_without_the_attribute_is_dropped_unlike_the_bare_join() { + // rec(p) = ; rec(s1) = ; rec(s2) = (no Year) + let sem = sem_of(&[ + ("x", 1, 0, 0), ("2024", 1, 1, 0), + ("y", 2, 0, 0), ("2024", 2, 1, 0), ("10", 2, 2, 0), + ("z", 3, 0, 0), ("12", 3, 2, 0), + ]); + let mut ws = init(&sem, false); + name(&mut ws, &sem, 1, "Year"); + name(&mut ws, &sem, 3, "Year"); + name(&mut ws, &sem, 4, "Sales"); + name(&mut ws, &sem, 6, "Sales"); + ws.apply_rec(&sem, ItemId::Cell(0), &cells(&[1])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(2), &cells(&[3, 4])).unwrap(); + ws.apply_rec(&sem, ItemId::Cell(5), &cells(&[6])).unwrap(); + + ws.apply_join(&sem, ItemId::Cell(0), &cells(&[2, 5]), &key_names(&["Year"])).unwrap(); + + assert_eq!( + ws.rec(0), + Some(&vec![cells(&[0, 1, 2, 4])]), + "only the pair carrying Year on both sides survives; the joined key is not repeated" + ); + assert_eq!(ws.all_joined(), &FxHashSet::from_iter([2, 5])); + } + + // ------------------------------------------------ provider index equivalence + + /// Reference definition Υ^{J,k}_{τ,κ}(anchor) = Ω_τ(Φ_κ(anchor, J \ {anchor}))[:k] + /// — full scan and sort (the implementation up to 0.5.1). + #[allow(clippy::too_many_arguments)] + fn provide_reference( + cond: &FilterCond, + order: TraversalOrder, + cardinality: i64, + kind: CellKind, + exclude_anchor: bool, + anchor_idx: usize, + sem: &SemanticsCore, + env: &EvalEnv, + ) -> Vec { + let anch = &sem.cell_items[anchor_idx]; + let mut filtered: Vec = Vec::new(); + for (i, cand) in sem.cell_items.iter().enumerate() { + if exclude_anchor && i == anchor_idx { + continue; + } + let type_ok = match kind { + CellKind::Unrestricted | CellKind::Aux => true, + CellKind::Val => cand.ty == ItemType::Value, + CellKind::Attr => cand.ty == ItemType::Attribute, + }; + if type_ok && cond.eval(anch, cand, env).unwrap() { + filtered.push(i); + } + } + filtered.sort_by(|&x, &y| { + let a = &sem.cell_items[x]; + let b = &sem.cell_items[y]; + if a.row == b.row && a.col == b.col { + return a.index.cmp(&b.index); + } + match order { + TraversalOrder::RowMajor => a.row.cmp(&b.row).then(a.col.cmp(&b.col)), + TraversalOrder::ReverseRowMajor => b.row.cmp(&a.row).then(b.col.cmp(&a.col)), + TraversalOrder::ColumnMajor => a.col.cmp(&b.col).then(a.row.cmp(&b.row)), + TraversalOrder::ReverseColumnMajor => b.col.cmp(&a.col).then(b.row.cmp(&a.row)), + } + }); + if cardinality != UNBOUNDED && filtered.len() as i64 > cardinality { + filtered.truncate(cardinality as usize); + } + filtered.into_iter().map(ItemId::Cell).collect() + } + + /// Tiny deterministic PRNG (xorshift) — no dev-dependency needed. + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + } + + /// Every filter term the scope derivation distinguishes, plus position, + /// content and tag terms (scope ALL), in random combinations. + fn random_cond(rng: &mut Rng) -> FilterCond { + fn term(rng: &mut Rng) -> FilterTerm { + let d = rng.below(5) as i64 - 2; + match rng.below(24) { + 0 => FilterTerm::LeftOf, + 1 => FilterTerm::RightOf, + 2 => FilterTerm::Above, + 3 => FilterTerm::Below, + 4 => FilterTerm::SameSubrow, + 5 => FilterTerm::SameSubcol, + 6 => FilterTerm::SameSubtable, + 7 => FilterTerm::SameRow, + 8 => FilterTerm::SameCol, + 9 => FilterTerm::NotSameCell, + 10 => FilterTerm::SameCell, + 11 => FilterTerm::ColExact(rng.below(6) as i64), + 12 => FilterTerm::ColOffset(d), + 13 => FilterTerm::ColRange(d, if rng.below(2) == 0 { UNBOUNDED } else { d + rng.below(3) as i64 }), + 14 => FilterTerm::ColAbsoluteRange(rng.below(4) as i64, if rng.below(2) == 0 { UNBOUNDED } else { rng.below(6) as i64 }), + 15 => FilterTerm::RowExact(rng.below(6) as i64), + 16 => FilterTerm::RowOffset(d), + 17 => FilterTerm::RowAbsoluteRange(rng.below(4) as i64, if rng.below(2) == 0 { UNBOUNDED } else { rng.below(6) as i64 }), + 18 => FilterTerm::PosExact(rng.below(3) as i64), + 19 => FilterTerm::PosOffset(d), + 20 => FilterTerm::Blank, + 21 => FilterTerm::NotBlank, + 22 => FilterTerm::SameStr, + _ => FilterTerm::Tagged("#t".into()), + } + } + match rng.below(4) { + 0 => FilterCond::Bare(term(rng)), + 1 => FilterCond::Or(vec![vec![term(rng)], vec![term(rng), term(rng)]]), + _ => FilterCond::And((0..1 + rng.below(3)).map(|_| term(rng)).collect()), + } + } + + #[test] + fn indexed_provider_equals_the_reference_definition_on_random_tables() { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + let orders = [ + TraversalOrder::RowMajor, + TraversalOrder::ReverseRowMajor, + TraversalOrder::ColumnMajor, + TraversalOrder::ReverseColumnMajor, + ]; + let kinds = [CellKind::Unrestricted, CellKind::Val, CellKind::Attr, CellKind::Aux]; + let mut checked = 0usize; + for _ in 0..25 { + let num_rows = 2 + rng.below(6); + let num_cols = 2 + rng.below(6); + let mut syntax = SyntaxCore::new(num_rows, num_cols).unwrap(); + // random subtable boundaries and subrows + let mut starts = vec![0]; + for r in 1..num_rows { + if rng.below(3) == 0 { + starts.push(r); + } + } + syntax.define_subtables(&starts).unwrap(); + for r in 0..num_rows { + if num_cols > 1 && rng.below(2) == 0 { + let split = 1 + rng.below(num_cols - 1); + syntax.define_subrow(r, 0, split - 1).unwrap(); + syntax.define_subrow(r, split, num_cols - 1).unwrap(); + } + } + // 0–3 items per cell, random types, some blank / tagged / repeated texts + let mut sem = SemanticsCore::default(); + let texts = ["a", "b", "", " ", "c"]; + for r in 0..num_rows { + for c in 0..num_cols { + let n = rng.below(4); + for i in 0..n { + let t = texts[rng.below(texts.len())]; + syntax.cell_mut(r, c).set_text(t.to_string()); + sem.cell_items.push(CellItem { + s: t.to_string(), + tags: if rng.below(3) == 0 { vec!["#t".into()] } else { Vec::new() }, + index: i, + row: r, + col: c, + ty: [ItemType::Value, ItemType::Attribute, ItemType::Auxiliary][rng.below(3)], + span: (0, t.len()), + }); + } + } + } + if sem.cell_items.is_empty() { + continue; + } + let env = EvalEnv { syntax: &syntax, py_table: None }; + // the index must serve subtable-scoped column-major providers too + sem.actions.push(ActionInst { + anchor: ItemId::Cell(0), + providers: vec![ProviderInst::Cell { + cond: FilterCond::Bare(FilterTerm::SameSubtable), + order: TraversalOrder::ColumnMajor, + cardinality: UNBOUNDED, + kind: CellKind::Unrestricted, + exclude_anchor: true, + lenient: true, + scope: CandidateScope::subtable(), + }], + op: OpInst::Rec, + inherited: false, + }); + let index = ItemIndex::build(&sem, &syntax); + for _ in 0..60 { + let cond = random_cond(&mut rng); + let scope = CandidateScope::of_cond(&cond); + for &order in &orders { + for &cardinality in &[UNBOUNDED, 1, 2, 3, 0] { + for &kind in &kinds { + for exclude_anchor in [true, false] { + let anchor_idx = rng.below(sem.cell_items.len()); + let anch = &sem.cell_items[anchor_idx]; + let compatible = match kind { + CellKind::Unrestricted => true, + CellKind::Val | CellKind::Attr => anch.ty == ItemType::Value, + CellKind::Aux => anch.ty != ItemType::Auxiliary, + }; + if !compatible { + continue; + } + let provider = ProviderInst::Cell { + cond: cond.clone(), + order, + cardinality, + kind, + exclude_anchor, + lenient: false, + scope, + }; + let got = provider + .provide(ItemId::Cell(anchor_idx), &sem, &env, &index) + .unwrap(); + let want = provide_reference( + &cond, order, cardinality, kind, exclude_anchor, anchor_idx, &sem, &env, + ); + assert_eq!(got, want, "cond={cond:?} order={order:?} k={cardinality} kind={kind:?} anchor={anchor_idx}"); + checked += 1; + } + } + } + } + } + } + assert!(checked > 10_000, "checked only {checked} provider calls"); + } +} diff --git a/src/spec.rs b/src/spec.rs index 029ecd2..4154539 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -53,9 +53,75 @@ pub enum OperationType { Suffix, Avp, Rec, + Concat, Join, } +impl OperationType { + /// The RTL keyword of the operation. + pub fn rtl_name(self) -> &'static str { + match self { + OperationType::Fill => "FILL", + OperationType::Prefix => "PREFIX", + OperationType::Suffix => "SUFFIX", + OperationType::Avp => "AVP", + OperationType::Rec => "REC", + OperationType::Concat => "CONCAT", + OperationType::Join => "JOIN", + } + } +} + +// ---------------------------------------------------------------- RecordKey + +/// The key K of `CONCAT(K)` / `JOIN(K)` (port of `RecordKey`): a set of 0-based +/// *positions* in the item-based record (0 is the anchor, the following +/// positions are the items in the order the `REC` providers supplied them) +/// and/or a set of attribute *names*. A name is resolved to a position per +/// record at apply time — the position of the item whose attribute-value pair +/// carries that name — so the key does not depend on the order of the fields. +/// +/// RTL: `CONCAT(0, 1, 'A', 'B')`, `JOIN('Year')`. The manuscript defines K over +/// positions only; names are an extension of the implementation with the same +/// semantics. +#[derive(Clone, PartialEq, Eq, Debug, Default)] +pub struct RecordKey { + pub positions: BTreeSet, + pub names: BTreeSet, +} + +impl RecordKey { + pub const fn empty() -> RecordKey { + RecordKey { positions: BTreeSet::new(), names: BTreeSet::new() } + } + + pub fn new(positions: BTreeSet, names: BTreeSet) -> CoreResult { + for &k in &positions { + if k < 0 { + return Err(format!("key position must be non-negative: {k}").into()); + } + } + for a in &names { + if java_is_blank(a) { + return Err("key attribute name must not be blank".into()); + } + } + Ok(RecordKey { positions, names }) + } + + pub fn positions(positions: impl IntoIterator) -> CoreResult { + RecordKey::new(positions.into_iter().collect(), BTreeSet::new()) + } + + pub fn names(names: impl IntoIterator) -> CoreResult { + RecordKey::new(BTreeSet::new(), names.into_iter().collect()) + } + + pub fn is_empty(&self) -> bool { + self.positions.is_empty() && self.names.is_empty() + } +} + #[cfg_attr(feature = "python", pyo3::pyclass(eq, eq_int, rename_all = "SCREAMING_SNAKE_CASE"))] #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum TraversalOrder { @@ -108,9 +174,11 @@ impl Quantifier { pub const ONE_OR_MORE: Quantifier = Quantifier { kind: QKind::OneOrMore, n: 0 }; pub const ZERO_OR_MORE: Quantifier = Quantifier { kind: QKind::ZeroOrMore, n: 0 }; + /// Exactly `n` occurrences, `n ≥ 0`: `{1}` is equivalent to no quantifier, + /// `{0}` to zero occurrences (an empty match); only a negative `n` is rejected. pub fn exactly(n: i64) -> CoreResult { - if n < 2 { - return Err(format!("EXACTLY requires n >= 2, got: {n}").into()); + if n < 0 { + return Err(format!("EXACTLY requires n >= 0, got: {n}").into()); } Ok(Quantifier { kind: QKind::Exactly, n }) } @@ -646,7 +714,8 @@ pub struct ActionSpec { pub providers: Vec, pub anchor_pos: Option, pub split_delimiter: Option, - pub key_positions: BTreeSet, + /// The key K of a `CONCAT`/`JOIN` action (empty for the other operations). + pub key: RecordKey, pub inherited: bool, } @@ -657,14 +726,22 @@ impl ActionSpec { providers: Vec, anchor_pos: Option, split_delimiter: Option, - key_positions: BTreeSet, + key: RecordKey, inherited: bool, ) -> CoreResult { + let is_record_op = matches!( + operation_type, + OperationType::Rec | OperationType::Concat | OperationType::Join + ); for p in &providers { if let Some(ctx) = &p.context_literal { let is_const_avp = ctx.const_value.is_some(); - if operation_type == OperationType::Join { - return Err("JOIN action does not allow context literals".into()); + if matches!(operation_type, OperationType::Concat | OperationType::Join) { + return Err(format!( + "{} action does not allow context literals", + operation_type.rtl_name() + ) + .into()); } if operation_type == OperationType::Rec && !is_const_avp && ctx.ty != ItemType::Value { return Err(format!( @@ -682,11 +759,10 @@ impl ActionSpec { } } else { let kind = p.target_item_kind.unwrap_or(CellKind::Unrestricted); - if (operation_type == OperationType::Rec || operation_type == OperationType::Join) - && kind != CellKind::Val - { + if is_record_op && kind != CellKind::Val { return Err(format!( - "{operation_type:?} action requires a VAL provider, got {kind:?}" + "{} action requires a VAL provider, got {kind:?}", + operation_type.rtl_name() ) .into()); } @@ -703,11 +779,16 @@ impl ActionSpec { providers, anchor_pos, split_delimiter, - key_positions, + key, inherited, }) } + /// The positional part of the key (kept for callers written against 0.5.x). + pub fn key_positions(&self) -> &BTreeSet { + &self.key.positions + } + pub fn as_inherited(&self) -> ActionSpec { if self.inherited { self.clone() diff --git a/src/tests.rs b/src/tests.rs index e4d18dc..29b27ba 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -131,7 +131,9 @@ fn end_to_end_match_and_interpret() { .unwrap(); assert_eq!(atomic.span, (0, "IKT".len())); - let rs = interpret(&InterpreterCfg::default(), &syntax, &sem, None).unwrap(); + let out = interpret(&InterpreterCfg::default(), &syntax, &sem, None).unwrap(); + assert!(out.diagnostics.is_empty()); + let rs = out.recordset; assert_eq!(rs.schema.attributes, vec!["ND", "AIRLINE", "AIRPORT", "MON"]); assert_eq!(rs.records.len(), 4); diff --git a/src/util.rs b/src/util.rs index 922b7c4..3fca01f 100644 --- a/src/util.rs +++ b/src/util.rs @@ -34,6 +34,17 @@ impl From for CoreErr { } } +impl CoreErr { + /// The error text (for `Py` errors, the Python exception's `str`). + pub fn text(&self) -> String { + match self { + CoreErr::Msg(m) => m.clone(), + #[cfg(feature = "python")] + CoreErr::Py(e) => e.to_string(), + } + } +} + pub type CoreResult = Result; fn cache() -> &'static Mutex>> { diff --git a/tests/atp_patterns.py b/tests/atp_patterns.py index 4048034..0047fde 100644 --- a/tests/atp_patterns.py +++ b/tests/atp_patterns.py @@ -107,7 +107,7 @@ def pattern_015(): def pattern_016(): RIGHT_OF = ItemFilterConditionSpec.right_of() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, 1)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, 1)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(AtomicContentSpec.val())))) def pattern_017(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) @@ -141,7 +141,7 @@ def pattern_023(): SAME_SUBROW = ItemFilterConditionSpec.same_subrow() RIGHT_OF = ItemFilterConditionSpec.right_of() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.exactly(3), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(AtomicContentSpec.attr(ActionSpec.suffix("", ProviderSpec.any(RIGHT_OF, 1, traversal_order=TraversalOrder.ROW_MAJOR)))), CellPattern.of(AtomicContentSpec.aux()), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.exactly(3), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(AtomicContentSpec.attr(ActionSpec.suffix("", ProviderSpec.any(RIGHT_OF, 1, traversal_order=TraversalOrder.ROW_MAJOR)))), CellPattern.of(AtomicContentSpec.aux()), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) def pattern_024(): BELOW = ItemFilterConditionSpec.below() @@ -152,7 +152,7 @@ def pattern_025(): RIGHT_OF = ItemFilterConditionSpec.right_of() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) SUBROW_AFTER_ANCHOR = ItemFilterConditionSpec.and_(FilterTerm.right_of(), FilterTerm.col_range(2, UNBOUNDED)) - return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.suffix(SEP, ProviderSpec.any(RIGHT_OF, 1)), ActionSpec.rec(ProviderSpec.val(SUBROW_AFTER_ANCHOR, UNBOUNDED), split_delimiter=SEP), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.suffix(SEP, ProviderSpec.any(RIGHT_OF, 1)), ActionSpec.rec(ProviderSpec.val(SUBROW_AFTER_ANCHOR, UNBOUNDED), split_delimiter=SEP), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) def pattern_026(): SAME_SUBTABLE_COL2 = ItemFilterConditionSpec.and_(FilterTerm.same_subtable(), FilterTerm.col_exact(2)) @@ -189,7 +189,7 @@ def pattern_032(): def pattern_033(): SAME_SUBROW = ItemFilterConditionSpec.same_subrow() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) def pattern_034(): BELOW = ItemFilterConditionSpec.below() @@ -266,13 +266,13 @@ def pattern_046(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) SAME_SUBROW = ItemFilterConditionSpec.same_subrow() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.attr()), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.attr()), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) def pattern_047(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) SAME_SUBROW = ItemFilterConditionSpec.same_subrow() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(Quantifier.one_or_more(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val())))) def pattern_048(): BLANK = CellMatchCondition(CellPredicate.blank()) @@ -292,7 +292,7 @@ def pattern_050(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) SAME_SUBROW = ItemFilterConditionSpec.same_subrow() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.attr()), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(""), ActionSpec.rec(ProviderSpec.val(SAME_SUBROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key=0))), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.attr()), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW))))))) def pattern_051(): SAME_COL = ItemFilterConditionSpec.same_col() @@ -313,7 +313,7 @@ def pattern_053(): BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) ABOVE = ItemFilterConditionSpec.above() SAME_SUBROW = ItemFilterConditionSpec.same_subrow() - return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(CellPattern.skip(), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.aux())), RowPattern.of(Quantifier.one_or_more(), SubrowPattern.of(CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_ROW, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, 1), key_positions=0), ActionSpec.avp("ID")))), SubrowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.attr(ActionSpec.prefix("_", ProviderSpec.any(ABOVE, 1)))), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW)))))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(CellPattern.skip(), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.aux())), RowPattern.of(Quantifier.one_or_more(), SubrowPattern.of(CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_ROW, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, 1), key=0), ActionSpec.avp("ID")))), SubrowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.attr(ActionSpec.prefix("_", ProviderSpec.any(ABOVE, 1)))), CellPattern.of(AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW)))))))) def pattern_054(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) @@ -412,7 +412,7 @@ def pattern_069(): ROW_TAG2 = ItemFilterConditionSpec.and_(FilterTerm.same_row(), FilterTerm.tagged("#2")) avpSR = ActionSpec.avp(ProviderSpec.attr(SAME_SUBROW)) recBW = ActionSpec.rec(ProviderSpec.val(BELOW, UNBOUNDED)) - return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one(), SubrowPattern.of(Quantifier.zero_or_more(), CellPattern.of(AtomicContentSpec.attr()), CellPattern.of(AtomicContentSpec.val_tagged("#1", avpSR, recBW, ActionSpec.join(ProviderSpec.val(ROW_TAG1, UNBOUNDED)))), CellPattern.of(AtomicContentSpec.val_tagged("#2", avpSR, recBW, ActionSpec.join(ProviderSpec.val(ROW_TAG2, UNBOUNDED)))))), RowPattern.of(Quantifier.zero_or_more(), SubrowPattern.of(Quantifier.zero_or_more(), CellPattern.of(AtomicContentSpec.attr()), CellPattern.of(Quantifier.exactly(2), AtomicContentSpec.val(avpSR)))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one(), SubrowPattern.of(Quantifier.zero_or_more(), CellPattern.of(AtomicContentSpec.attr()), CellPattern.of(AtomicContentSpec.val_tagged("#1", avpSR, recBW, ActionSpec.concat(ProviderSpec.val(ROW_TAG1, UNBOUNDED)))), CellPattern.of(AtomicContentSpec.val_tagged("#2", avpSR, recBW, ActionSpec.concat(ProviderSpec.val(ROW_TAG2, UNBOUNDED)))))), RowPattern.of(Quantifier.zero_or_more(), SubrowPattern.of(Quantifier.zero_or_more(), CellPattern.of(AtomicContentSpec.attr()), CellPattern.of(Quantifier.exactly(2), AtomicContentSpec.val(avpSR)))))) def pattern_070(): BLANK = CellMatchCondition(CellPredicate.blank()) @@ -599,7 +599,7 @@ def pattern_094(): sameCol = ItemFilterConditionSpec.same_col() rowColRightStr = ItemFilterConditionSpec.and_(FilterTerm.same_row(), FilterTerm.col_range(1, UNBOUNDED), FilterTerm.same_str()) colRec = ActionSpec.rec(ProviderSpec.val(sameCol, UNBOUNDED)) - rowJoin = ActionSpec.join(ProviderSpec.val(rowColRightStr, UNBOUNDED), key_positions=0) + rowJoin = ActionSpec.concat(ProviderSpec.val(rowColRightStr, UNBOUNDED), key=0) headerCell = CellPattern.of(NOT_BLANK, Quantifier.one_or_more(), AtomicContentSpec.val(colRec, rowJoin)) optBlank = CellPattern.of(BLANK, Quantifier.zero_or_one(), None) dataCell = CellPattern.of(NOT_BLANK, Quantifier.one_or_more(), AtomicContentSpec.val()) @@ -627,13 +627,13 @@ def pattern_096(): def pattern_097(): RIGHT_OF = ItemFilterConditionSpec.right_of() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) - return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions={0, 1}))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key={0, 1}))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) def pattern_098(): RIGHT_OF = ItemFilterConditionSpec.right_of() BELOW_STR = ItemFilterConditionSpec.and_(FilterTerm.below(), FilterTerm.same_str()) SAME_COL = ItemFilterConditionSpec.same_col() - return TablePattern.of(SubtablePattern.of(RowPattern.of(CellPattern.skip(), CellPattern.skip(), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.attr())), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, UNBOUNDED)), ActionSpec.join(ProviderSpec.val(BELOW_STR, UNBOUNDED), key_positions={0, 1}))), CellPattern.of(AtomicContentSpec.val()), CellPattern.of(Quantifier.exactly(2), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_COL)))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) + return TablePattern.of(SubtablePattern.of(RowPattern.of(CellPattern.skip(), CellPattern.skip(), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.attr())), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(RIGHT_OF, UNBOUNDED)), ActionSpec.concat(ProviderSpec.val(BELOW_STR, UNBOUNDED), key={0, 1, 2, 3}))), CellPattern.of(AtomicContentSpec.val()), CellPattern.of(Quantifier.exactly(2), AtomicContentSpec.val(ActionSpec.avp(ProviderSpec.attr(SAME_COL)))), CellPattern.of(Quantifier.one_or_more(), AtomicContentSpec.val())))) def pattern_099(): CL_P0 = ItemFilterConditionSpec.and_(FilterTerm.same_cell(), FilterTerm.pos_exact(0)) diff --git a/tests/task_runner.py b/tests/task_runner.py index 8b0930c..b2b062d 100644 --- a/tests/task_runner.py +++ b/tests/task_runner.py @@ -152,11 +152,10 @@ def run_task_variant(task_id: str, variant: int, pattern) -> None: itm = AtpMatcher.match(pattern, syntax) assert itm is not None, f"Task {task_id} pattern did not match variant {variant}" - actual = pattern.transform( - TableInterpreter() - .with_strategy(SchemaConstructionStrategy.RECORD_FIRST) - .interpret(itm) - ) + interpreter = TableInterpreter().with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + actual = pattern.transform(interpreter.interpret(itm)) + # The task corpus must not skip a single CONCAT/JOIN action (RtlTaskBase). + assert interpreter.diagnostics() == [], [str(d) for d in interpreter.diagnostics()] opts = load_match_options(task_id) expected_path = task_dir / f"expected_{variant}.csv" diff --git a/tests/test_dsl.py b/tests/test_dsl.py index a21d7cb..d85384b 100644 --- a/tests/test_dsl.py +++ b/tests/test_dsl.py @@ -129,12 +129,12 @@ def test_task015(): def test_task016(): assert_mirrors( r""" - [ [VAL : RT->REC, BW&STR*->JOIN(0)] [VAL] ]+ + [ [VAL : RT->REC, BW&STR*->CONCAT(0)] [VAL] ]+ """, table( subtable( row( - cell(VAL, rec(RT), join(0, BW.and_(STR).unbounded())), + cell(VAL, rec(RT), concat(0, BW.and_(STR).unbounded())), cell(VAL), ).one_or_more() ) @@ -163,12 +163,12 @@ def test_task022(): def test_task023(): assert_mirrors( r""" - { [ [VAL : ''->AVP, SR*->REC, BW&STR*->JOIN(0)] [ATTR : RT->SUFFIX] [AUX] [VAL : SR->AVP] ]{3} }+ + { [ [VAL : ''->AVP, SR*->REC, BW&STR*->CONCAT(0)] [ATTR : RT->SUFFIX] [AUX] [VAL : SR->AVP] ]{3} }+ """, table( subtable( row( - cell(VAL, avp(""), rec(SR.unbounded()), join(0, BW.and_(STR).unbounded())), + cell(VAL, avp(""), rec(SR.unbounded()), concat(0, BW.and_(STR).unbounded())), cell(ATTR, suffix(RT)), cell(AUX), cell(VAL, avp(SR)), @@ -181,7 +181,7 @@ def test_task023(): def test_task025(): assert_mirrors( r""" - [ [VAL : RT->SUFFIX('/'), RT&C+2..*->REC('/'), BW&STR*->JOIN(0)] [VAL]+ ]+ + [ [VAL : RT->SUFFIX('/'), RT&C+2..*->REC('/'), BW&STR*->CONCAT(0)] [VAL]+ ]+ """, table( subtable( @@ -190,7 +190,7 @@ def test_task025(): VAL, suffix("/", RT), rec_split("/", RT.and_(CrelFrom(2)).unbounded()), - join(0, BW.and_(STR).unbounded()), + concat(0, BW.and_(STR).unbounded()), ), cell(VAL).one_or_more(), ).one_or_more() @@ -278,7 +278,7 @@ def test_task068(): def test_task069(): assert_mirrors( r""" - [ BW*->REC { [ATTR] [VAL#'1': ROW&#'1'*->JOIN][VAL#'2': ROW&#'2'*->JOIN] }* ] + [ BW*->REC { [ATTR] [VAL#'1': ROW&#'1'*->CONCAT][VAL#'2': ROW&#'2'*->CONCAT] }* ] """, table( subtable( @@ -286,8 +286,8 @@ def test_task069(): acts(rec(BW.unbounded())), subrow( cell(ATTR), - cell(val(join(ROW.and_(tag("1")).unbounded())).tagged("1")), - cell(val(join(ROW.and_(tag("2")).unbounded())).tagged("2")), + cell(val(concat(ROW.and_(tag("1")).unbounded())).tagged("1")), + cell(val(concat(ROW.and_(tag("2")).unbounded())).tagged("2")), ).zero_or_more(), ) ) diff --git a/tests/test_join_concat.py b/tests/test_join_concat.py new file mode 100644 index 0000000..eaa3d30 --- /dev/null +++ b/tests/test_join_concat.py @@ -0,0 +1,347 @@ +"""CONCAT (fold) vs JOIN (record product), named keys, diagnostics, strict +preconditions, zero-width subrows and the {0}/{1} quantifiers — parity with +jRegTab 0.6.0–0.7.1 (ports of TableInterpreterMultiRecordTest, the named-key +compiler checks and the SyntaxMatcher zero-width cases).""" + +import pytest + +from pyregtab import ( + ActionSpec, + AtpMatcher, + OperationType, + Quantifier, + RecordKey, + RtlCompileError, + RtlCompiler, + SchemaConstructionStrategy, + TableInterpreter, + TableSyntax, + AtpToRtlSerializer, +) +from pyregtab.dsl import BW, C, COL, RT, STR, VAL, cell, concat, join, rec, row, subtable, table +from pyregtab import FilterTerm, ProviderSpec, ItemFilterConditionSpec as F + + +def make_table(rows): + t = TableSyntax(len(rows), max(len(r) for r in rows)) + for r, line in enumerate(rows): + for c, v in enumerate(line): + t.cell(r, c).set_text(v) + return t + + +def run(interpreter, rtl, rows): + pattern = RtlCompiler.compile(rtl) + itm = AtpMatcher.match(pattern, make_table(rows)) + assert itm is not None, f"pattern did not match:\n{rtl}" + return pattern.transform(interpreter.interpret(itm)) + + +def rows_of(rs, *attrs): + return [[rec[a] for a in attrs] for rec in rs.records] + + +# id | x | y / a;b | 1 | 2 / c | 3 | 4 — explode × stack, six records. +EXPLODE_STACK = [["id", "x", "y"], ["a;b", "1", "2"], ["c", "3", "4"]] + +JOIN_PRODUCT = """ +[ [ATTR] [VAL: 'var'->AVP]+ ] +[ [(VAL: COL->AVP, ()->REC, RT*->JOIN){';'}] [VAL: 'value'->AVP, COL->REC]+ ]+ +""" + +# k | v / A | 5 / A | 7 — the two rows share the named attribute v: a CONCAT conflict. +CONFLICT = [["k", "v"], ["A", "5"], ["A", "7"]] + +CONCAT_CONFLICT = """ +[ [ATTR]+ ] +[ [VAL: COL->AVP, RT->REC, BW&STR*->CONCAT(0)] [VAL: COL->AVP] ]+ +""" + +PQ = [["p", "q"]] + + +# ------------------------------------------------------------------ JOIN: record product + + +def test_join_product_record_first(): + interpreter = TableInterpreter() + rs = run(interpreter, JOIN_PRODUCT, EXPLODE_STACK) + + assert rs.schema.attributes == ["id", "value", "var"] + assert len(rs) == 6 + assert rows_of(rs, "id", "var", "value") == [ + ["a", "x", "1"], ["a", "y", "2"], + ["b", "x", "1"], ["b", "y", "2"], + ["c", "x", "3"], ["c", "y", "4"], + ] + assert interpreter.diagnostics() == [] + + +def test_join_product_position_first_same_records(): + interpreter = TableInterpreter().with_strategy(SchemaConstructionStrategy.POSITION_FIRST) + rs = run(interpreter, JOIN_PRODUCT, EXPLODE_STACK) + + assert rs.schema.attributes == ["id", "value", "var"] + assert len(rs) == 6 + assert rows_of(rs, "id", "var", "value")[3] == ["b", "y", "2"] + + +def test_join_equi_key_by_position_and_by_name_agree(): + # k | v | k | u / X | 5 | X | kg / Y | 8 | Y | pc (conformance: join_equi_key / join_named_key) + rows = [["k", "v", "k", "u"], ["X", "5", "X", "kg"], ["Y", "8", "Y", "pc"]] + head = "[ [ATTR]+ ]\n" + by_pos = head + "[ [VAL: COL->AVP, RT->REC, C2*->JOIN(0)] [VAL: COL->AVP] [VAL: COL->AVP, RT->REC] [VAL: COL->AVP] ]+" + by_name = head + "[ [VAL: COL->AVP, RT->REC, C2*->JOIN('k')] [VAL: COL->AVP] [VAL: COL->AVP, RT->REC] [VAL: COL->AVP] ]+" + outs = [run(TableInterpreter(), rtl, rows) for rtl in (by_pos, by_name)] + for rs in outs: + assert rs.schema.attributes == ["k", "v", "u"] + assert rows_of(rs, "k", "v", "u") == [["X", "5", "kg"], ["Y", "8", "pc"]] + + +# ------------------------------------------------------------------ CONCAT: fold, conflicts, diagnostics + + +def test_concat_folds_rows_with_the_same_key(): + # book | 5 / book | 6 / book | 7 -> book,5,6,7 (task 016 shape) + rs = run( + TableInterpreter(), + "[ [VAL : RT->REC, BW&STR*->CONCAT(0)] [VAL] ]+", + [["book", "5"], ["book", "6"], ["book", "7"], ["cat", "2"], ["cat", "3"]], + ) + assert [list(r.values().values()) for r in rs.records] == [ + ["book", "5", "6", "7"], + ["cat", "2", "3", None], + ] + + +def test_concat_conflict_no_effect_both_records_survive_diagnostic_reported(): + interpreter = TableInterpreter() + rs = run(interpreter, CONCAT_CONFLICT, CONFLICT) + + assert len(rs) == 2, "neither row is folded, nothing is lost silently" + assert rows_of(rs, "k", "v") == [["A", "5"], ["A", "7"]] + diags = interpreter.diagnostics() + assert len(diags) == 1 + d = diags[0] + assert d.operation == "CONCAT" + assert "'v'" in d.message + assert d.anchor.str == "A" and d.anchor.cell.row == 1 + assert str(d).startswith("CONCAT skipped at CellDerivedItem[") + + +def test_concat_conflict_strict_preconditions_raises(): + interpreter = TableInterpreter().with_strict_preconditions(True) + with pytest.raises(RuntimeError, match="CONCAT"): + run(interpreter, CONCAT_CONFLICT, CONFLICT) + + +def test_concat_named_key_independent_of_field_order(): + # Repeated named fields to the RIGHT of the varying column: CONCAT(0,1,'A','B') + # is the same key as the positional CONCAT(0,1,3,4) (conformance: concat_named_key). + rows = [ + ["", "", "C", "A", "B"], + ["k1", "k11", "c1", "a1", "b1"], + ["k1", "k11", "c2", "a1", "b1"], + ["k2", "k22", "c3", "a2", "b2"], + ["k2", "k22", "c4", "a2", "b2"], + ] + head = "[ [] [] [ATTR]+ ]\n" + named = head + "[ [VAL: RT*->REC, (BW&STR)*->CONCAT(0, 1, 'A', 'B')] [VAL] [VAL] [VAL: COL->AVP]{2} ]+" + positional = head + "[ [VAL: RT*->REC, (BW&STR)*->CONCAT(0, 1, 3, 4)] [VAL] [VAL] [VAL: COL->AVP]{2} ]+" + for rtl in (named, positional): + interpreter = TableInterpreter().with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + rs = run(interpreter, rtl, rows) + assert [list(r.values().values()) for r in rs.records] == [ + ["k1", "k11", "c1", "a1", "b1", "c2"], + ["k2", "k22", "c3", "a2", "b2", "c4"], + ] + assert interpreter.diagnostics() == [] + + +# ------------------------------------------------------------------ anchors without records + + +def test_explicit_join_anchor_without_record_reported(): + interpreter = TableInterpreter() + rs = run(interpreter, "[ [VAL: RT*->JOIN] [VAL] ]", PQ) + + assert len(rs) == 0, "the anchor has no record, so nothing is extracted" + diags = interpreter.diagnostics() + assert len(diags) == 1 + assert diags[0].operation == "JOIN" + assert diags[0].anchor.str == "p" + assert "REC missing" in diags[0].message + + +def test_explicit_join_with_own_rec_not_reported(): + interpreter = TableInterpreter() + rs = run(interpreter, JOIN_PRODUCT, EXPLODE_STACK) + assert len(rs) == 6 + assert interpreter.diagnostics() == [], "()->REC gives the token its record" + + +def test_inherited_join_anchor_without_record_not_reported(): + interpreter = TableInterpreter() + run(interpreter, "[ RT*->JOIN [VAL] [VAL] ]", PQ) + assert interpreter.diagnostics() == [], "row-level JOIN reaches cells without records routinely" + + +def test_explicit_concat_concatenated_away_anchors_not_reported(): + interpreter = TableInterpreter() + rs = run( + interpreter, + "[ [VAL: RT->REC, BW&STR*->CONCAT(0)] [VAL] ]+", + [["A", "5"], ["A", "7"], ["A", "9"]], + ) + assert len(rs) == 1, "the three rows fold into one record" + assert interpreter.diagnostics() == [], "the CONCAT of a folded-away anchor is not an anchor without REC" + + +def test_explicit_join_no_provided_item_has_record_reported(): + interpreter = TableInterpreter() + rs = run(interpreter, "[ [VAL: ()->REC, RT*->JOIN] [VAL] ]", PQ) + + assert len(rs) == 1, "the anchor keeps its single-field record" + diags = interpreter.diagnostics() + assert len(diags) == 1 and diags[0].operation == "JOIN" + assert "provider side" in diags[0].message + + +def test_explicit_join_anchor_without_record_strict_preconditions_raises(): + interpreter = TableInterpreter().with_strict_preconditions(True) + with pytest.raises(RuntimeError, match="REC missing"): + run(interpreter, "[ [VAL: RT*->JOIN] [VAL] ]", PQ) + + +def test_diagnostics_are_reset_per_interpretation(): + interpreter = TableInterpreter() + run(interpreter, CONCAT_CONFLICT, CONFLICT) + assert len(interpreter.diagnostics()) == 1 + run(interpreter, JOIN_PRODUCT, EXPLODE_STACK) + assert interpreter.diagnostics() == [] + + +# ------------------------------------------------------------------ RecordKey and the spec API + + +def test_record_key_api(): + k = RecordKey.of({1, 0}, {"B", "A"}) + assert k.positions == {0, 1} and k.names == {"A", "B"} + assert RecordKey.of([0, 1]) == RecordKey({0, 1}) + assert RecordKey.of(names=["A"]) == RecordKey(names=["A"]) + assert RecordKey.empty().is_empty() and RecordKey() == RecordKey.empty() + assert "positions=[0, 1]" in repr(k) + with pytest.raises(Exception): + RecordKey.of(names=[" "]) + with pytest.raises(Exception): + RecordKey.of([-1]) + + +def test_action_spec_concat_and_join_keys(): + prov = ProviderSpec.val(F.and_(FilterTerm.below(), FilterTerm.same_str()), cardinality=2**31) + a = ActionSpec.concat(prov, key=0) + assert a.operation_type == OperationType.CONCAT + assert a.key == RecordKey.of([0]) and a.key_positions == {0} + b = ActionSpec.concat(prov, key=(0, 1, "A", "B")) + assert b.key == RecordKey.of({0, 1}, {"A", "B"}) + c = ActionSpec.concat(prov, key="A") + assert c.key == RecordKey.of(names=["A"]) + # legacy spelling still works + d = ActionSpec.join(prov, key_positions={0, 1}) + assert d.operation_type == OperationType.JOIN and d.key == RecordKey.of([0, 1]) + e = ActionSpec(OperationType.JOIN, providers=[prov], key=RecordKey.of(names=["k"])) + assert e.key == RecordKey.of(names=["k"]) + + +def test_compiler_and_serializer_named_keys(): + canonical = AtpToRtlSerializer.serialize( + RtlCompiler.compile("[ [VAL: RT*->REC, (BW&STR)*->CONCAT(\"B\", 'A', 1, 0)] [VAL] ]+") + ) + assert "CONCAT(0, 1, 'A', 'B')" in canonical + assert "JOIN('it''s')" in AtpToRtlSerializer.serialize( + RtlCompiler.compile("[ [VAL: RT*->REC, RT*->JOIN('it''s')] [VAL] ]+") + ) + with pytest.raises(RtlCompileError): + RtlCompiler.compile("[ [VAL : RT->REC, BW*->CONCAT('')] ]") + + +def test_dsl_concat_and_join_mirror_the_compiler(): + def canon(p): + return AtpToRtlSerializer.serialize(p) + + compiled = RtlCompiler.compile("[ [VAL : RT->REC, BW&STR*->CONCAT(0, 'A')] [VAL] ]+") + built = table(subtable(row( + cell(VAL, rec(RT), concat((0, "A"), BW.and_(STR).unbounded())), + cell(VAL), + ).one_or_more())) + assert canon(built) == canon(compiled) + + compiled = RtlCompiler.compile("[ [VAL: COL->AVP, RT->REC, C2*->JOIN('k')] [VAL: COL->AVP] ]+") + built = table(subtable(row( + cell(VAL, __import__("pyregtab.dsl", fromlist=["avp"]).avp(COL), rec(RT), join("k", C(2).unbounded())), + cell(VAL, __import__("pyregtab.dsl", fromlist=["avp"]).avp(COL)), + ).one_or_more())) + assert canon(built) == canon(compiled) + + +# ------------------------------------------------------------------ zero-width subrows and {n}, n < 2 + + +ZERO_WIDTH_ROWS = [["A", "B"], ["x", "1"], ["y", "2"]] + + +@pytest.mark.parametrize( + "body", + [ + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* } [VAL: COL->AVP]", # mid + "[VAL: COL->AVP, ROW*->REC] [VAL: COL->AVP] { [BLANK]* }", # tail + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* }+ [VAL: COL->AVP] { [BLANK]* }+", + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* }* [VAL: COL->AVP] { [BLANK]* }*", + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* }{1} [VAL: COL->AVP]", + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* }{0} [VAL: COL->AVP]", + "[VAL: COL->AVP, ROW*->REC] { [BLANK]* }{3} [VAL: COL->AVP]", + ], +) +def test_zero_width_subrow_matches_the_empty_sequence(body): + rs = run(TableInterpreter(), f"[ [ATTR]+ ]\n[ {body} ]+", ZERO_WIDTH_ROWS) + assert rows_of(rs, "A", "B") == [["x", "1"], ["y", "2"]] + + +def test_empty_subrow_is_not_materialized_in_the_syntax_layer(): + t = make_table(ZERO_WIDTH_ROWS) + pattern = RtlCompiler.compile("[ [ATTR]+ ]\n[ [VAL: COL->AVP, ROW*->REC] { [BLANK]* } [VAL: COL->AVP] ]+") + assert AtpMatcher.match(pattern, t) is not None + # the empty subrow covers no cells: the header row keeps its single subrow, + # the data rows are split into the two implicit subrows around it, and + # every subrow is non-empty + assert [len(t.row(r).subrows()) for r in range(3)] == [1, 2, 2] + for r in range(3): + for sr in t.row(r).subrows(): + assert sr.col_end >= sr.col_start + + +def test_empty_subtable_all_rows_optional(): + rs = run( + TableInterpreter(), + "{ [ [BLANK]+ ]* } [ [ATTR]+ ] [ [VAL: COL->AVP, ROW*->REC] [VAL: COL->AVP] ]+", + ZERO_WIDTH_ROWS, + ) + assert rows_of(rs, "A", "B") == [["x", "1"], ["y", "2"]] + + +def test_quantifier_small_n(): + assert Quantifier.exactly(1).min() == 1 and Quantifier.exactly(1).max() == 1 + assert Quantifier.exactly(0).min() == 0 and Quantifier.exactly(0).max() == 0 + with pytest.raises(Exception): + Quantifier.exactly(-1) + # {1} is equivalent to no quantifier, {0} matches nothing + assert AtpToRtlSerializer.serialize(RtlCompiler.compile("[ [VAL]{1} [VAL]{0} { [BLANK]* }{1} ]")) == \ + "[ [ VAL ]{1} [ VAL ]{0} { [ BLANK ]* }{1} ]" + with pytest.raises(RtlCompileError): + RtlCompiler.compile("[ [VAL]{-1} ]") + err = None + try: + RtlCompiler.compile("[ [VAL]{99999999999} ]") + except RtlCompileError as e: + err = e + assert err is not None and (err.line, err.col) == (1, 8) # the INT token, as in Java diff --git a/tests/test_semantic_conformance.py b/tests/test_semantic_conformance.py index d20c708..eff5aba 100644 --- a/tests/test_semantic_conformance.py +++ b/tests/test_semantic_conformance.py @@ -76,11 +76,11 @@ def test_semantic_case(case): itm = AtpMatcher.match(pattern, syntax) assert itm is not None, f"pattern did not match {case.name}/{INPUT}" - actual = pattern.transform( - TableInterpreter() - .with_strategy(SchemaConstructionStrategy.RECORD_FIRST) - .interpret(itm) - ) + interpreter = TableInterpreter().with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + actual = pattern.transform(interpreter.interpret(itm)) + # A semantic case must not skip a single CONCAT/JOIN action + # (RtlSemanticConformanceTest asserts empty diagnostics). + assert interpreter.diagnostics() == [], [str(d) for d in interpreter.diagnostics()] opts = load_case_options(case) expected_path = case / EXPECTED diff --git a/tools/translate_atp.py b/tools/translate_atp.py index a649fd2..7bc4a44 100644 --- a/tools/translate_atp.py +++ b/tools/translate_atp.py @@ -299,14 +299,18 @@ def is_str(e: Expr) -> bool: kw.append(f"split_delimiter={head.py}") parts = [a.py for a in rest] + kw return Expr("call", f"ActionSpec.rec({', '.join(parts)})", "ActionSpec") - if leaf == "join": + if leaf in ("concat", "join"): + # Java overloads: (providers...), (int, providers...), + # (Set, providers...), (String keyName, providers...), + # (RecordKey, providers...) -> Python keyword `key=` kw = [] rest = args - if args and args[0].kind in ("int", "set"): - kw.append(f"key_positions={args[0].py}") + if args and (args[0].kind in ("int", "set") or is_str(args[0]) + or args[0].root == "RecordKey"): + kw.append(f"key={args[0].py}") rest = args[1:] parts = [a.py for a in rest] + kw - return Expr("call", f"ActionSpec.join({', '.join(parts)})", "ActionSpec") + return Expr("call", f"ActionSpec.{leaf}({', '.join(parts)})", "ActionSpec") if leaf in ("fill", "prefix", "suffix", "avp"): return Expr("call", f"ActionSpec.{leaf}({join(args)})", "ActionSpec") raise SyntaxError(f"unknown ActionSpec factory: {leaf}")