From 417daff194977521c3d36e3b98252a51cf854889 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 22 Nov 2026 19:07:09 +0000 Subject: [PATCH 1/5] perf(executor): division on term ids (#11) / in BIND, filters and orders under --executor ids: a quotient of integers or decimals is the evaluator's (spareval::decimal_divide), of doubles a double, by zero an error. A decimal quotient that does not end within 18 fraction digits gives up to the evaluator. --- CHANGELOG.md | 1 + crates/tripletrail/src/direct.rs | 7 +++--- crates/tripletrail/src/idtree.rs | 4 ++-- crates/tripletrail/src/idvalue.rs | 32 ++++++++++++++++++++++++-- crates/tripletrail/tests/id_rows.rs | 35 ++++++++++++++++++++++++----- docs/store.md | 12 ++++++---- 6 files changed, 75 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ed60d..c61def1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ versions may change the store format and the API; the release notes say how to u - More of one query runs on several cores (#4): the sorted joins of a basic graph pattern (chunks of rows, each walking the index on its own), `GROUP BY` of many rows (split by the hash of the key) and the `SUM`, `AVG`, `MIN`, `MAX` and `SAMPLE` of the groups. The answers, and their order, are those of one core. On 2M orders and 4 cores, a join of three patterns grouped by country went from 500 to 220 ms, a count over three patterns from 480 to 190 ms, and 200,000 customers' sums from 420 to 260 ms. Rows to sort for a join are sorted as (key, row) pairs, which also helps one core (570 → 470 ms). `--query-threads` sizes the pool, as before; see [the server options](docs/server.md#main-options). - Under `--executor ids`, `BIND`s of strings and IRIs run on term ids too (#11): `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` with a constant pattern, each computed once per distinct input. On a production rule set of 55 queries, four more stay on ids, with the same answers; the one that rewrites five IRIs with `IRI(REPLACE(STR(?x), …))` takes 142 instead of 239 ms. See [the store](docs/store.md). - Under `--executor ids`, a `REPLACE` whose pattern or flags are expressions without variables (`REPLACE(STR(?x), STR(ex:), "")`) runs on term ids too (#11), computed once. On a production rule set of 55 queries, three more stay on ids, with the same answers: 1.07 s → 0.17 s, 120 → 25 ms and 32 → 12 ms. +- Under `--executor ids`, `/` runs on term ids too (#11), in `BIND`, filters and orders: a quotient of integers or decimals is the evaluator's (exact, as Jena's), of doubles a double, and division by zero an error. On 133k orders, `BIND(?q / 4 AS ?h) FILTER(?h > 20)` takes 33 ms instead of 191. A decimal quotient that doesn't end within 18 fraction digits (`1 / 3`) is still left to the evaluator. - Under `--executor ids`, `ORDER BY` an expression the id kernels compute (`DESC(?q * 3)`, `ABS(?x)`, `?d - 1`) runs on term ids too (#11), as a column of the computed values. On 133k orders, `ORDER BY DESC(?q * 3) ?o LIMIT 10` takes 15 ms instead of 117, and with a join and `LIMIT 100` 20 ms instead of 245. Rows with equal keys may come in another order than under `spareval`. ## 0.1.0 - 2026-11-21 diff --git a/crates/tripletrail/src/direct.rs b/crates/tripletrail/src/direct.rs index 9a41aaa..7c865b4 100644 --- a/crates/tripletrail/src/direct.rs +++ b/crates/tripletrail/src/direct.rs @@ -209,9 +209,10 @@ fn computable(e: &Expression, top: bool) -> bool { Expression::Variable(_) => true, Expression::Literal(_) => true, Expression::NamedNode(_) => top, - Expression::Add(a, b) | Expression::Subtract(a, b) | Expression::Multiply(a, b) => { - computable(a, false) && computable(b, false) - } + Expression::Add(a, b) + | Expression::Subtract(a, b) + | Expression::Multiply(a, b) + | Expression::Divide(a, b) => computable(a, false) && computable(b, false), Expression::UnaryMinus(a) => computable(a, false), Expression::FunctionCall( F::Abs diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index 6184703..bc106b0 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -1530,12 +1530,12 @@ fn lang_matches(tag: &str, range: &str) -> bool { } /// Whether `e` is made of what [`crate::idvalue::compile`] reads: variables, literals, -/// `+ - *` and the number and date functions it knows. +/// `+ - * /` and the number and date functions it knows. pub(crate) fn value_expression(e: &Expression) -> bool { use Expression as E; match e { E::Variable(_) | E::Literal(_) => true, - E::Add(a, b) | E::Subtract(a, b) | E::Multiply(a, b) => { + E::Add(a, b) | E::Subtract(a, b) | E::Multiply(a, b) | E::Divide(a, b) => { value_expression(a) && value_expression(b) } E::UnaryMinus(a) => value_expression(a), diff --git a/crates/tripletrail/src/idvalue.rs b/crates/tripletrail/src/idvalue.rs index c97d4d0..d5cdcb5 100644 --- a/crates/tripletrail/src/idvalue.rs +++ b/crates/tripletrail/src/idvalue.rs @@ -112,11 +112,12 @@ pub(crate) fn compile<'k>( let value = constant(l, snapshot)?; Box::new(move |_| Some(Ok(value))) } - E::Add(a, b) | E::Subtract(a, b) | E::Multiply(a, b) => { + E::Add(a, b) | E::Subtract(a, b) | E::Multiply(a, b) | E::Divide(a, b) => { let op = match e { E::Add(..) => Op::Add, E::Subtract(..) => Op::Subtract, - _ => Op::Multiply, + E::Multiply(..) => Op::Multiply, + _ => Op::Divide, }; let (a, b) = (compile(a, t, snapshot)?, compile(b, t, snapshot)?); Box::new(move |row| { @@ -124,6 +125,13 @@ pub(crate) fn compile<'k>( (Ok(x), Ok(y)) => (x, y), _ => return Some(Err(())), }; + // A decimal or integer divided by zero: an error, as the evaluator's. + if matches!(op, Op::Divide) + && matches!(y, Value::Integer(0) | Value::Decimal(0)) + && !matches!(x, Value::Double(_)) + { + return Some(Err(())); + } arithmetic(op, x, y).map(Ok) }) } @@ -310,6 +318,7 @@ enum Op { Add, Subtract, Multiply, + Divide, } #[derive(Clone, Copy)] @@ -336,13 +345,31 @@ fn arithmetic(op: Op, x: Value, y: Value) -> Option { Op::Add => a + b, Op::Subtract => a - b, Op::Multiply => a * b, + Op::Divide => a / b, }; r.is_finite().then_some(Value::Double(r)) } + // A quotient of integers or decimals is a decimal, the evaluator's + // (`spareval::decimal_divide`, as Jena: exact, else 24 fraction digits, which the + // evaluator writes as a literal of its own and the kernels give up on). By zero is an + // error, which the caller tells apart. + (Value::Integer(_) | Value::Decimal(_), Value::Integer(_) | Value::Decimal(_)) + if matches!(op, Op::Divide) => + { + let (a, b) = (decimal(x)?, decimal(y)?); + let dec = |d: i128| oxsdatatypes::Decimal::from_be_bytes(d.to_be_bytes()); + match spareval::decimal_divide(dec(a), dec(b))? { + spareval::ExpressionTerm::DecimalLiteral(q) => { + Some(Value::Decimal(i128::from_be_bytes(q.to_be_bytes()))) + } + _ => None, + } + } (Value::Integer(a), Value::Integer(b)) => Some(Value::Integer(match op { Op::Add => a.checked_add(b)?, Op::Subtract => a.checked_sub(b)?, Op::Multiply => a.checked_mul(b)?, + Op::Divide => unreachable!(), })), (Value::Integer(_) | Value::Decimal(_), Value::Integer(_) | Value::Decimal(_)) => { let (a, b) = (decimal(x)?, decimal(y)?); @@ -350,6 +377,7 @@ fn arithmetic(op: Op, x: Value, y: Value) -> Option { Op::Add => a.checked_add(b)?, Op::Subtract => a.checked_sub(b)?, Op::Multiply => scaled_product(a, b)?, + Op::Divide => unreachable!(), })) } _ => None, diff --git a/crates/tripletrail/tests/id_rows.rs b/crates/tripletrail/tests/id_rows.rs index 4464e05..150a93b 100644 --- a/crates/tripletrail/tests/id_rows.rs +++ b/crates/tripletrail/tests/id_rows.rs @@ -412,6 +412,35 @@ const BINDS: &[(&str, bool)] = &[ "SELECT DISTINCT ?y WHERE { ?s ex:label ?l BIND(STR(?l) AS ?y) }", true, ), + // Division: of integers and decimals a decimal (exact, or the evaluator's 24 digits, which + // give up), of doubles a double, by zero an error; in filters and orders too. + ("SELECT ?s ?y WHERE { ?s ex:n ?x BIND(?x / 2 AS ?y) }", true), + ( + "SELECT ?s ?y WHERE { ?s ex:d ?x BIND(ROUND(?x * 1000) / 1000 AS ?y) }", + true, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:n ?x BIND(?x / 3 AS ?y) }", + false, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:n ?x BIND(?x / (?x - ?x) AS ?y) }", + true, + ), + ("SELECT ?s ?y WHERE { ?s ex:f ?x BIND(?x / 4 AS ?y) }", true), + ( + "SELECT ?s ?y WHERE { ?s ex:f ?x BIND(?x / 0 AS ?y) }", + false, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:d ?x ; ex:n ?n BIND(?x / ?n AS ?y) }", + false, + ), + ("SELECT ?s WHERE { ?s ex:n ?x FILTER(?x / 4 > 100) }", true), + ( + "SELECT ?s ?x WHERE { ?s ex:n ?x } ORDER BY DESC(?x / 8) ?s LIMIT 7", + true, + ), // `REPLACE` with a constant pattern (and flags), `STRLANG`, `STRDT`. ( "SELECT ?s ?i WHERE { ?s ex:p ?o BIND(IRI(REPLACE(STR(?o), \"example\", \"other\")) AS ?i) }", @@ -506,7 +535,7 @@ const BINDS: &[(&str, bool)] = &[ true, ), // Left to the evaluator: an order on a computed value, values the ids don't decide, a - // division, a string function the kernels don't have, an overflow. + // string function the kernels don't have, an overflow. ( "SELECT ?s ?y WHERE { ?s ex:n ?x BIND(?x * 2 AS ?y) } ORDER BY DESC(?y) ?s LIMIT 9", false, @@ -519,10 +548,6 @@ const BINDS: &[(&str, bool)] = &[ "SELECT ?s ?y WHERE { ?s ex:val ?x BIND(?x + 1 AS ?y) }", false, ), - ( - "SELECT ?s ?y WHERE { ?s ex:n ?x BIND(?x / 2 AS ?y) }", - false, - ), ( "SELECT ?s ?y WHERE { ?s ex:label ?l BIND(UCASE(?l) AS ?y) }", false, diff --git a/docs/store.md b/docs/store.md index 972cb74..a16a873 100644 --- a/docs/store.md +++ b/docs/store.md @@ -91,10 +91,14 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se order, so the same query can round differently on other data. The evaluator, and Fuseki, round after each addition in row order, so the last digit can differ: ten times `0.1` sums to `1` here and to `0.9999999999999999` there. An `AVG` of integers and decimals only (a decimal division) stays with the evaluator. - Comparisons and `+ - *` take inline doubles too (#11), with integers and decimals turned + Under `--executor ids`, `/` is computed too: a quotient of integers or decimals as the + evaluator's (`spareval::decimal_divide`, Jena's), of doubles a double, by zero an error + (`BIND(?q / 4 AS ?h) FILTER(?h > 20)` on 133k orders 191 → 33 ms). + Comparisons and `+ - * /` take inline doubles too (#11), with integers and decimals turned into doubles as the evaluator turns them, and `-0` equal to `0`. Dictionary literals (`1.5E2`), a double result that is not finite, overflow, decimal products with more than 18 - fraction digits (they are exact on 256 bits otherwise), dates + fraction digits (they are exact on 256 bits otherwise), decimal quotients that don't end + within 18 fraction digits (`1 / 3`: the evaluator's 24 digits are its own literal), dates against date-times or with and without a time zone, and unbound values give the expression back to the evaluator. On 133 k doubles: `FILTER(?a > 4000)` 60 → 15 ms, `FILTER(?a * 1.21 > 5000 && ?a < 9000)` 89 → 27 ms. On 300 k rows: `SUM(YEAR(?d))` 83 → 14 ms, @@ -177,12 +181,12 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se `2.5`) gets an id made up for the query, which joins only itself; a filter that would have to compare it by value (`=`, `IN`, `isLiteral`, `DATATYPE`) leaves the query to the evaluator. Before any work, the plan checks that each `BIND` expression is one the kernels - compute, so a query with `UCASE` or a division doesn't run its patterns on ids first. An + compute, so a query with `UCASE` doesn't run its patterns on ids first. An order on an expression the kernels compute (`ORDER BY DESC(?q * 3)`) orders a column of its values, computed as a `BIND` would be (133k orders, `LIMIT 10`: 117 → 15 ms). Rows with equal keys may come in another order than under `spareval`. A `LIMIT` without an order, other order expressions, columns the ids don't order, an expression the kernels - don't compute (`UCASE`, a division), and every other operator (`GROUP BY` beyond the count + don't compute (`UCASE`, a decimal quotient that doesn't end), and every other operator (`GROUP BY` beyond the count shortcuts, property paths outside the tables) leave the query to the evaluator, which then runs it as before. - **Geometries.** Geometry literals (`geo:wktLiteral`, `geoJSONLiteral`, `gmlLiteral`, From f309bb8904211ff5d7c0ba24b7708d1a575bddae Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 22 Nov 2026 20:24:08 +0000 Subject: [PATCH 2/5] docs: changelog timings on synthetic data only (#11) --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b785b..87a2638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,8 @@ versions may change the store format and the API; the release notes say how to u ### Changed - More of one query runs on several cores (#4): the sorted joins of a basic graph pattern (chunks of rows, each walking the index on its own), `GROUP BY` of many rows (split by the hash of the key) and the `SUM`, `AVG`, `MIN`, `MAX` and `SAMPLE` of the groups. The answers, and their order, are those of one core. On 2M orders and 4 cores, a join of three patterns grouped by country went from 500 to 220 ms, a count over three patterns from 480 to 190 ms, and 200,000 customers' sums from 420 to 260 ms. Rows to sort for a join are sorted as (key, row) pairs, which also helps one core (570 → 470 ms). `--query-threads` sizes the pool, as before; see [the server options](docs/server.md#main-options). -- Under `--executor ids`, `BIND`s of strings and IRIs run on term ids too (#11): `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` with a constant pattern, each computed once per distinct input. On a production rule set of 55 queries, four more stay on ids, with the same answers; the one that rewrites five IRIs with `IRI(REPLACE(STR(?x), …))` takes 142 instead of 239 ms. See [the store](docs/store.md). -- Under `--executor ids`, a `REPLACE` whose pattern or flags are expressions without variables (`REPLACE(STR(?x), STR(ex:), "")`) runs on term ids too (#11), computed once. On a production rule set of 55 queries, three more stay on ids, with the same answers: 1.07 s → 0.17 s, 120 → 25 ms and 32 → 12 ms. +- Under `--executor ids`, `BIND`s of strings and IRIs run on term ids too (#11): `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` with a constant pattern, each computed once per distinct input. On 133k orders, `BIND(CONCAT(STR(?c), "/order") AS ?k)` takes 45 instead of 91 ms, and with a join on an `IRI` made that way 57 instead of 196 ms. See [the store](docs/store.md). +- Under `--executor ids`, a `REPLACE` whose pattern or flags are expressions without variables (`REPLACE(STR(?x), STR(ex:), "")`) runs on term ids too (#11), computed once, so the queries that rewrite IRIs of one namespace into another stay on ids. - Under `--executor ids`, `/` runs on term ids too (#11), in `BIND`, filters and orders: a quotient of integers or decimals is the evaluator's (exact, as Jena's), of doubles a double, and division by zero an error. On 133k orders, `BIND(?q / 4 AS ?h) FILTER(?h > 20)` takes 33 ms instead of 191. A decimal quotient that doesn't end within 18 fraction digits (`1 / 3`) is still left to the evaluator. - Under `--executor ids`, `ORDER BY` an expression the id kernels compute (`DESC(?q * 3)`, `ABS(?x)`, `?d - 1`) runs on term ids too (#11), as a column of the computed values. On 133k orders, `ORDER BY DESC(?q * 3) ?o LIMIT 10` takes 15 ms instead of 117, and with a join and `LIMIT 100` 20 ms instead of 245. Rows with equal keys may come in another order than under `spareval`. - `tripletrail load` keeps WKT literals in binary form in a new dataset (#152): `--render-wkt` is the default, so a literal whose text can be written back from its geometry exactly is kept once, not as text and geometry; `--keep-wkt-text` keeps the text. On an OpenStreetMap extract of Norway the store takes 364 MB instead of 731 MB, and writing all 290,000 WKT literals out takes 1.9 s instead of 1.3 s; spatial queries are unchanged. Existing datasets keep their setting. @@ -81,7 +81,7 @@ The first release: an Import page, a faster and richer map, SHACL messages and l - Row filters can hide a shared set of resources but for a few of each filter's own (`RowFilter::hide_shared`, `SharedHidden`, enterprise #7), held as a bitmap when the ids are dense. An "own records" rule (`sh:hasValue sg:requestingAgent`) uses it in TripleTrail Enterprise: every agent's filter shares the targets and lists only the agent's records, instead of holding every hidden target (about 4.5 MB per agent on 200,000 offers). A row filter skips its empty sets when testing a quad. `Snapshot::store_id` and `RowShapes::id` give keys for caches of what is derived from a store version and a set of shapes. - Under `--executor ids`, `BIND` and `SELECT (expr AS ?v)` run on term ids too (#11): copies, constants and `+ - *`, `ABS`, `ROUND` and the other number kernels, with the literals the evaluator writes. A `FILTER` on a computed value is computed from its expression. On 133k orders, `BIND(?a * ?q AS ?g) FILTER(?g > 400000)` took 222 ms and takes 49 ms, and a select expression over a join 197 → 59 ms. Decimal products no longer give up past about 170 (they are exact on 256 bits), and numeric constants such as `2.5e0` are read by value, so more filters and sums run on ids in either executor. See [the store](docs/store.md). -- Under `--executor ids`, `VALUES` runs on term ids too (#11), and a `BIND` the ids can't compute (`STR`, a division) is found before the patterns run, not after: on a production rule set of 55 queries, the two that took 1.4 s and 3.1 s under `ids` take 0.67 s and 1.2 s again, as under `spareval`. A computed number the store inlines is made straight into its id, without a literal or a lookup: `BIND(?a + 1 AS ?b)` over 133k doubles 116 → 77 ms. A `BIND` or `VALUES` constant the store doesn't hold (`2.5e0` when it holds `2.5`) compared by value in a filter is left to the evaluator. +- Under `--executor ids`, `VALUES` runs on term ids too (#11), and a `BIND` the ids can't compute (`STR`, a division) is found before the patterns run, not after, so such a query takes as long under `ids` as under `spareval` again. A computed number the store inlines is made straight into its id, without a literal or a lookup: `BIND(?a + 1 AS ?b)` over 133k doubles 116 → 77 ms. A `BIND` or `VALUES` constant the store doesn't hold (`2.5e0` when it holds `2.5`) compared by value in a filter is left to the evaluator. - Under `--executor ids`, a `GROUP BY` with 1,024 groups or more is answered from the table of ids its counts and sums are kept in (#11), and so is any sub-select whose pattern runs on ids. Over 10,000 customers of 133k orders, `GROUP BY ?c ?name` with a count takes 11 ms instead of 20, and `SUM` with `ORDER BY` and `LIMIT` 20 instead of 27. - Under `--executor ids`, sub-selects with `DISTINCT`, `REDUCED`, an `ORDER BY` of variables, or `LIMIT` and `OFFSET` under an order run on term ids too (#11). On 133k orders, `{ SELECT DISTINCT ?c ?q … }` takes 49 ms instead of 214, and a `DISTINCT` sub-select joined with a pattern 17 ms instead of 37. - Under `--executor ids`, SPARQL JSON, CSV and TSV answers are written straight from the term ids (#11): each distinct term of the answer is decoded and written once, on the query threads, and copied into the rows where it occurs, byte for byte as before. A three-pattern join returning 133k rows went from 142 to 88 ms as CSV and from 228 to 119 ms as JSON (from 333 and 402 ms with the evaluator). The result cache keeps such answers as they are. See [the server options](docs/server.md#main-options). From 87d0d2037011f6a3eeb6bde1983fc852f35be946 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 22 Nov 2026 20:58:54 +0000 Subject: [PATCH 3/5] perf(executor): rounding of doubles and numeric casts on term ids (#11) --- CHANGELOG.md | 1 + crates/tripletrail/src/direct.rs | 9 +- crates/tripletrail/src/idstring.rs | 4 +- crates/tripletrail/src/idtree.rs | 25 +++-- crates/tripletrail/src/idvalue.rs | 150 +++++++++++++++++++++++----- crates/tripletrail/tests/id_rows.rs | 63 ++++++++++++ docs/store.md | 7 +- 7 files changed, 223 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 556764a..e6207a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ versions may change the store format and the API; the release notes say how to u - Under `--executor ids`, `BIND`s of strings and IRIs run on term ids too (#11): `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` with a constant pattern, each computed once per distinct input. On 133k orders, `BIND(CONCAT(STR(?c), "/order") AS ?k)` takes 45 instead of 91 ms, and with a join on an `IRI` made that way 57 instead of 196 ms. See [the store](docs/store.md). - Under `--executor ids`, a `REPLACE` whose pattern or flags are expressions without variables (`REPLACE(STR(?x), STR(ex:), "")`) runs on term ids too (#11), computed once, so the queries that rewrite IRIs of one namespace into another stay on ids. - Under `--executor ids`, `/` runs on term ids too (#11), in `BIND`, filters and orders: a quotient of integers or decimals is the evaluator's (exact, as Jena's), of doubles a double, and division by zero an error. On 133k orders, `BIND(?q / 4 AS ?h) FILTER(?h > 20)` takes 33 ms instead of 191. A decimal quotient that doesn't end within 18 fraction digits (`1 / 3`) is still left to the evaluator. +- Under `--executor ids`, `ABS`, `CEIL`, `FLOOR` and `ROUND` of doubles, and `xsd:integer` and `xsd:decimal` casts of numbers, run on term ids too (#11), with the evaluator's answers. A rule such as `BIND(ROUND(?amount * 1000) / 1000 AS ?r)` over doubles no longer sends the query to the evaluator. - Under `--executor ids`, `ORDER BY` an expression the id kernels compute (`DESC(?q * 3)`, `ABS(?x)`, `?d - 1`) runs on term ids too (#11), as a column of the computed values. On 133k orders, `ORDER BY DESC(?q * 3) ?o LIMIT 10` takes 15 ms instead of 117, and with a join and `LIMIT 100` 20 ms instead of 245. Rows with equal keys may come in another order than under `spareval`. - `tripletrail load` keeps WKT literals in binary form in a new dataset (#152): `--render-wkt` is the default, so a literal whose text can be written back from its geometry exactly is kept once, not as text and geometry; `--keep-wkt-text` keeps the text. On an OpenStreetMap extract of Norway the store takes 364 MB instead of 731 MB, and writing all 290,000 WKT literals out takes 1.9 s instead of 1.3 s; spatial queries are unchanged. Existing datasets keep their setting. - WKT kept in binary form takes about a quarter of the space (#152): coordinates with at most 9 decimals are kept as fixed-point integers, each the difference from the one before, in 2 or 3 bytes instead of 8. On an OpenStreetMap extract of Norway (290,000 geometries, 7 decimals) a new store takes 178 MB instead of 364 MB (731 MB with the text kept); results and query times are the same. Such dictionary files are format 5 (`OXGSDIC5`), which an older binary refuses; format 4 stores still open. diff --git a/crates/tripletrail/src/direct.rs b/crates/tripletrail/src/direct.rs index 7c865b4..3147d2a 100644 --- a/crates/tripletrail/src/direct.rs +++ b/crates/tripletrail/src/direct.rs @@ -233,7 +233,14 @@ fn computable(e: &Expression, top: bool) -> bool { Expression::FunctionCall(F::Custom(f), args) if f.as_ref() == oxigraph::model::vocab::xsd::INTEGER => { - args.len() == 1 && !matches!(args[0], Expression::Literal(_)) + args.len() == 1 + && (crate::idtree::text_test(&args[0]).is_some() || computable(&args[0], false)) + } + // `xsd:decimal` of a number. + Expression::FunctionCall(F::Custom(f), args) + if f.as_ref() == oxigraph::model::vocab::xsd::DECIMAL => + { + args.len() == 1 && computable(&args[0], false) } _ => false, } diff --git a/crates/tripletrail/src/idstring.rs b/crates/tripletrail/src/idstring.rs index ec63f93..7d22e24 100644 --- a/crates/tripletrail/src/idstring.rs +++ b/crates/tripletrail/src/idstring.rs @@ -231,10 +231,10 @@ pub(crate) fn compile<'k>( } // A number the value kernels compute, as the literal the evaluator writes. e => { - let k = crate::idvalue::compile(e, t, snapshot)?; + let k = crate::idvalue::compile_output(e, t, snapshot)?; Box::new(move |row| { Some(match k(row)? { - Ok(v) => Ok(crate::idvalue::literal(v)?.into()), + Ok((v, form)) => Ok(crate::idvalue::literal(v, form)?.into()), Err(()) => Err(()), }) }) diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index bc106b0..12c88d9 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -340,27 +340,28 @@ impl Eval<'_> { unbound } _ => { - let kernel = crate::idvalue::compile(expression, &t, self.snapshot)?; - // Each value once: the same value gives the same id. - let mut ids: HashMap<(u8, i128), TermId> = HashMap::new(); + let kernel = crate::idvalue::compile_output(expression, &t, self.snapshot)?; + // Each value once: the same value (in the same form) gives the same id. + let mut ids: HashMap<((u8, i128), crate::idvalue::Form), TermId> = HashMap::new(); let mut unbound = false; for r in 0..rows { if r % 4096 == 0 && (self.cancelled)() { return None; } match kernel(t.row(r))? { - Ok(v) => { + Ok((v, form)) => { // An inline number is its id: no lookup, no literal. - if let Some(id) = crate::idvalue::inline_id(v) { + if let Some(id) = crate::idvalue::inline_id(v, form) { column.push(id); continue; } - let key = value_key(v)?; + let key = (value_key(v)?, form); let id = match ids.get(&key) { Some(&id) => id, None => { - let id = - self.snapshot.id_of(crate::idvalue::literal(v)?.into()); + let id = self + .snapshot + .id_of(crate::idvalue::literal(v, form)?.into()); ids.insert(key, id); id } @@ -1558,7 +1559,13 @@ pub(crate) fn value_expression(e: &Expression) -> bool { E::FunctionCall(Function::Custom(f), args) if f.as_ref() == oxigraph::model::vocab::xsd::INTEGER => { - matches!(args.as_slice(), [a] if text_test(a).is_some()) + matches!(args.as_slice(), [a] if text_test(a).is_some() || value_expression(a)) + } + // `xsd:decimal` of a number (#11). + E::FunctionCall(Function::Custom(f), args) + if f.as_ref() == oxigraph::model::vocab::xsd::DECIMAL => + { + matches!(args.as_slice(), [a] if value_expression(a)) } _ => false, } diff --git a/crates/tripletrail/src/idvalue.rs b/crates/tripletrail/src/idvalue.rs index d5cdcb5..f96e022 100644 --- a/crates/tripletrail/src/idvalue.rs +++ b/crates/tripletrail/src/idvalue.rs @@ -69,12 +69,59 @@ pub(crate) fn of_id(id: TermId) -> Option { } } -/// The literal the evaluator gives for a value it computed (`BIND(?a * 1.21 AS ?b)`), if -/// the ids decide it: dates and date-times are left to the evaluator. -pub(crate) fn literal(v: Value) -> Option { +/// How the evaluator writes a whole decimal it made: `3.0` as Jena writes a computed one, or +/// `3`, the text of the integer an `xsd:decimal` cast keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) enum Form { + Computed, + Cast, +} + +/// A kernel that also gives the form its decimals are written in. +pub(crate) type OutputKernel<'k> = + Box Option> + Sync + 'k>; + +/// [`compile`] for a value that becomes a term (a `BIND`, `STR`), whose lexical form counts. +/// `xsd:decimal` keeps the text of its argument, as Jena's: an integer's (`7`), a decimal's, +/// and the computed form of a double's (`1500.0`). A whole decimal argument may be written +/// either way (`3` or `3.0`), so it gives up. +pub(crate) fn compile_output<'k>( + e: &Expression, + t: &Table, + snapshot: &'k Snapshot, +) -> Option> { + match e { + Expression::FunctionCall(Function::Custom(f), args) if f.as_ref() == xsd::DECIMAL => { + let [a] = args.as_slice() else { return None }; + let a = compile(a, t, snapshot)?; + Some(Box::new(move |row| match a(row)? { + Ok(x) => { + let form = match x { + Value::Integer(_) => Form::Cast, + Value::Decimal(d) if d % ONE == 0 => return None, + _ => Form::Computed, + }; + Some(Ok((cast(false, x)?, form))) + } + Err(()) => Some(Err(())), + })) + } + _ => { + let k = compile(e, t, snapshot)?; + Some(Box::new(move |row| { + Some(k(row)?.map(|v| (v, Form::Computed))) + })) + } + } +} + +/// The literal the evaluator gives for a value it computed (`BIND(?a * 1.21 AS ?b)`), with +/// whole decimals written in `form`, if the ids decide it: dates and date-times are left to +/// the evaluator. +pub(crate) fn literal(v: Value, form: Form) -> Option { Some(match v { Value::Integer(i) => Literal::from(i), - Value::Decimal(d) => Literal::new_typed_literal(decimal_lexical(d), xsd::DECIMAL), + Value::Decimal(d) => Literal::new_typed_literal(decimal_lexical(d, form), xsd::DECIMAL), Value::Double(d) => Literal::from(oxsdatatypes::Double::from(d)), Value::DateTime { .. } => return None, }) @@ -82,10 +129,10 @@ pub(crate) fn literal(v: Value) -> Option { /// The inline id of the literal [`literal`] gives for `v`, if the store inlines it: the id /// it gives that literal, made without the literal. -pub(crate) fn inline_id(v: Value) -> Option { +pub(crate) fn inline_id(v: Value, form: Form) -> Option { match v { Value::Integer(i) => TermId::inline_integer(i), - Value::Decimal(d) => TermId::inline_number(false, &decimal_lexical(d)), + Value::Decimal(d) => TermId::inline_number(false, &decimal_lexical(d, form)), Value::Double(d) if d.is_finite() => { use std::io::Write; // `oxsdatatypes::Double` writes a finite double as `f64` does. @@ -165,7 +212,11 @@ pub(crate) fn compile<'k>( }) } // `xsd:integer(CONTAINS(?v, "x"))` and the other text tests: 1 or 0. - E::FunctionCall(Function::Custom(f), args) if f.as_ref() == xsd::INTEGER => { + E::FunctionCall(Function::Custom(f), args) + if f.as_ref() == xsd::INTEGER + && args.len() == 1 + && crate::idtree::text_test(&args[0]).is_some() => + { let [a] = args.as_slice() else { return None }; let (v, test) = crate::idtree::text_test(a)?; let c = t.col(v); @@ -178,6 +229,19 @@ pub(crate) fn compile<'k>( Some(Ok(Value::Integer(i64::from(is)))) }) } + // `xsd:integer` and `xsd:decimal` of a number (#11), as the evaluator casts. A cast to + // `xsd:double` keeps the lexical form as Jena does, so it is left to the evaluator. + E::FunctionCall(Function::Custom(f), args) + if f.as_ref() == xsd::INTEGER || f.as_ref() == xsd::DECIMAL => + { + let [a] = args.as_slice() else { return None }; + let to_integer = f.as_ref() == xsd::INTEGER; + let a = compile(a, t, snapshot)?; + Box::new(move |row| match a(row)? { + Ok(x) => cast(to_integer, x).map(Ok), + Err(()) => Some(Err(())), + }) + } E::FunctionCall(f, args) => { let [a] = args.as_slice() else { return None }; let f = match f { @@ -463,6 +527,13 @@ fn unary(f: Unary, x: Value) -> Option { (Unary::Round, Value::Decimal(d)) => Some(Value::Decimal( d.checked_add(ONE / 2)?.div_euclid(ONE).checked_mul(ONE)?, )), + // As `oxsdatatypes::Double`: halves round up, and a negative that rounds to zero is -0. + (Unary::Abs, Value::Double(d)) => Some(Value::Double(d.abs())), + (Unary::Ceil, Value::Double(d)) => Some(Value::Double(d.ceil())), + (Unary::Floor, Value::Double(d)) => Some(Value::Double(d.floor())), + (Unary::Round, Value::Double(d)) => Some(Value::Double(f64::from( + oxsdatatypes::Double::from(d).round(), + ))), (Unary::Year | Unary::Month | Unary::Day, Value::DateTime { instant, tz, .. }) => { let local = instant.checked_add(i64::from(tz.unwrap_or(0)) * 60_000)?; let (y, m, d) = civil(local.div_euclid(MS_PER_DAY)); @@ -491,6 +562,34 @@ fn unary(f: Unary, x: Value) -> Option { } } +/// `xsd:integer` (`to_integer`) or `xsd:decimal` of a number, as `spareval` casts: towards +/// zero to an integer, and a double to the shortest decimal that reads back as it (Jena's +/// `BigDecimal.valueOf`). Out of range, or a date: give up. +fn cast(to_integer: bool, x: Value) -> Option { + let decimal = match x { + Value::Integer(i) if to_integer => return Some(Value::Integer(i)), + Value::Integer(i) => i128::from(i).checked_mul(ONE)?, + Value::Decimal(d) => d, + Value::Double(d) if to_integer => { + let whole = d.trunc(); + // i64::MAX as f64 rounds up to 2^63, which is out of range. + #[expect(clippy::cast_precision_loss, clippy::cast_possible_truncation)] + return (whole >= i64::MIN as f64 && whole < i64::MAX as f64) + .then_some(Value::Integer(whole as i64)); + } + Value::Double(d) => { + let d: oxsdatatypes::Decimal = format!("{d}").parse().ok()?; + i128::from_be_bytes(d.to_be_bytes()) + } + Value::DateTime { .. } => return None, + }; + Some(if to_integer { + Value::Integer(i64::try_from(decimal / ONE).ok()?) + } else { + Value::Decimal(decimal) + }) +} + /// Year, month and day of the day `days` after 1970-01-01 (proleptic Gregorian). fn civil(days: i64) -> (i64, i64, i64) { let z = days + 719_468; @@ -586,7 +685,7 @@ impl Sum { // evaluator turns each into a double when it adds it to a double. let rest: f64 = match exact { Exact::Integer(i) => i.to_string().parse().ok()?, - Exact::Decimal(d) => decimal_lexical(d).parse().ok()?, + Exact::Decimal(d) => decimal_lexical(d, Form::Computed).parse().ok()?, }; f.add(rest)?; Total::Double(f.value()) @@ -598,7 +697,9 @@ impl Sum { pub(crate) fn literal(&self) -> Option { Some(match self.total()? { Total::Integer(i) => Literal::from(i), - Total::Decimal(d) => Literal::new_typed_literal(decimal_lexical(d), xsd::DECIMAL), + Total::Decimal(d) => { + Literal::new_typed_literal(decimal_lexical(d, Form::Computed), xsd::DECIMAL) + } Total::Double(d) => Literal::from(oxsdatatypes::Double::from(d)), }) } @@ -700,8 +801,8 @@ impl Fsum { } /// The canonical form of a decimal times 10^18, as the evaluator writes it: no exponent, no -/// trailing fraction zeros, no point for a whole number. -fn decimal_lexical(d: i128) -> String { +/// trailing fraction zeros, and a whole number as `form` has it. +fn decimal_lexical(d: i128, form: Form) -> String { let negative = d < 0; let magnitude = d.unsigned_abs(); let whole = magnitude / ONE.unsigned_abs(); @@ -715,7 +816,7 @@ fn decimal_lexical(d: i128) -> String { let digits = format!("{fraction:018}"); s.push('.'); s.push_str(digits.trim_end_matches('0')); - } else { + } else if form == Form::Computed { // Jena's form of a whole computed decimal (see `spareval::decimal_lexical`). s.push_str(".0"); } @@ -785,13 +886,15 @@ mod tests { values.push(Value::Double(d)); } for v in values { - let stored = literal(v).map(|l| snapshot.id_of(l.into())); - match inline_id(v) { - Some(id) => assert_eq!(Some(id), stored, "{v:?}"), - None => assert!( - stored.is_none_or(|id| id.kind() == Kind::Ephemeral), - "{v:?} is inline: {stored:?}" - ), + for form in [Form::Computed, Form::Cast] { + let stored = literal(v, form).map(|l| snapshot.id_of(l.into())); + match inline_id(v, form) { + Some(id) => assert_eq!(Some(id), stored, "{v:?} {form:?}"), + None => assert!( + stored.is_none_or(|id| id.kind() == Kind::Ephemeral), + "{v:?} is inline: {stored:?}" + ), + } } } } @@ -839,10 +942,11 @@ mod tests { #[test] fn decimals_print_as_the_evaluator_does() { - assert_eq!(decimal_lexical(0), "0.0"); - assert_eq!(decimal_lexical(5 * ONE), "5.0"); - assert_eq!(decimal_lexical(-15 * ONE / 10), "-1.5"); - assert_eq!(decimal_lexical(ONE / 100), "0.01"); + assert_eq!(decimal_lexical(0, Form::Computed), "0.0"); + assert_eq!(decimal_lexical(5 * ONE, Form::Computed), "5.0"); + assert_eq!(decimal_lexical(-15 * ONE / 10, Form::Computed), "-1.5"); + assert_eq!(decimal_lexical(ONE / 100, Form::Computed), "0.01"); + assert_eq!(decimal_lexical(5 * ONE, Form::Cast), "5"); } #[test] diff --git a/crates/tripletrail/tests/id_rows.rs b/crates/tripletrail/tests/id_rows.rs index 150a93b..20b843e 100644 --- a/crates/tripletrail/tests/id_rows.rs +++ b/crates/tripletrail/tests/id_rows.rs @@ -441,6 +441,69 @@ const BINDS: &[(&str, bool)] = &[ "SELECT ?s ?x WHERE { ?s ex:n ?x } ORDER BY DESC(?x / 8) ?s LIMIT 7", true, ), + // Doubles rounded (halves up, -0.5 to -0), and `xsd:integer` and `xsd:decimal` of numbers. + ( + "SELECT ?s ?a ?b ?c ?d WHERE { ?s ex:f ?x BIND(ROUND(?x) AS ?a) BIND(CEIL(?x) AS ?b) BIND(FLOOR(?x) AS ?c) BIND(ABS(?x) AS ?d) }", + true, + ), + ( + "SELECT ?s ?y ?z WHERE { ?s ex:f ?x BIND(ROUND(?x - ?x - 0.5) AS ?y) BIND(CEIL(?x - ?x - 0.5) AS ?z) }", + true, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:f ?x BIND(ROUND(?x * 1000) / 1000 AS ?y) FILTER(?y > 7.0) }", + true, + ), + ( + "SELECT ?s ?a ?b ?c WHERE { ?s ex:f ?x ; ex:d ?d ; ex:n ?n BIND(xsd:integer(?x) AS ?a) BIND(xsd:integer(?d) AS ?b) BIND(xsd:integer(?n / 8) AS ?c) }", + true, + ), + ( + "SELECT ?s ?a ?b ?c WHERE { ?s ex:f ?x ; ex:d ?d ; ex:n ?n BIND(xsd:decimal(?x) AS ?a) BIND(xsd:decimal(?n) AS ?b) BIND(xsd:decimal(?d * 3) AS ?c) }", + true, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:f ?x BIND(xsd:decimal(?x / 3) AS ?y) }", + true, + ), + ( + "SELECT ?s ?t ?u WHERE { ?s ex:n ?n BIND(STR(xsd:decimal(?n)) AS ?t) BIND(xsd:decimal(?n) + 0 AS ?u) }", + true, + ), + ( + "SELECT DISTINCT ?y WHERE { ?s ex:n ?n BIND(xsd:decimal(?n * 0) AS ?y) }", + true, + ), + // The cast keeps the text: `7` of an integer, `1500.0` of a double; a whole decimal is + // the evaluator's. + ( + "SELECT DISTINCT ?y WHERE { ?s ex:f ?x BIND(xsd:decimal(?x * 2) AS ?y) }", + true, + ), + ( + "SELECT ?s ?y ?z WHERE { ?s ex:n ?n BIND(xsd:decimal(?n * 2.5e0) AS ?y) BIND(STR(xsd:decimal(?n * 2e0)) AS ?z) }", + true, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:d ?d BIND(xsd:decimal(ROUND(?d)) AS ?y) }", + false, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:d ?d BIND(ROUND(xsd:decimal(?d * 2e0)) AS ?y) }", + true, + ), + ( + "SELECT ?y WHERE { ?s ex:huge ?x BIND(xsd:integer(?x * 1.0e3) AS ?y) }", + false, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:n ?x BIND(xsd:double(?x) AS ?y) }", + false, + ), + ( + "SELECT ?s ?y WHERE { ?s ex:label ?l BIND(xsd:integer(?l) AS ?y) }", + false, + ), // `REPLACE` with a constant pattern (and flags), `STRLANG`, `STRDT`. ( "SELECT ?s ?i WHERE { ?s ex:p ?o BIND(IRI(REPLACE(STR(?o), \"example\", \"other\")) AS ?i) }", diff --git a/docs/store.md b/docs/store.md index 2130e01..d0f7cfd 100644 --- a/docs/store.md +++ b/docs/store.md @@ -95,7 +95,12 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se evaluator's (`spareval::decimal_divide`, Jena's), of doubles a double, by zero an error (`BIND(?q / 4 AS ?h) FILTER(?h > 20)` on 133k orders 191 → 33 ms). Comparisons and `+ - * /` take inline doubles too (#11), with integers and decimals turned - into doubles as the evaluator turns them, and `-0` equal to `0`. Dictionary literals + into doubles as the evaluator turns them, and `-0` equal to `0`. So do `ABS`, `CEIL`, + `FLOOR` and `ROUND` (halves up, `ROUND(-0.5)` is `-0`). `xsd:integer` and `xsd:decimal` of + a number are cast as the evaluator casts: towards zero to an integer, and a double to the + shortest decimal that reads back as it. `xsd:decimal` keeps the text of its argument, as + Jena's: `7` of the integer 7, `1500.0` of the double 1500; a whole decimal argument (`3` or + `3.0`) stays with the evaluator, and so does `xsd:double`, which keeps the text too. Dictionary literals (`1.5E2`), a double result that is not finite, overflow, decimal products with more than 18 fraction digits (they are exact on 256 bits otherwise), decimal quotients that don't end within 18 fraction digits (`1 / 3`: the evaluator's 24 digits are its own literal), dates From f2e5b30d1b4eb9a3f399bf02cea6c9a6a38edb34 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 22 Nov 2026 21:49:02 +0000 Subject: [PATCH 4/5] perf(executor): GROUP BY over a BIND counts per input first under ids (#11) --- CHANGELOG.md | 1 + crates/tripletrail/src/count.rs | 12 ++++++++++++ crates/tripletrail/tests/id_rows.rs | 14 ++++++++++++++ docs/store.md | 5 ++++- 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc777fe..d2e48b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ versions may change the store format and the API; the release notes say how to u - Under `--executor ids`, `BIND`s of strings and IRIs run on term ids too (#11): `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` with a constant pattern, each computed once per distinct input. On 133k orders, `BIND(CONCAT(STR(?c), "/order") AS ?k)` takes 45 instead of 91 ms, and with a join on an `IRI` made that way 57 instead of 196 ms. See [the store](docs/store.md). - Under `--executor ids`, a `REPLACE` whose pattern or flags are expressions without variables (`REPLACE(STR(?x), STR(ex:), "")`) runs on term ids too (#11), computed once, so the queries that rewrite IRIs of one namespace into another stay on ids. - Under `--executor ids`, `/` runs on term ids too (#11), in `BIND`, filters and orders: a quotient of integers or decimals is the evaluator's (exact, as Jena's), of doubles a double, and division by zero an error. On 133k orders, `BIND(?q / 4 AS ?h) FILTER(?h > 20)` takes 33 ms instead of 191. A decimal quotient that doesn't end within 18 fraction digits (`1 / 3`) is still left to the evaluator. +- Under `--executor ids`, a `GROUP BY` over a `BIND` with few distinct inputs counts the rows per input first, as under `spareval`, and computes the expression once per input instead of once per row (#11): `BIND(?q * 2 AS ?k) … GROUP BY ?k` over 133k orders takes as long as under `spareval` (11 ms) instead of 16 to 21. - Under `--executor ids`, `ABS`, `CEIL`, `FLOOR` and `ROUND` of doubles, and `xsd:integer` and `xsd:decimal` casts of numbers, run on term ids too (#11), with the evaluator's answers. A rule such as `BIND(ROUND(?amount * 1000) / 1000 AS ?r)` over doubles no longer sends the query to the evaluator. - Under `--executor ids`, `ORDER BY` an expression the id kernels compute (`DESC(?q * 3)`, `ABS(?x)`, `?d - 1`) runs on term ids too (#11), as a column of the computed values. On 133k orders, `ORDER BY DESC(?q * 3) ?o LIMIT 10` takes 15 ms instead of 117, and with a join and `LIMIT 100` 20 ms instead of 245. Rows with equal keys may come in another order than under `spareval`. - `tripletrail load` keeps WKT literals in binary form in a new dataset (#152): `--render-wkt` is the default, so a literal whose text can be written back from its geometry exactly is kept once, not as text and geometry; `--keep-wkt-text` keeps the text. On an OpenStreetMap extract of Norway the store takes 364 MB instead of 731 MB, and writing all 290,000 WKT literals out takes 1.9 s instead of 1.3 s; spatial queries are unchanged. Existing datasets keep their setting. diff --git a/crates/tripletrail/src/count.rs b/crates/tripletrail/src/count.rs index ae126fe..977fbe9 100644 --- a/crates/tripletrail/src/count.rs +++ b/crates/tripletrail/src/count.rs @@ -674,6 +674,18 @@ impl Pass<'_> { ], }; } + // Over a `BIND`, counting the rows per distinct input first computes the + // expression once per input value, not once per row: under the id executor + // the grouped counts below would compute it for every row (#11). + let computed = matches!(*inner, G::Extend { .. }) + || matches!(&*inner, G::Filter { inner, .. } if matches!(**inner, G::Extend { .. })); + if computed + && crate::executor_ids() + && let Some(group) = self.eager_group(&inner, &variables, &aggregates) + { + self.done += 1; + return group; + } // Without keys, counts are answered above; MIN, MAX and SAMPLE here. if (!variables.is_empty() || count_aggregates(&aggregates).is_none()) && let Some(values) = diff --git a/crates/tripletrail/tests/id_rows.rs b/crates/tripletrail/tests/id_rows.rs index 20b843e..d02abfe 100644 --- a/crates/tripletrail/tests/id_rows.rs +++ b/crates/tripletrail/tests/id_rows.rs @@ -729,6 +729,20 @@ fn binds_on_ids_are_the_evaluators() { /// Groups a count pass reads back from a table of ids (1,024 or more): the id executor answers /// the query from that table (#11), with the evaluator's answers. const GROUPS: &[(&str, bool)] = &[ + // Over a `BIND` with few distinct inputs: counted per input first, the expression then + // computed per input (#11). + ( + "SELECT ?k (COUNT(*) AS ?n) WHERE { ?o ex:q ?q BIND(?q * 2 AS ?k) } GROUP BY ?k", + false, + ), + ( + "SELECT ?k (SUM(?q) AS ?t) (MAX(?q) AS ?m) WHERE { ?o ex:q ?q BIND(?q - 3 AS ?k) FILTER(?k > 1) } GROUP BY ?k", + false, + ), + ( + "SELECT ?c (COUNT(DISTINCT ?k) AS ?n) WHERE { ?o ex:by ?c ; ex:q ?q BIND(?q / 2 AS ?k) } GROUP BY ?c", + false, + ), ( "SELECT ?c (COUNT(*) AS ?n) WHERE { ?o ex:by ?c } GROUP BY ?c", true, diff --git a/docs/store.md b/docs/store.md index e2a5da5..b422057 100644 --- a/docs/store.md +++ b/docs/store.md @@ -82,7 +82,10 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se without `GROUP BY`: IRIs and simple strings compared by their dictionary keys, integers and booleans by value; other values (blank nodes, language strings, other typed literals, mixed literal kinds) leave the group to the evaluator. Under `ORDER BY` of counts with `LIMIT k`, - only the first k groups are kept, before their other aggregates are computed. BSBM 1M: + only the first k groups are kept, before their other aggregates are computed. A `GROUP BY` + over `BIND`s with few distinct inputs (`BIND(?q * 2 AS ?k) … GROUP BY ?k`) counts the rows + per input first and computes the expression once per input, under `--executor ids` too + (#11). BSBM 1M: `group-by-complex-aggregate` 54 → 16 ms, `group-by-implicit-string-min` 11 → 4 ms. Inline integers, decimals, dates and date-times (#125) are read as values from their ids (`crates/tripletrail/src/idvalue.rs`, #170): `<`, `<=`, `>`, `>=` and `=` between them and From 00f275ac905b140a2e3529a566be44a9a1c2e625 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 22 Nov 2026 22:08:43 +0000 Subject: [PATCH 5/5] docs(server): the --executor list names the newer id kernels (#11) --- docs/server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server.md b/docs/server.md index 6ffe336..5249d8a 100644 --- a/docs/server.md +++ b/docs/server.md @@ -206,7 +206,7 @@ tripletrail serve --store oxigraph --location /data/tripletrail --dataset kg - `--executor` (`TRIPLETRAIL_EXECUTOR`, `spareval` by default) picks what answers a `SELECT` whose pattern runs on term ids (#11). `ids` answers it straight from the id tables (triple patterns, joins, `OPTIONAL`, `MINUS`, `UNION`, `FILTER (NOT) EXISTS`, the filters on ids, - `BIND` and `SELECT (expr AS ?v)` of copies, constants, arithmetic, `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE`, `VALUES`, sub-selects (with `DISTINCT`, an order and a slice), the `GROUP BY` counts and sums kept as ids, `ORDER BY` of variables and of the expressions the kernels compute, `DISTINCT`, `REDUCED`, `OFFSET` and `LIMIT` with an order), decoding + `BIND` and `SELECT (expr AS ?v)` of copies, constants, arithmetic with division, `ABS`, `CEIL`, `FLOOR`, `ROUND`, `xsd:integer` and `xsd:decimal` casts, `IF`, `COALESCE`, `STR`, `CONCAT`, `IRI`, `STRLANG`, `STRDT` and `REPLACE` (the full list and its limits are in [the store](store.md)), `VALUES`, sub-selects (with `DISTINCT`, an order and a slice), the `GROUP BY` counts and sums kept as ids, `ORDER BY` of variables and of the expressions the kernels compute, `DISTINCT`, `REDUCED`, `OFFSET` and `LIMIT` with an order), decoding only the terms of the answer, each once, on the query threads; anything else still goes to the evaluator. SPARQL JSON, CSV and TSV answers are written from the ids: each distinct term is written once and copied into the rows where it occurs (the bytes are those of the usual