From f59d6fc3d12e5592796fa29c9d30f8af36923dc0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 16 Jul 2026 21:29:03 +0000 Subject: [PATCH 01/86] Backport #108433 to 26.6: Fix flaky test_parallel_quorum_actually_quorum --- tests/integration/test_quorum_inserts_parallel/test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py index 3631c41e5806..416866a90ae5 100644 --- a/tests/integration/test_quorum_inserts_parallel/test.py +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -78,7 +78,10 @@ def long_insert(node): def test_parallel_quorum_actually_quorum(started_cluster): for i, node in enumerate([node1, node2, node3]): node.query( - "CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple()".format( + # node2 stays partitioned off node1/node3 (port 9009) for the whole test, so its + # GET_PART entries fail repeatedly and accumulate exponential fetch backoff. Disable + # it so node2 catches up promptly once the partition heals (see SYSTEM SYNC REPLICA). + "CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple() SETTINGS max_postpone_time_for_failed_replicated_fetches_ms = 0".format( num=i ) ) @@ -175,5 +178,8 @@ def insert_fail_quorum_timeout(node, settings): p.close() p.join() - node2.query("SYSTEM SYNC REPLICA q", timeout=10) + # Generous barrier timeout: under the parallel sanitizer flaky-check the post-heal + # catch-up fetch can take several seconds. Correctness is still gated by the retrying + # assert below, so a real "node2 never syncs" bug fails there rather than being hidden. + node2.query("SYSTEM SYNC REPLICA q", timeout=60) assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "3") From 3f48d5f8a120cc17d0779a260290e21d186821fd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 16 Jul 2026 21:29:16 +0000 Subject: [PATCH 02/86] Backport #108362 to 26.6: Reduce test keeper checks --- src/Common/ZooKeeper/TestKeeper.cpp | 9 +++++++++ src/Common/ZooKeeper/TestKeeper.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 1962ca03af41..20cd0202dcb4 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -938,6 +938,15 @@ void TestKeeper::clearExpiredTTLNodes() { const int64_t now_ms = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); + /// This is called on every iteration of the processing thread, i.e. before processing every + /// request. Scanning the whole container each time would make request processing O(container size), + /// which is prohibitively slow when there are many nodes (see TestConcurrentUpdatesFromHints). + /// TTL granularity is coarse anyway, so it is enough to sweep at most once per `TTL_CLEANUP_INTERVAL_MS`. + static constexpr int64_t TTL_CLEANUP_INTERVAL_MS = 100; + if (now_ms - last_ttl_cleanup_ms < TTL_CLEANUP_INTERVAL_MS) + return; + last_ttl_cleanup_ms = now_ms; + std::vector expired_paths; for (const auto & [path, node] : container) if (node.is_ttl && now_ms >= node.stat.mtime + node.ttl) diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index da9d9097420d..36de896f4cd7 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -159,6 +159,8 @@ class TestKeeper final : public IKeeper Watches watches; Watches list_watches; /// Watches for 'list' request (watches on children). + int64_t last_ttl_cleanup_ms = 0; + using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1}; From 14f29f41e25fbe903f0f1ec3e976bf5bca090858 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Jul 2026 08:31:12 +0000 Subject: [PATCH 03/86] Backport #110623 to 26.6: Account for Map subcolumn on-disk size in prewhere optimization --- src/Interpreters/InterpreterSelectQuery.cpp | 5 +- .../Optimizations/optimizePrewhere.cpp | 6 +-- src/Storages/IStorage.h | 4 ++ src/Storages/MergeTree/MergeTreeData.cpp | 35 +++++++++++++ src/Storages/MergeTree/MergeTreeData.h | 2 + src/Storages/StorageAlias.h | 1 + src/Storages/StorageMerge.cpp | 13 +++++ src/Storages/StorageMerge.h | 1 + src/Storages/StorageProxy.h | 1 + ...prewhere_map_subcolumn_read_cost.reference | 7 +++ ...04513_prewhere_map_subcolumn_read_cost.sql | 49 +++++++++++++++++++ 11 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.reference create mode 100644 tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ffcd0816f946..080bc6fbcc1a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -865,7 +865,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( && !query.hasJoin()) /// Join may produce rows with nulls or default values, it's difficult to analyze if they affected or not. { /// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable - if (const auto & column_sizes = storage->getColumnSizes(); !column_sizes.empty()) + Names queried_columns = syntax_analyzer_result->requiredSourceColumns(); + if (const auto & column_sizes = storage->getColumnSizes(queried_columns); !column_sizes.empty()) { /// Extract column compressed sizes. std::unordered_map column_compressed_sizes; @@ -875,8 +876,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( SelectQueryInfo current_info; current_info.query = query_ptr; current_info.syntax_analyzer_result = syntax_analyzer_result; - - Names queried_columns = syntax_analyzer_result->requiredSourceColumns(); const auto & supported_prewhere_columns = storage->supportedPrewhereColumns(); RangesInDataParts parts_for_estimator; diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp index 9b2bf726b36b..932e4641de98 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp @@ -175,7 +175,9 @@ void optimizePrewhere(QueryPlan::Node & parent_node, const bool remove_unused_co if (!optimize) return; - auto column_sizes = storage.getColumnSizes(); + const auto & queried_columns = source_step_with_filter->requiredSourceColumns(); + + auto column_sizes = storage.getColumnSizes(queried_columns); if (column_sizes.empty()) return; @@ -192,8 +194,6 @@ void optimizePrewhere(QueryPlan::Node & parent_node, const bool remove_unused_co for (const auto & [name, sizes] : column_sizes) column_compressed_sizes[name] = sizes.data_compressed; - const auto & queried_columns = source_step_with_filter->requiredSourceColumns(); - /// Statistics are only used to reorder conditions, so skip if there is just one. const auto & filter_root_node = filter_step->getExpression().findInOutputs(filter_step->getFilterColumnName()); const bool has_multiple_conditions = filter_root_node.type == ActionsDAG::ActionType::FUNCTION diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index e3a4396159a2..e794b9ea3efa 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -196,6 +196,10 @@ class IStorage : public std::enable_shared_from_this, public TypePromo using ColumnSizeByName = std::unordered_map; virtual ColumnSizeByName getColumnSizes() const { return {}; } + /// Same as parameterless overload but also includes sizes for requested subcolumns + /// The default implementation falls back to the parameterless version. + virtual ColumnSizeByName getColumnSizes(const Names & /*columns*/) const { return getColumnSizes(); } + /// Same as getColumnSizes() but may return nullopt in some specific engines like Merge/Alias virtual std::optional tryGetColumnSizes() const { return getColumnSizes(); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6f3fbad4b9b8..6dc0a3238b4b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6621,6 +6621,41 @@ void MergeTreeData::addPartContributionToColumnAndSecondaryIndexSizesUnlocked(co primary_index_size.add(part->getIndexSizeFromFile()); } +IStorage::ColumnSizeByName MergeTreeData::getColumnSizes(const Names & columns) const +{ + auto result = getColumnSizes(); + + /// Collect subcolumn names that are not already in the result. + Names subcolumn_names; + for (const auto & col_name : columns) + { + if (result.contains(col_name)) + continue; + + subcolumn_names.push_back(col_name); + } + + if (subcolumn_names.empty()) + return result; + + /// For each requested column that is a subcolumn and not already in the result, + /// aggregate its size across all active parts using getSubcolumnSize. + /// This gives the correct on-disk size for subcolumns based on required substreams. + auto parts_lock = readLockParts(); + auto committed_parts_range = getDataPartsStateRange(DataPartState::Active); + for (const auto & part : committed_parts_range) + { + for (const auto & col_name : subcolumn_names) + { + auto column = part->tryGetColumn(col_name); + if (column && column->isSubcolumn()) + result[col_name].add(part->getSubcolumnSize(col_name)); + } + } + + return result; +} + void MergeTreeData::removePartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part) const { /// If sizes are calculated lazily, don't remove part contribution. All sizes from all active parts will be calculated later. diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 826f4eead8f5..a84ed7252a1c 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1104,6 +1104,8 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr return column_sizes; } + ColumnSizeByName getColumnSizes(const Names & columns) const override; + IndexSizeByName getSecondaryIndexSizes() const override { /// Always keep locks order parts_lock -> sizes_lock diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index ab72b63a1f50..0b161a2559f9 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -261,6 +261,7 @@ class StorageAlias final : public IStorage, WithContext } ColumnSizeByName getColumnSizes() const override { auto target = tryGetTargetTable(); return target ? target->getColumnSizes() : ColumnSizeByName{}; } + ColumnSizeByName getColumnSizes(const Names & columns) const override { auto target = tryGetTargetTable(); return target ? target->getColumnSizes(columns) : ColumnSizeByName{}; } std::optional tryGetColumnSizes() const override { auto target = tryGetTargetTable(); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f9696523f70a..8a7d6cfeeaca 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1797,6 +1797,19 @@ IStorage::ColumnSizeByName StorageMerge::getColumnSizes() const return column_sizes; } +IStorage::ColumnSizeByName StorageMerge::getColumnSizes(const Names & columns) const +{ + ColumnSizeByName column_sizes; + + forEachTable([&](const auto & table) + { + for (const auto & [name, size] : table->getColumnSizes(columns)) + column_sizes[name].add(size); + }); + + return column_sizes; +} + std::optional StorageMerge::tryGetColumnSizes() const { try diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 8f807e9e985e..8f8eb45103d2 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -147,6 +147,7 @@ class StorageMerge final : public IStorage, WithContext const IStorage * ignore_self); ColumnSizeByName getColumnSizes() const override; + ColumnSizeByName getColumnSizes(const Names & columns) const override; std::optional tryGetColumnSizes() const override; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index dc1c0570e9b6..77b1fce2d6ff 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -33,6 +33,7 @@ class StorageProxy : public IStorage bool supportsColumnsWithDynamicStructure() const override { return getNested()->supportsColumnsWithDynamicStructure(); } ColumnSizeByName getColumnSizes() const override { return getNested()->getColumnSizes(); } + ColumnSizeByName getColumnSizes(const Names & columns) const override { return getNested()->getColumnSizes(columns); } StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & base_metadata, ContextPtr query_context) const override { diff --git a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.reference b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.reference new file mode 100644 index 000000000000..b228761c64b3 --- /dev/null +++ b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.reference @@ -0,0 +1,7 @@ +-- cheap modality filter is placed before the Map-key predicate +1 +-- correctness: result is the same regardless of ordering +0 +0 +-- legacy InterpreterSelectQuery path: cheap filter first +1 diff --git a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql new file mode 100644 index 000000000000..602bd35806d4 --- /dev/null +++ b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql @@ -0,0 +1,49 @@ +-- Regression test: the PREWHERE optimizer must account for Map-key subcolumn read cost. +-- When `optimize_functions_to_subcolumns` rewrites `h['k']` to the `h.key_k` subcolumn, +-- the optimizer must see its actual on-disk size (the whole Map) via getSubcolumnSize +-- and place the cheap `modality` filter before the expensive Map-key predicate. +-- +-- We disable statistics so that only columns_size drives the ordering, +-- and use equality conditions on both sides so that both are "good" conditions +-- (isConditionGood only returns true for equals). + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; +SET allow_reorder_prewhere_conditions = 1; +SET use_statistics = 0; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_prewhere_map_cost; +CREATE TABLE t_prewhere_map_cost (id UInt64, modality LowCardinality(String), h Map(String, String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_prewhere_map_cost +SELECT number, if(number < 1000, 'active', ''), map('k', repeat('v', 300), 'k2', repeat('w', 300)) +FROM numbers(200000); +OPTIMIZE TABLE t_prewhere_map_cost FINAL; + +SELECT '-- cheap modality filter is placed before the Map-key predicate'; +SELECT position(explain, 'modality') > 0 AND position(explain, 'modality') < position(explain, 'h.key_k') AS cheap_first +FROM ( + EXPLAIN actions = 1 SELECT count() FROM t_prewhere_map_cost WHERE modality = '' AND h['k'] = 'nope' +) WHERE explain LIKE '%Prewhere filter column%'; + +SELECT '-- correctness: result is the same regardless of ordering'; +SELECT count() FROM t_prewhere_map_cost WHERE modality = '' AND h['k'] = 'nope'; +SELECT count() FROM t_prewhere_map_cost WHERE modality = '' AND h['k'] = 'nope' +SETTINGS allow_reorder_prewhere_conditions = 0; + +-- Same check through the legacy InterpreterSelectQuery PREWHERE path +-- (disable both the analyzer and the plan-based PREWHERE optimizer). +SET enable_analyzer = 0; +SET query_plan_optimize_prewhere = 0; + +SELECT '-- legacy InterpreterSelectQuery path: cheap filter first'; +SELECT position(explain, 'modality') > 0 AND position(explain, 'modality') < position(explain, 'arrayElement') AS cheap_first +FROM ( + EXPLAIN actions = 1 SELECT count() FROM t_prewhere_map_cost WHERE modality = '' AND h['k'] = 'nope' +) WHERE explain LIKE '%Prewhere filter column%'; + +DROP TABLE t_prewhere_map_cost; From 7cee4b90401244cbeeed0ec484813b434eb1ef90 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Jul 2026 08:34:05 +0000 Subject: [PATCH 04/86] Backport #107053 to 26.6: Fix Virtual row with union produces wrong result --- .../Algorithms/MergingSortedAlgorithm.cpp | 22 ++---- .../Optimizations/optimizeReadInOrder.cpp | 33 +++++++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 1 + ..._read_in_order_virtual_row_union.reference | 34 +++++++++ .../04324_read_in_order_virtual_row_union.sql | 73 +++++++++++++++++++ 5 files changed, 142 insertions(+), 21 deletions(-) create mode 100644 tests/queries/0_stateless/04324_read_in_order_virtual_row_union.reference create mode 100644 tests/queries/0_stateless/04324_read_in_order_virtual_row_union.sql diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index 78d72e3cf174..c0409b36d92d 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -135,15 +135,10 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) } #ifndef NDEBUG - /// Boundary is only meaningful when this merge applies the per-source virtual-row conversion; - /// otherwise `setVirtualRow` may fall back to default column values that the next chunk trips. - if (apply_virtual_row_conversions) + for (size_t source_num = 0; source_num < current_inputs.size(); ++source_num) { - for (size_t source_num = 0; source_num < current_inputs.size(); ++source_num) - { - if (current_inputs[source_num].skip_last_row && !has_collation) - rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); - } + if (current_inputs[source_num].skip_last_row && !has_collation) + rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); } #endif @@ -181,13 +176,10 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) #ifndef NDEBUG /// See `initialize` for why we gate on `apply_virtual_row_conversions`. - if (apply_virtual_row_conversions) - { - if (is_virtual_row && !has_collation) - rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); - else - checkVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num], description, source_num); - } + if (is_virtual_row && !has_collation) + rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); + else + checkVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num], description, source_num); #else UNUSED(rememberVirtualRowBoundary); UNUSED(checkVirtualRowBoundary); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index e969b99ebf4f..2c1412ec856e 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -1148,7 +1148,12 @@ InputOrder buildInputOrderFromUnorderedKeys( return order_info; } -InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, bool & apply_virtual_row, QueryPlan::Node & node, const QueryPlanOptimizationSettings & optimization_settings) +InputOrderInfoPtr buildInputOrderInfo( + SortingStep & sorting, + bool & apply_virtual_row, + ReadFromMergeTree *& virtual_row_reader, + QueryPlan::Node & node, + const QueryPlanOptimizationSettings & optimization_settings) { FindReadingStepContext find_reading_ctx{ .allow_existing_order = false, @@ -1188,11 +1193,14 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, bool & apply_virtua if (order_info.input_order) { - apply_virtual_row = order_info.virtual_row_conversion != std::nullopt; + apply_virtual_row = apply_virtual_row && order_info.virtual_row_conversion != std::nullopt; bool uses_virtual_row = false; if (order_info.virtual_row_conversion) + { uses_virtual_row = reading->setVirtualRowConversions(std::move(*order_info.virtual_row_conversion)); + virtual_row_reader = reading; + } if (!uses_virtual_row) { @@ -1609,7 +1617,8 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes, const if (sorting->hasPartitions() && !optimization_settings.reuse_storage_ordering_for_window_functions) return; - bool apply_virtual_row = false; + bool apply_virtual_row = true; + ReadFromMergeTree * virtual_row_reader = nullptr; if (auto * union_step = typeid_cast(node.children.front()->step.get())) { @@ -1631,9 +1640,13 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes, const return; } + std::vector virtual_row_readers; for (auto * child : union_node->children) { - infos.push_back(buildInputOrderInfo(*sorting, apply_virtual_row, *child, optimization_settings)); + ReadFromMergeTree * child_virtual_row_reader = nullptr; + infos.push_back(buildInputOrderInfo(*sorting, apply_virtual_row, child_virtual_row_reader, *child, optimization_settings)); + if (child_virtual_row_reader) + virtual_row_readers.push_back(child_virtual_row_reader); if (infos.back()) { @@ -1682,15 +1695,23 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes, const sort_node.step = std::move(additional_sorting); sort_node.children.push_back(child); child = &sort_node; + apply_virtual_row = false; } } + /// Virtual rows were enabled per child before the union-wide decision, undo them if the merge cannot use them. + if (!apply_virtual_row) + { + for (auto * reader : virtual_row_readers) + reader->resetVirtualRowConversions(); + } + /// FinishSorting's `MergingSortedTransform` requires every input stream of the union /// to be sorted by `max_sort_descr`; the union must not concatenate (narrow) them. union_step->disableNarrowing(); - sorting->convertToFinishSorting(*max_sort_descr, use_buffering, false); + sorting->convertToFinishSorting(*max_sort_descr, use_buffering, apply_virtual_row); } - else if (auto order_info = buildInputOrderInfo(*sorting, apply_virtual_row, *node.children.front(), optimization_settings)) + else if (auto order_info = buildInputOrderInfo(*sorting, apply_virtual_row, virtual_row_reader, *node.children.front(), optimization_settings)) { /// Use buffering only if have filter or don't have limit. bool use_buffering = order_info->limit == 0; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index e0990c8a0b16..61b274850676 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -330,6 +330,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter /// Returns `false` if requested reading cannot be performed. bool requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit, size_t query_limit = 0); bool setVirtualRowConversions(ActionsDAG virtual_row_conversion_); + void resetVirtualRowConversions() { virtual_row_conversion = nullptr; } bool readsInOrder() const; const InputOrderInfoPtr & getInputOrder() const { return query_info.input_order_info; } const SortDescription & getSortDescription() const override { return result_sort_description; } diff --git a/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.reference b/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.reference new file mode 100644 index 000000000000..31a8057424eb --- /dev/null +++ b/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.reference @@ -0,0 +1,34 @@ +-- union of in-order branches, DESC +100 d +60 c +50 b +-50 a +-- union of in-order branches, DESC, virtual row per block +100 d +60 c +50 b +-50 a +-- union of in-order branches with negative boundary, ASC +-100 a +-60 c +-50 b +50 d +-- union of in-order branches with negative boundary, ASC, virtual row per block +-100 a +-60 c +-50 b +50 d +-- union with a branch sorted additionally, DESC +100 d +70 f +60 c +50 b +-50 a +-60 e +-- union with a branch sorted additionally, ASC +-60 e +-50 a +50 b +60 c +70 f +100 d diff --git a/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.sql b/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.sql new file mode 100644 index 000000000000..c39f45a0cb69 --- /dev/null +++ b/tests/queries/0_stateless/04324_read_in_order_virtual_row_union.sql @@ -0,0 +1,73 @@ +DROP TABLE IF EXISTS t_in_order; +DROP TABLE IF EXISTS t_in_order_neg; +DROP TABLE IF EXISTS t_no_order; + +CREATE TABLE t_in_order (timestamp Int64, id String) ENGINE = MergeTree ORDER BY timestamp; +CREATE TABLE t_in_order_neg (timestamp Int64, id String) ENGINE = MergeTree ORDER BY timestamp; +CREATE TABLE t_no_order (timestamp Int64, id String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_in_order VALUES (-50, 'a'), (50, 'b'), (60, 'c'), (100, 'd'); +INSERT INTO t_in_order_neg VALUES (-100, 'a'), (-60, 'c'), (-50, 'b'), (50, 'd'); +INSERT INTO t_no_order VALUES (-60, 'e'), (70, 'f'); + +SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1; + +SELECT '-- union of in-order branches, DESC'; +WITH A AS +( + SELECT * FROM t_in_order WHERE timestamp != 60 + UNION ALL + SELECT * FROM t_in_order WHERE timestamp = 60 +) +SELECT timestamp, id FROM A ORDER BY timestamp DESC LIMIT 10; + +SELECT '-- union of in-order branches, DESC, virtual row per block'; +WITH A AS +( + SELECT * FROM t_in_order WHERE timestamp != 60 + UNION ALL + SELECT * FROM t_in_order WHERE timestamp = 60 +) +SELECT timestamp, id FROM A ORDER BY timestamp DESC LIMIT 10 +SETTINGS read_in_order_use_virtual_row_per_block = 1; + +SELECT '-- union of in-order branches with negative boundary, ASC'; +WITH A AS +( + SELECT * FROM t_in_order_neg WHERE timestamp != -60 + UNION ALL + SELECT * FROM t_in_order_neg WHERE timestamp = -60 +) +SELECT timestamp, id FROM A ORDER BY timestamp ASC LIMIT 10; + +SELECT '-- union of in-order branches with negative boundary, ASC, virtual row per block'; +WITH A AS +( + SELECT * FROM t_in_order_neg WHERE timestamp != -60 + UNION ALL + SELECT * FROM t_in_order_neg WHERE timestamp = -60 +) +SELECT timestamp, id FROM A ORDER BY timestamp ASC LIMIT 10 +SETTINGS read_in_order_use_virtual_row_per_block = 1; + +SELECT '-- union with a branch sorted additionally, DESC'; +WITH A AS +( + SELECT * FROM t_in_order + UNION ALL + SELECT * FROM t_no_order +) +SELECT timestamp, id FROM A ORDER BY timestamp DESC LIMIT 10; + +SELECT '-- union with a branch sorted additionally, ASC'; +WITH A AS +( + SELECT * FROM t_in_order + UNION ALL + SELECT * FROM t_no_order +) +SELECT timestamp, id FROM A ORDER BY timestamp ASC LIMIT 10; + +DROP TABLE t_in_order; +DROP TABLE t_in_order_neg; +DROP TABLE t_no_order; From e2a00edb1c9fc5880c468186232a96d7580fbba4 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Mon, 20 Jul 2026 12:23:04 +0200 Subject: [PATCH 05/86] Remove unsupported `explain_query_plan_default` from test The setting is not available on the backport branches and is not required by the explicit `EXPLAIN` query. https://github.com/ClickHouse/clickhouse-private/pull/64824 https://github.com/ClickHouse/ClickHouse/pull/110800 https://github.com/ClickHouse/clickhouse-private/pull/64825 --- .../0_stateless/04513_prewhere_map_subcolumn_read_cost.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql index 602bd35806d4..80110f6e1cf6 100644 --- a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql +++ b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql @@ -13,7 +13,6 @@ SET optimize_move_to_prewhere = 1; SET query_plan_optimize_prewhere = 1; SET allow_reorder_prewhere_conditions = 1; SET use_statistics = 0; -SET explain_query_plan_default = 'legacy'; DROP TABLE IF EXISTS t_prewhere_map_cost; CREATE TABLE t_prewhere_map_cost (id UInt64, modality LowCardinality(String), h Map(String, String)) From 993c5057a20d0bd6f6f797fd0df7b0715975e851 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 13:44:10 +0000 Subject: [PATCH 06/86] Backport #109532 to 26.6: Fix rollup with materialized and virtual columns --- .../TTL/TTLAggregationAlgorithm.cpp | 56 +++++++++++++++++-- src/Storages/MergeTree/MergeTask.cpp | 12 ++-- src/Storages/TTLDescription.cpp | 28 ---------- ...group_by_ttl_block_number_offset.reference | 14 +++++ ...04501_group_by_ttl_block_number_offset.sql | 36 ++++++++++++ ...group_by_ttl_materialized_column.reference | 9 +++ ...04502_group_by_ttl_materialized_column.sql | 33 +++++++++++ 7 files changed, 152 insertions(+), 36 deletions(-) create mode 100644 tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.reference create mode 100644 tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.sql create mode 100644 tests/queries/0_stateless/04502_group_by_ttl_materialized_column.reference create mode 100644 tests/queries/0_stateless/04502_group_by_ttl_materialized_column.sql diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index 9c640ef0a3a0..27f7448a87d5 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -1,8 +1,13 @@ -#include -#include +#include + #include #include -#include + +#include + +#include + +#include #include @@ -27,6 +32,49 @@ namespace Setting extern const SettingsBool serialize_string_in_memory_with_zero_byte; } +namespace +{ + +bool isCoveredByGroupByOrSet(const TTLDescription & description, const std::string & column_name) +{ + return std::ranges::contains(description.group_by_keys, column_name) + || std::ranges::contains(description.set_parts | std::views::transform(&TTLAggregateDescription::column_name), column_name); +} + +std::pair prepareAnyAggregate(const ColumnWithTypeAndName & column, const ContextPtr & context) +{ + AggregateDescription aggregate; + aggregate.column_name = column.name; + aggregate.argument_names = {column.name}; + AggregateFunctionProperties properties; + aggregate.function = AggregateFunctionFactory::instance().get("any", NullsAction::EMPTY, {column.type}, {}, properties); + + TTLAggregateDescription set_part; + set_part.column_name = column.name; + set_part.expression_result_column_name = column.name; + set_part.expression = std::make_shared(ActionsDAG(NamesAndTypesList{{column.name, aggregate.function->getResultType()}}), ExpressionActionsSettings(context)); + + return {std::move(aggregate), std::move(set_part)}; +} + +TTLDescription addImplicitlyAggregatedColumns(TTLDescription description, const Block & header, const ContextPtr & context) +{ + for (const auto & column : header) + { + if (isCoveredByGroupByOrSet(description, column.name)) + continue; + + auto [aggregate, set_part] = prepareAnyAggregate(column, context); + + description.aggregate_descriptions.push_back(std::move(aggregate)); + description.set_parts.push_back(std::move(set_part)); + } + + return description; +} + +} + TTLAggregationAlgorithm::TTLAggregationAlgorithm( const TTLExpressions & ttl_expressions_, const TTLDescription & description_, @@ -35,7 +83,7 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( bool force_, const Block & header_, const MergeTreeData & storage_) - : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) + : ITTLAlgorithm(ttl_expressions_, addImplicitlyAggregatedColumns(description_, header_, storage_.getContext()), old_ttl_info_, current_time_, force_) , header(header_) { current_key_value.resize(description.group_by_keys.size()); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5e7d3c71435b..16edb0c10a8d 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -979,14 +979,18 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const bool MergeTask::enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) { - return (*global_ctx->data_settings)[MergeTreeSetting::enable_block_number_column] - && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + if (global_ctx->parent_part) + return false; + + return (*global_ctx->data_settings)[MergeTreeSetting::enable_block_number_column]; } bool MergeTask::enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) { - return (*global_ctx->data_settings)[MergeTreeSetting::enable_block_offset_column] - && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + if (global_ctx->parent_part) + return false; + + return (*global_ctx->data_settings)[MergeTreeSetting::enable_block_offset_column]; } void MergeTask::addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 1bb3587b28c2..e0d766a266cb 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -271,14 +271,11 @@ TTLDescription TTLDescription::getTTLFromAST( throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key"); NameSet aggregation_columns_set; - NameSet used_primary_key_columns_set; for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i) { if (ttl_element->group_by_key[i]->getColumnName() != pk_columns[i]) throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key {} {}", ttl_element->group_by_key[i]->getColumnName(), pk_columns[i]); - - used_primary_key_columns_set.insert(pk_columns[i]); } std::vector> aggregations; @@ -304,31 +301,6 @@ TTLDescription TTLDescription::getTTLFromAST( result.group_by_keys = Names(pk_columns.begin(), pk_columns.begin() + ttl_element->group_by_key.size()); - const auto & primary_key_expressions = primary_key.expression_list_ast->children; - - /// Wrap with 'any' aggregate function primary key columns, - /// which are not in 'GROUP BY' key and was not set explicitly. - /// The separate step, because not all primary key columns are ordinary columns. - for (size_t i = ttl_element->group_by_key.size(); i < primary_key_expressions.size(); ++i) - { - if (!aggregation_columns_set.contains(pk_columns[i])) - { - ASTPtr expr = makeASTFunction("any", primary_key_expressions[i]->clone()); - aggregations.emplace_back(pk_columns[i], std::move(expr)); - aggregation_columns_set.insert(pk_columns[i]); - } - } - - /// Wrap with 'any' aggregate function other columns, which was not set explicitly. - for (const auto & column : columns.getOrdinary()) - { - if (!aggregation_columns_set.contains(column.name) && !used_primary_key_columns_set.contains(column.name)) - { - ASTPtr expr = makeASTFunction("any", make_intrusive(column.name)); - aggregations.emplace_back(column.name, std::move(expr)); - } - } - for (auto [name, value] : aggregations) { auto syntax_result = TreeRewriter(context).analyze(value, columns.getAllPhysical(), {}, {}, true); diff --git a/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.reference b/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.reference new file mode 100644 index 000000000000..d1596162c348 --- /dev/null +++ b/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.reference @@ -0,0 +1,14 @@ +before +1 0 2001-09-18 10:03:30.000 17349 +2 0 2001-09-18 10:03:30.000 17349 +3 0 2001-09-18 10:03:30.000 17349 +4 0 2001-09-18 10:03:30.000 17349 +after merge +1 0 2001-09-18 10:03:30.000 17349 +2 0 2001-09-18 10:03:30.000 17349 +3 0 2001-09-18 10:03:30.000 17349 +4 0 2001-09-18 10:03:30.000 17349 +after ttl +1 0 2001-09-18 10:03:30.000 17349 +after rewrite +1 0 2001-09-18 10:03:30.000 17349 diff --git a/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.sql b/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.sql new file mode 100644 index 000000000000..e6a1aa9ec1b2 --- /dev/null +++ b/tests/queries/0_stateless/04501_group_by_ttl_block_number_offset.sql @@ -0,0 +1,36 @@ +-- Rows rolled up by `GROUP BY` TTL get `any` values of the persisted virtual columns +-- `_block_number` / `_block_offset` instead of an exception during the merge or the mutation. + +SET mutations_sync = 2; +SET materialize_ttl_after_modify = 1; + +DROP TABLE IF EXISTS t_ttl_group_by_block_columns SYNC; + +CREATE TABLE t_ttl_group_by_block_columns (v1 DateTime64(3), v2 Int16) +ENGINE = MergeTree ORDER BY v1 +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, min_bytes_for_wide_part = 0; + +INSERT INTO t_ttl_group_by_block_columns (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_block_columns (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_block_columns (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_block_columns (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); + +SELECT 'before'; +SELECT _block_number, _block_offset, * FROM t_ttl_group_by_block_columns ORDER BY ALL; + +OPTIMIZE TABLE t_ttl_group_by_block_columns FINAL; + +SELECT 'after merge'; +SELECT _block_number, _block_offset, * FROM t_ttl_group_by_block_columns ORDER BY ALL; + +ALTER TABLE t_ttl_group_by_block_columns MODIFY TTL toStartOfDay(v1) + INTERVAL 1 DAY GROUP BY v1; + +SELECT 'after ttl'; +SELECT _block_number, _block_offset, * FROM t_ttl_group_by_block_columns ORDER BY ALL; + +ALTER TABLE t_ttl_group_by_block_columns REWRITE PARTS; + +SELECT 'after rewrite'; +SELECT _block_number, _block_offset, * FROM t_ttl_group_by_block_columns ORDER BY ALL; + +DROP TABLE t_ttl_group_by_block_columns; diff --git a/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.reference b/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.reference new file mode 100644 index 000000000000..15fa08119174 --- /dev/null +++ b/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.reference @@ -0,0 +1,9 @@ +before +34698 2001-09-18 10:03:30.000 17349 +34698 2001-09-18 10:03:30.000 17349 +34698 2001-09-18 10:03:30.000 17349 +34698 2001-09-18 10:03:30.000 17349 +after merge +34698 2001-09-18 10:03:30.000 17349 +after rewrite +34698 2001-09-18 10:03:30.000 17349 diff --git a/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.sql b/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.sql new file mode 100644 index 000000000000..b742673f5e3a --- /dev/null +++ b/tests/queries/0_stateless/04502_group_by_ttl_materialized_column.sql @@ -0,0 +1,33 @@ +-- Rows rolled up by `GROUP BY` TTL get `any` values of MATERIALIZED columns +-- instead of an exception during the merge or the mutation. + +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_ttl_group_by_materialized SYNC; + +CREATE TABLE t_ttl_group_by_materialized (v1 DateTime64(3), v2 Int16, m UInt64 MATERIALIZED v2 * 2) +ENGINE = MergeTree ORDER BY v1 +TTL toStartOfDay(v1) + INTERVAL 1 DAY GROUP BY v1 +SETTINGS min_bytes_for_wide_part = 0; + +SYSTEM STOP TTL MERGES t_ttl_group_by_materialized; +INSERT INTO t_ttl_group_by_materialized (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_materialized (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_materialized (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); +INSERT INTO t_ttl_group_by_materialized (v1, v2) VALUES (toDateTime('2001-09-18 10:03:30'), 17349); + +SELECT 'before'; +SELECT m, * FROM t_ttl_group_by_materialized ORDER BY ALL; + +SYSTEM START TTL MERGES t_ttl_group_by_materialized; +OPTIMIZE TABLE t_ttl_group_by_materialized FINAL; + +SELECT 'after merge'; +SELECT m, * FROM t_ttl_group_by_materialized ORDER BY ALL; + +ALTER TABLE t_ttl_group_by_materialized REWRITE PARTS; + +SELECT 'after rewrite'; +SELECT m, * FROM t_ttl_group_by_materialized ORDER BY ALL; + +DROP TABLE t_ttl_group_by_materialized; From c78c3cb0fe193f588866d905855a7e384ffe61a5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 13:44:38 +0000 Subject: [PATCH 07/86] Backport #107719 to 26.6: Fix Block structure mismatch in UnionStep/IntersectOrExceptStep when a branch constant-folds --- src/Interpreters/InterpreterSelectQuery.cpp | 26 ++- .../QueryPlan/IntersectOrExceptStep.cpp | 70 ++++++- .../Optimizations/filterPushDown.cpp | 11 + .../QueryPlan/Optimizations/liftUpUnion.cpp | 9 + src/Processors/QueryPlan/UnionStep.cpp | 60 +++++- ...ion_branch_const_header_mismatch.reference | 20 ++ ...327_union_branch_const_header_mismatch.sql | 191 ++++++++++++++++++ 7 files changed, 377 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/04327_union_branch_const_header_mismatch.reference create mode 100644 tests/queries/0_stateless/04327_union_branch_const_header_mismatch.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 080bc6fbcc1a..42a79abffda2 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1204,12 +1204,34 @@ void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan) { executeImpl(query_plan, std::move(input_pipe)); + /// The old analyzer computes result_header from the ExpressionAnalyzer sample block, + /// independently of the actual query plan. A plan step may legitimately materialize a + /// column the sample considered constant (e.g. a remote(...) UnionStep whose shards fold + /// randConstant() to different values). Forcing such a full column back to a Const in the + /// convert target would fail in makeConvertingActions. Reconcile a local convert target: + /// where a column is full in the plan but Const of the same type in the result header, + /// take the plan's full column so the conversion keeps it full. result_header itself is + /// left untouched so getSampleBlock() (seen by an enclosing subquery) is unchanged. + ColumnsWithTypeAndName convert_target = result_header->getColumnsWithTypeAndName(); + { + const auto & plan_header = *query_plan.getCurrentHeader(); + for (auto & res_col : convert_target) + { + if (!res_col.column || !isColumnConst(*res_col.column)) + continue; + const auto * plan_col = plan_header.findByName(res_col.name); + if (plan_col && plan_col->column && !isColumnConst(*plan_col->column) + && res_col.type->equals(*plan_col->type)) + res_col.column = plan_col->column; + } + } + /// We must guarantee that result structure is the same as in getSampleBlock() - if (!blocksHaveEqualStructure(*query_plan.getCurrentHeader(), *result_header)) + if (!blocksHaveEqualStructure(*query_plan.getCurrentHeader(), Block(convert_target))) { auto convert_actions_dag = ActionsDAG::makeConvertingActions( query_plan.getCurrentHeader()->getColumnsWithTypeAndName(), - result_header->getColumnsWithTypeAndName(), + convert_target, ActionsDAG::MatchColumnsMode::Name, context, true); diff --git a/src/Processors/QueryPlan/IntersectOrExceptStep.cpp b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp index 009e25464eae..73dd4b64ff1d 100644 --- a/src/Processors/QueryPlan/IntersectOrExceptStep.cpp +++ b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp @@ -1,5 +1,9 @@ #include +#include +#include +#include +#include #include #include #include @@ -22,11 +26,61 @@ static SharedHeader checkHeaders(const SharedHeaders & input_headers) if (input_headers.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot perform intersect/except on empty set of query plan steps"); - SharedHeader res = input_headers.front(); + /// Branches are optimized independently, so filter push-down may constant-fold a + /// column in one branch but not its sibling. Tolerate exactly that: compare with the + /// top-level Const stripped, which keeps the check otherwise strict (different types + /// and divergent Sparse/Replicated wrappers are still rejected). The strip is guarded + /// by isColumnConst because convertToFullColumnIfConst is broader for some columns + /// (e.g. ColumnArray materializes const nested data too), and the conversion path at + /// execution time only reconciles a top-level Const. Header columns may legitimately + /// have a null column pointer (e.g. __grouping_set from WithMergeableState); leave + /// those untouched, names and types are still validated below. + auto without_top_level_const = [](const Block & header) + { + ColumnsWithTypeAndName columns = header.getColumnsWithTypeAndName(); + for (auto & column : columns) + if (column.column && isColumnConst(*column.column)) + column.column = column.column->convertToFullColumnIfConst(); + return Block(std::move(columns)); + }; + + Block reference = without_top_level_const(*input_headers.front()); for (const auto & header : input_headers) - assertBlocksHaveEqualStructure(*header, *res, "IntersectOrExceptStep"); + assertBlocksHaveEqualStructure(without_top_level_const(*header), reference, "IntersectOrExceptStep"); + + /// Build the common header following the same rule as getLeastSuperColumn: keep a + /// column Const only when every branch is Const with the same value, otherwise + /// materialize it. This matches the execution-time makeConvertingActions path, which + /// can convert a branch to a full column but not to a different branch's Const value. + ColumnsWithTypeAndName common = input_headers.front()->getColumnsWithTypeAndName(); + bool materialized = false; + for (size_t col = 0; col < common.size(); ++col) + { + if (!common[col].column || !isColumnConst(*common[col].column)) + continue; + + const Field value = assert_cast(*common[col].column).getField(); + bool keep_const = true; + for (const auto & header : input_headers) + { + const auto & branch = header->getByPosition(col).column; + if (!branch || !isColumnConst(*branch) || assert_cast(*branch).getField() != value) + { + keep_const = false; + break; + } + } + + if (!keep_const) + { + common[col].column = common[col].column->convertToFullColumnIfConst(); + materialized = true; + } + } - return res; + if (!materialized) + return input_headers.front(); + return std::make_shared(std::move(common)); } IntersectOrExceptStep::IntersectOrExceptStep( @@ -56,8 +110,14 @@ QueryPipelineBuilderPtr IntersectOrExceptStep::updatePipeline(QueryPipelineBuild for (auto & cur_pipeline : pipelines) { - /// Just in case. - if (!isCompatibleHeader(cur_pipeline->getHeader(), *getOutputHeader())) + /// The check must be strict about constness (blocksHaveEqualStructure, not + /// isCompatibleHeader): when a branch constant-folds, the common header + /// materializes the column, and the converting expression must be applied to + /// every stream of the branch pipeline - including the totals and extremes + /// ports, which addSimpleTransform covers but the main-stream processors + /// added below do not. Otherwise a Const totals port survives next to full + /// main streams and fails the per-stream structure check downstream. + if (!blocksHaveEqualStructure(cur_pipeline->getHeader(), *getOutputHeader())) { QueryPipelineProcessorsCollector collector(*cur_pipeline, this); auto converting_dag = ActionsDAG::makeConvertingActions( diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 16ce49dfba29..98ba6cc62cad 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -1134,6 +1135,16 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * union_step = typeid_cast(child.get())) { + /// This rewrite forces every union branch input header to the pushed-down filter's + /// output header, which assumes the union forwards each branch unchanged. Skip it + /// when the union normalizes a branch (its output differs from some input header), + /// e.g. it drops a Const that diverged across branches. Otherwise a branch still + /// outputting Const would get a full input header and the mismatch would move here. + const auto & union_output = *union_step->getOutputHeader(); + for (const auto & input_header : union_step->getInputHeaders()) + if (!blocksHaveEqualStructure(*input_header, union_output)) + return 0; + /// Union does not change header. /// We can push down filter and update header. auto union_input_headers = child->getInputHeaders(); diff --git a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp index 3cdd0e31c3e7..a5c1f0a2c1ac 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB::QueryPlanOptimizations { @@ -20,6 +21,14 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, c if (!union_step) return 0; + /// Both rewrites below assume the union forwards each branch unchanged. Skip them when + /// the union normalizes a branch (output differs from some input header), e.g. it drops + /// a Const that diverged across branches. + const auto & union_output = *union_step->getOutputHeader(); + for (const auto & input_header : union_step->getInputHeaders()) + if (!blocksHaveEqualStructure(*input_header, union_output)) + return 0; + if (auto * expression = typeid_cast(parent.get())) { /// Union does not change header. diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index fc7437c65e40..a7a1e1d85e31 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -1,4 +1,8 @@ +#include +#include #include +#include +#include #include #include #include @@ -23,11 +27,61 @@ static SharedHeader checkHeaders(const SharedHeaders & input_headers) if (input_headers.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unite an empty set of query plan steps"); - auto res = input_headers.front(); + /// Branches are optimized independently, so filter push-down may constant-fold a + /// column in one branch but not its sibling. Tolerate exactly that: compare with the + /// top-level Const stripped, which keeps the check otherwise strict (different types + /// and divergent Sparse/Replicated wrappers are still rejected). The strip is guarded + /// by isColumnConst because convertToFullColumnIfConst is broader for some columns + /// (e.g. ColumnArray materializes const nested data too), and the conversion path at + /// execution time only reconciles a top-level Const. Header columns may legitimately + /// have a null column pointer (e.g. __grouping_set from WithMergeableState); leave + /// those untouched, names and types are still validated below. + auto without_top_level_const = [](const Block & header) + { + ColumnsWithTypeAndName columns = header.getColumnsWithTypeAndName(); + for (auto & column : columns) + if (column.column && isColumnConst(*column.column)) + column.column = column.column->convertToFullColumnIfConst(); + return Block(std::move(columns)); + }; + + Block reference = without_top_level_const(*input_headers.front()); for (const auto & header : input_headers) - assertBlocksHaveEqualStructure(*header, *res, "UnionStep"); + assertBlocksHaveEqualStructure(without_top_level_const(*header), reference, "UnionStep"); + + /// Build the common header following the same rule as getLeastSuperColumn: keep a + /// column Const only when every branch is Const with the same value, otherwise + /// materialize it. This matches the execution-time makeConvertingActions path, which + /// can convert a branch to a full column but not to a different branch's Const value. + ColumnsWithTypeAndName common = input_headers.front()->getColumnsWithTypeAndName(); + bool materialized = false; + for (size_t col = 0; col < common.size(); ++col) + { + if (!common[col].column || !isColumnConst(*common[col].column)) + continue; + + const Field value = assert_cast(*common[col].column).getField(); + bool keep_const = true; + for (const auto & header : input_headers) + { + const auto & branch = header->getByPosition(col).column; + if (!branch || !isColumnConst(*branch) || assert_cast(*branch).getField() != value) + { + keep_const = false; + break; + } + } + + if (!keep_const) + { + common[col].column = common[col].column->convertToFullColumnIfConst(); + materialized = true; + } + } - return res; + if (!materialized) + return input_headers.front(); + return std::make_shared(std::move(common)); } UnionStep::UnionStep(SharedHeaders input_headers_, size_t max_threads_, bool allow_narrowing_) diff --git a/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.reference b/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.reference new file mode 100644 index 000000000000..9523fd913dd2 --- /dev/null +++ b/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.reference @@ -0,0 +1,20 @@ +1 1 +1 1 +1 1 +1 1 +1 1 +1 0 +1 1 +1 0 +1 1 +1 1 +4 +4 +4 +1 +1 +1 + +\N \N + +\N \N diff --git a/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.sql b/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.sql new file mode 100644 index 000000000000..757f18a6881b --- /dev/null +++ b/tests/queries/0_stateless/04327_union_branch_const_header_mismatch.sql @@ -0,0 +1,191 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/106956 + +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +UNION ALL +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +UNION DISTINCT +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +SELECT t2.a, CAST(t1.b = t2.b, 'Nullable(UInt8)') +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +UNION ALL +SELECT t2.a, CAST(t1.b = t2.b, 'Nullable(UInt8)') +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +SELECT DISTINCT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +UNION ALL +SELECT DISTINCT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +-- The same divergence reaches IntersectOrExceptStep. +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +INTERSECT +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE t1.b = t2.b +EXCEPT +SELECT t2.a, t1.b = t2.b +FROM (SELECT 1 AS a, 2 AS b) AS t1 +INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a +WHERE NOT (t1.b = t2.b); + +-- Both branches keep rows but fold the same column to different constants (1 and 0). +SELECT a, c FROM ( + SELECT t2.a AS a, (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b + UNION ALL + SELECT t2.a AS a, NOT (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b +) ORDER BY a, c; + +SELECT a, c FROM ( + SELECT t2.a AS a, (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b + UNION DISTINCT + SELECT t2.a AS a, NOT (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b +) ORDER BY a, c; + +SELECT a, c FROM ( + SELECT t2.a AS a, (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b + INTERSECT + SELECT t2.a AS a, NOT (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b +) ORDER BY a, c; + +SELECT a, c FROM ( + SELECT t2.a AS a, (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b + EXCEPT + SELECT t2.a AS a, NOT (t1.b = t2.b) AS c + FROM (SELECT 1 AS a, 2 AS b) AS t1 + INNER JOIN (SELECT 1 AS a, 2 AS b) AS t2 ON t1.a = t2.a + WHERE t1.b = t2.b +) ORDER BY a, c; + +-- https://github.com/ClickHouse/ClickHouse/issues/107951 (site 11). A second path to the same +-- IntersectOrExceptStep divergence: one branch folds a column to Const(Nullable(Nothing)) (here via +-- QUALIFY + LIMIT 0 over a const-NULL CTE) while the sibling keeps the full Nullable(Nothing) column. +-- QUALIFY is only implemented in the Analyzer, so pin it (CI randomizes enable_analyzer). +WITH cte AS (SELECT DISTINCT NULL WHERE isNullable('') GROUP BY 1) +SELECT DISTINCT *, toNullable(NULL) FROM cte QUALIFY materialize(100) LIMIT 0 +INTERSECT DISTINCT +SELECT toNullable(NULL), * FROM cte +SETTINGS enable_analyzer = 1; + +-- Filter push-down rebuilds the UnionStep assuming it forwards every branch header +-- unchanged. With a divergent Const across shards (each remote shard folds randConstant() +-- to a different value, so the union materializes the column) a pushed-down branch filter +-- would still output Const and the mismatch would move into the filter-pushdown rewrite. +-- count() keeps the output deterministic (randConstant() varies per shard). +SELECT count() FROM ( + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) + UNION ALL + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) +) WHERE c >= 0 +SETTINGS enable_analyzer = 0; + +-- Parent set-operation interpreters under the old analyzer cache result_header from the child +-- sample blocks before any child plan is built. A child whose plan materializes the constant +-- (the remote(...) union above) must still pass the parent's "Conversion before UNION" cleanly: +-- branch headers only diverge during plan optimization, which runs after the parent interpreters +-- have decided their conversions, so no full-vs-Const conversion is requested at plan-build time. +SELECT count() FROM ( + SELECT serverUUID() AS c FROM remote('127.0.0.{1,2}', system.one) + UNION ALL + SELECT serverUUID() AS c FROM remote('127.0.0.{1,2}', system.one) +) SETTINGS enable_analyzer = 0; + +SELECT count() FROM ( + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) + UNION ALL + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) +) SETTINGS enable_analyzer = 0; + +-- The value of these set operations depends on random constants colliding or not, so only +-- assert that they execute (the plan-time header checks are what used to throw). +SELECT count() >= 0 FROM ( + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) + INTERSECT ALL + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) +) SETTINGS enable_analyzer = 0; + +SELECT count() >= 0 FROM ( + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) + EXCEPT ALL + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) +) SETTINGS enable_analyzer = 0; + +SELECT count() >= 1 FROM ( + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) + UNION DISTINCT + SELECT toUInt32(42) AS c FROM remote('127.0.0.{1,2}', system.one) + UNION ALL + SELECT randConstant() AS c FROM remote('127.0.0.{1,2}', system.one) +) SETTINGS enable_analyzer = 0; + +-- AST fuzzer (STID 0993-2a62). Same Const-vs-full divergence, but the branch pipelines carry a +-- totals port (WITH TOTALS in the CTE). IntersectOrExceptStep::updatePipeline must apply the +-- converting expression whenever the branch header differs from the output header in constness +-- (strict blocksHaveEqualStructure, not isCompatibleHeader): addSimpleTransform converts the +-- totals port too, while the main-stream processors below it do not, so skipping the conversion +-- left a Const totals port next to materialized full main streams and the per-stream structure +-- check in a downstream DistinctStep threw a logical error. +WITH cte AS (SELECT DISTINCT NULL WHERE isNullable('') GROUP BY 1 WITH TOTALS) +SELECT DISTINCT *, toNullable(NULL) FROM cte QUALIFY materialize(100) LIMIT 0 +INTERSECT DISTINCT +SELECT *, NULL FROM cte GROUP BY ALL +SETTINGS enable_analyzer = 1; + +WITH cte AS (SELECT DISTINCT NULL WHERE isNullable('') GROUP BY 1 WITH TOTALS) +SELECT DISTINCT *, toNullable(NULL) FROM cte QUALIFY materialize(100) LIMIT 0 +EXCEPT DISTINCT +SELECT *, NULL FROM cte GROUP BY ALL +SETTINGS enable_analyzer = 1; From a255e086f7728b74aa857e4174ddac9ee75ca43d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 14:37:37 +0000 Subject: [PATCH 08/86] Backport #109410 to 26.6: Fix infinite rollup --- .../TTL/TTLAggregationAlgorithm.cpp | 4 +-- src/Processors/TTL/TTLColumnAlgorithm.cpp | 2 +- src/Processors/TTL/TTLDeleteAlgorithm.cpp | 2 +- src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp | 8 +++--- .../MergeSelectors/TTLMergeSelector.cpp | 5 +++- .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 24 ++++++++-------- .../MergeTree/MergeTreeDataPartTTLInfo.h | 16 +++++++---- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- ...501_group_by_ttl_infinite_rollup.reference | 3 ++ .../04501_group_by_ttl_infinite_rollup.sql | 27 ++++++++++++++++++ ...p_by_ttl_merge_of_finished_parts.reference | 1 + ...2_group_by_ttl_merge_of_finished_parts.sql | 28 +++++++++++++++++++ 12 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.reference create mode 100644 tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.sql create mode 100644 tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.reference create mode 100644 tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.sql diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index 9c640ef0a3a0..9a7cc026b683 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -276,11 +276,11 @@ void TTLAggregationAlgorithm::finalize(const MutableDataPartPtr & data_part) con if (new_ttl_info.finished()) { data_part->ttl_infos.group_by_ttl[description.result_column] = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); return; } data_part->ttl_infos.group_by_ttl[description.result_column] = old_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(old_ttl_info.min, old_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(old_ttl_info); } } diff --git a/src/Processors/TTL/TTLColumnAlgorithm.cpp b/src/Processors/TTL/TTLColumnAlgorithm.cpp index 88a4a26cae35..91cb9f5080fe 100644 --- a/src/Processors/TTL/TTLColumnAlgorithm.cpp +++ b/src/Processors/TTL/TTLColumnAlgorithm.cpp @@ -88,7 +88,7 @@ void TTLColumnAlgorithm::execute(Block & block) void TTLColumnAlgorithm::finalize(const MutableDataPartPtr & data_part) const { data_part->ttl_infos.columns_ttl[column_name] = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); if (is_fully_empty) data_part->expired_columns.insert(column_name); } diff --git a/src/Processors/TTL/TTLDeleteAlgorithm.cpp b/src/Processors/TTL/TTLDeleteAlgorithm.cpp index 0cdfadd45283..323d92d7f85b 100644 --- a/src/Processors/TTL/TTLDeleteAlgorithm.cpp +++ b/src/Processors/TTL/TTLDeleteAlgorithm.cpp @@ -63,7 +63,7 @@ void TTLDeleteAlgorithm::finalize(const MutableDataPartPtr & data_part) const else data_part->ttl_infos.table_ttl = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); } } diff --git a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp index 2bf098a385df..0ad85213e3bd 100644 --- a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp +++ b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp @@ -43,22 +43,22 @@ void TTLUpdateInfoAlgorithm::finalize(const MutableDataPartPtr & data_part) cons else if (ttl_update_field == TTLUpdateField::GROUP_BY_TTL) { data_part->ttl_infos.group_by_ttl[ttl_update_key] = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); } else if (ttl_update_field == TTLUpdateField::ROWS_WHERE_TTL) { data_part->ttl_infos.rows_where_ttl[ttl_update_key] = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); } else if (ttl_update_field == TTLUpdateField::TABLE_TTL) { data_part->ttl_infos.table_ttl = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); } else if (ttl_update_field == TTLUpdateField::COLUMNS_TTL) { data_part->ttl_infos.columns_ttl[ttl_update_key] = new_ttl_info; - data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max); + data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info); } } diff --git a/src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.cpp b/src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.cpp index 6da1b6e6c184..e78f1096a943 100644 --- a/src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.cpp +++ b/src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.cpp @@ -230,7 +230,10 @@ time_t TTLPartDropMergeSelector::getTTLForPart(const PartProperties & part) cons bool TTLPartDropMergeSelector::canConsiderPart(const PartProperties & part) const { - return part.general_ttl_info.has_value(); + if (!part.general_ttl_info.has_value()) + return false; + + return part.general_ttl_info->has_any_non_finished_ttls; } TTLRowDeleteMergeSelector::TTLRowDeleteMergeSelector(const PartitionIdToTTLs & merge_due_times_, time_t current_time_) diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp index a1aacfc027e6..b897ebf5576c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp @@ -38,19 +38,20 @@ void MergeTreeDataPartTTLInfos::update(const MergeTreeDataPartTTLInfos & other_i for (const auto & [name, ttl_info] : other_infos.columns_ttl) { columns_ttl[name].update(ttl_info); - updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + updatePartMinMaxTTL(ttl_info); } for (const auto & [name, ttl_info] : other_infos.rows_where_ttl) { rows_where_ttl[name].update(ttl_info); - updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + updatePartMinMaxTTL(ttl_info); } for (const auto & [name, ttl_info] : other_infos.group_by_ttl) { - group_by_ttl[name].update(ttl_info); - updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + const MergeTreeDataPartTTLInfo not_finished_ttl_info{ .min = ttl_info.min, .max = ttl_info.max, .ttl_finished = false }; + group_by_ttl[name].update(not_finished_ttl_info); + updatePartMinMaxTTL(not_finished_ttl_info); } for (const auto & [name, ttl_info] : other_infos.recompression_ttl) @@ -60,7 +61,7 @@ void MergeTreeDataPartTTLInfos::update(const MergeTreeDataPartTTLInfos & other_i moves_ttl[expression].update(ttl_info); table_ttl.update(other_infos.table_ttl); - updatePartMinMaxTTL(table_ttl.min, table_ttl.max); + updatePartMinMaxTTL(table_ttl); } @@ -86,7 +87,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) String name = col["name"].getString(); columns_ttl.emplace(name, ttl_info); - updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + updatePartMinMaxTTL(ttl_info); } } if (json.has("table")) @@ -98,7 +99,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) if (table.has("finished")) table_ttl.ttl_finished = table["finished"].getUInt(); - updatePartMinMaxTTL(table_ttl.min, table_ttl.max); + updatePartMinMaxTTL(table_ttl); } auto fill_ttl_info_map = [this](const JSON & json_part, TTLInfoMap & ttl_info_map, bool update_min_max) @@ -116,7 +117,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) ttl_info_map.emplace(expression, ttl_info); if (update_min_max) - updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + updatePartMinMaxTTL(ttl_info); } }; @@ -248,14 +249,13 @@ bool MergeTreeDataPartTTLInfos::hasAnyNonFinishedTTLs() const auto has_non_finished_ttl = [] (const TTLInfoMap & map) -> bool { for (const auto & [name, info] : map) - { - if (!info.finished()) + if (info.initialized() && !info.finished()) return true; - } + return false; }; - if (!table_ttl.finished()) + if (table_ttl.initialized() && !table_ttl.finished()) return true; if (has_non_finished_ttl(columns_ttl)) diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index 9924b3ce6f06..06611a9154ed 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace DB { @@ -26,6 +26,7 @@ struct MergeTreeDataPartTTLInfo /// again for merge with multiple parts. std::optional ttl_finished; bool finished() const { return ttl_finished.value_or(false); } + bool initialized() const { return min != 0 || max != 0; } void update(time_t time); void update(const MergeTreeDataPartTTLInfo & other_info); @@ -63,13 +64,16 @@ struct MergeTreeDataPartTTLInfos /// Has any TTLs which are not calculated on completely expired parts. bool hasAnyNonFinishedTTLs() const; - void updatePartMinMaxTTL(time_t time_min, time_t time_max) + void updatePartMinMaxTTL(const MergeTreeDataPartTTLInfo & ttl_info) { - if (time_min && (!part_min_ttl || time_min < part_min_ttl)) - part_min_ttl = time_min; + if (ttl_info.finished()) + return; - if (time_max && (!part_max_ttl || time_max > part_max_ttl)) - part_max_ttl = time_max; + if (ttl_info.min && (!part_min_ttl || ttl_info.min < part_min_ttl)) + part_min_ttl = ttl_info.min; + + if (ttl_info.max && (!part_max_ttl || ttl_info.max > part_max_ttl)) + part_max_ttl = ttl_info.max; } bool empty() const diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 8e64543510c3..9a2d8d75cadd 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -355,7 +355,7 @@ void updateTTL( } if (update_part_min_max_ttls) - ttl_infos.updatePartMinMaxTTL(ttl_info.min, ttl_info.max); + ttl_infos.updatePartMinMaxTTL(ttl_info); } void addSubcolumnsFromSortingKeyAndSkipIndicesExpression(const ExpressionActionsPtr & expr, Block & block) diff --git a/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.reference b/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.reference new file mode 100644 index 000000000000..bed8ae45f467 --- /dev/null +++ b/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.reference @@ -0,0 +1,3 @@ +1 2 +1 4 +1 diff --git a/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.sql b/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.sql new file mode 100644 index 000000000000..32d6376a55b3 --- /dev/null +++ b/tests/queries/0_stateless/04501_group_by_ttl_infinite_rollup.sql @@ -0,0 +1,27 @@ +-- Regression test for https://github.com/ClickHouse/ClickHouse/issues/105647 + +DROP TABLE IF EXISTS test_ttl_group_by SYNC; + +CREATE TABLE test_ttl_group_by (key UInt32, ts DateTime, value UInt32) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY key +TTL ts + INTERVAL 3 MONTH GROUP BY key SET value = sum(value), + ts + INTERVAL 50 YEAR DELETE +SETTINGS merge_with_ttl_timeout = 0; + +INSERT INTO test_ttl_group_by VALUES (1, '2020-01-01 00:00:00', 1), (1, '2021-01-01 00:00:00', 2), (1, '2020-01-01 00:00:00', 1), (1, '2021-01-01 00:00:00', 2); + +OPTIMIZE TABLE test_ttl_group_by FINAL; + +SELECT key, value FROM test_ttl_group_by ORDER BY ALL; + +-- Give the background merge selector time to reschedule the rollup (it must not). +SELECT sleep(3) FORMAT Null; +SELECT sleep(3) FORMAT Null; + +-- The partition holds the inserted part, the rolled up part and at most one part +-- of a background rollup that could run before OPTIMIZE. Any rescheduled rollup adds more. +SELECT count() <= 3 FROM system.parts WHERE database = currentDatabase() AND table = 'test_ttl_group_by' AND partition_id = '202001'; + +DROP TABLE test_ttl_group_by; diff --git a/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.reference b/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.reference new file mode 100644 index 000000000000..00750edc07d6 --- /dev/null +++ b/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.reference @@ -0,0 +1 @@ +3 diff --git a/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.sql b/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.sql new file mode 100644 index 000000000000..e21974145238 --- /dev/null +++ b/tests/queries/0_stateless/04502_group_by_ttl_merge_of_finished_parts.sql @@ -0,0 +1,28 @@ +-- Merging parts whose GROUP BY TTL is already finished must still re-aggregate +-- rows with the same key: the finished flag is a per-part property and is not +-- preserved when parts are combined. + +DROP TABLE IF EXISTS t_ttl_finished; + +CREATE TABLE t_ttl_finished (d DateTime, id UInt32, val UInt64) +ENGINE = MergeTree ORDER BY id +TTL d + INTERVAL 3 SECOND GROUP BY id SET val = sum(val); + +-- The rows are already expired at insert time. +-- Roll up each part alone; this marks its GROUP BY TTL as finished. +INSERT INTO t_ttl_finished VALUES ('2020-01-01 00:00:00', 0, 1); +OPTIMIZE TABLE t_ttl_finished FINAL; + +ALTER TABLE t_ttl_finished DETACH PARTITION ID 'all'; + +INSERT INTO t_ttl_finished VALUES ('2020-01-01 00:00:00', 0, 2); +OPTIMIZE TABLE t_ttl_finished FINAL; + +ALTER TABLE t_ttl_finished ATTACH PARTITION ID 'all'; + +-- Combine the two finished parts. +OPTIMIZE TABLE t_ttl_finished FINAL; + +SELECT val FROM t_ttl_finished ORDER BY val; + +DROP TABLE t_ttl_finished; From b4677e2e77dc084ca57892912a2c1a016bbe4e9b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 15:32:22 +0000 Subject: [PATCH 09/86] Backport #110710 to 26.6: Fix text-index direct-read double-registration over Merge/Distributed --- .../optimizeDirectReadFromTextIndex.cpp | 17 +++- ...where_dup_over_merge_distributed.reference | 6 ++ ...where_where_dup_over_merge_distributed.sql | 98 +++++++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.reference create mode 100644 tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index d2c725219646..dd3fcdf5e96e 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -883,9 +883,22 @@ void processAndOptimizeTextIndexFunctions(const Stack & stack, QueryPlan::Nodes if (text_index_read_infos.empty()) return; + /// This step can be visited by the pass more than once, because a Merge child plan is optimized + /// again after ReadFromMerge pushes a filter down into it (StorageMerge), and because the same + /// text-index predicate can reach the step from several filter stages. Direct read replaces a + /// text-search function with a synthetic __text_index_..._has_ column that is registered + /// into the step's read set exactly once (ReadFromMergeTree::createReadTasksForTextIndex). If the + /// step already carries index read tasks, direct read has already been applied to it, so a later + /// visit must not register the column a second time -- doing so throws "already added for reading" + /// (e.g. an identical predicate in both PREWHERE and WHERE over Merge -> Distributed -> MergeTree). + /// The tokenizer/preprocessor rewrite below still runs regardless: it is needed for correctness + /// (row-level evaluation when direct read is off or a part is not fully materialized) and does not + /// register any read column. + bool already_has_direct_read = !read_from_merge_tree_step->getIndexReadTasks().empty(); + bool optimized = false; if (auto prewhere_info = read_from_merge_tree_step->getPrewhereInfo()) - optimized = processAndOptimizeTextIndexFunctionsInPrewhere(*read_from_merge_tree_step, prewhere_info, text_index_read_infos, direct_read_from_text_index); + optimized = processAndOptimizeTextIndexFunctionsInPrewhere(*read_from_merge_tree_step, prewhere_info, text_index_read_infos, direct_read_from_text_index && !already_has_direct_read); if (stack.size() < 2) return; @@ -897,7 +910,7 @@ void processAndOptimizeTextIndexFunctions(const Stack & stack, QueryPlan::Nodes return; ActionsDAG & filter_dag = filter_step->getExpression(); - const auto * result_filter_node = processAndOptimizeTextIndexDAG(*read_from_merge_tree_step, filter_dag, text_index_read_infos, filter_step->getFilterColumnName(), direct_read_from_text_index && !optimized); + const auto * result_filter_node = processAndOptimizeTextIndexDAG(*read_from_merge_tree_step, filter_dag, text_index_read_infos, filter_step->getFilterColumnName(), direct_read_from_text_index && !optimized && !already_has_direct_read); if (!result_filter_node) return; diff --git a/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.reference b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.reference new file mode 100644 index 000000000000..64d78053a445 --- /dev/null +++ b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.reference @@ -0,0 +1,6 @@ +2 +1 +0 +1 +1 +1 diff --git a/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql new file mode 100644 index 000000000000..04143f1d1590 --- /dev/null +++ b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql @@ -0,0 +1,98 @@ +-- Tags: no-parallel-replicas + +-- Regression test for a query-plan optimizer abort: the same text-index predicate placed in both +-- PREWHERE and WHERE, walked through a Merge -> Distributed -> MergeTree plan, made +-- optimizeDirectReadFromTextIndex register the synthetic __text_index_..._has_ read column +-- twice for the same reading step (the step is optimized on more than one pass), aborting with +-- "Column ... already added for reading". See https://github.com/ClickHouse/ClickHouse/issues/110697 + +SET enable_analyzer = 1; +SET allow_experimental_full_text_index = 1; +-- The double-registration is reached only when the whole Merge -> Distributed -> MergeTree plan is +-- built and optimized on the initiator (local replica). +SET prefer_localhost_replica = 1; + +DROP TABLE IF EXISTS logs; +DROP TABLE IF EXISTS logs_dist; +DROP TABLE IF EXISTS logs_merge; + +CREATE TABLE logs +( + ts DateTime, + attributes Map(String, String), + msg String, + INDEX attributes_vals_idx mapValues(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1, + INDEX attributes_keys_idx mapKeys(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1, + INDEX msg_idx msg TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY ts; + +INSERT INTO logs VALUES (1, {'ip':'192.168.1.1'}, 'alpha beta'), (2, {'ip':'10.0.0.1'}, 'delta epsilon'), (3, {'ip':'192.168.1.1'}, 'delta zzz'); + +CREATE TABLE logs_dist AS logs ENGINE = Distributed(test_shard_localhost, currentDatabase(), logs); +CREATE TABLE logs_merge AS logs ENGINE = Merge(currentDatabase(), '^logs_dist$'); + +-- The trigger: identical text-index predicate in PREWHERE and WHERE over the Merge table. +-- force_data_skipping_indices guarantees the text-index direct-read path engages. +-- query_plan_direct_read_from_text_index is pinned to 1: the double-registration only happens when +-- direct read is actually enabled and the whole Merge -> Distributed -> MergeTree pipeline is built +-- (the abort is thrown during pipeline build, not plan optimization). The runner randomizes this +-- setting to 0 on ~5% of runs, which disables the optimization and would silently skip the crash +-- path, so this repro query must pin it to exercise the fix deterministically on every run. +SELECT count() FROM logs_merge +PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) +WHERE has(mapValues(attributes), toNullable('192.168.1.1')) +SETTINGS force_data_skipping_indices = 'attributes_vals_idx', query_plan_direct_read_from_text_index = 1; + +-- Direct read from the text index must remain engaged (optimization preserved, not disabled). +-- query_plan_direct_read_from_text_index is pinned to 1 here: it is randomized by the test runner and +-- when off it disables the whole optimization, so the __text_index_..._has_ column would be +-- absent and this visibility check would spuriously report the optimization as disabled. +SELECT count() > 0 FROM +( + EXPLAIN actions = 1 + SELECT count() FROM logs_merge + PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) + WHERE has(mapValues(attributes), toNullable('192.168.1.1')) + SETTINGS query_plan_direct_read_from_text_index = 1 +) +WHERE explain ILIKE '%__text_index_attributes_vals_idx_has%'; + +-- Non-equal predicates in PREWHERE and WHERE still return correct results. +SELECT count() FROM logs_merge +PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) +WHERE has(mapValues(attributes), toNullable('10.0.0.1')) +SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; + +-- Direct-read PREWHERE predicate plus a DIFFERENT text-index predicate in WHERE that uses a +-- tokenizing text function (hasAnyTokens). The fix gates only the direct-read virtual-column +-- registration on a re-visited step; the tokenizer/preprocessor rewrite of the WHERE predicate must +-- still run so row-level evaluation returns the same result whether direct read is on or off. +-- Compare both values of query_plan_direct_read_from_text_index: if the rewrite were dropped, the two +-- would diverge. rows with ip 192.168.1.1 = {1,3}; of those hasAnyTokens(msg, ['delta']) matches {3}. +SELECT count() FROM logs_merge +PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) +WHERE hasAnyTokens(msg, ['delta']) +SETTINGS query_plan_direct_read_from_text_index = 0; + +SELECT count() FROM logs_merge +PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) +WHERE hasAnyTokens(msg, ['delta']) +SETTINGS query_plan_direct_read_from_text_index = 1; + +-- The WHERE tokenizing predicate keeps its tokenizer rewrite (the 3-argument form with the index +-- tokenizer 'splitByNonAlpha' appended) even though the step is re-visited after the PREWHERE already +-- registered a direct-read column. This is the exact second-pass rewrite the fix preserves. +SELECT count() > 0 FROM +( + EXPLAIN actions = 1 + SELECT count() FROM logs_merge + PREWHERE has(mapValues(attributes), toNullable('192.168.1.1')) + WHERE hasAnyTokens(msg, ['delta']) + SETTINGS query_plan_direct_read_from_text_index = 1 +) +WHERE explain ILIKE '%hasAnyTokens(%splitByNonAlpha%'; + +DROP TABLE logs_merge; +DROP TABLE logs_dist; +DROP TABLE logs; From a2a7d601db63015e6cc07701a8e79cbea5319e7e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 17:36:15 +0000 Subject: [PATCH 10/86] Backport #110393 to 26.6: Make Field copy and destroy iterative to survive deeply nested literals --- src/Core/Field.cpp | 163 ++++++++++++++++++ src/Core/Field.h | 49 ++++-- src/Core/tests/gtest_field.cpp | 56 ++++++ ...ep_nested_literal_clone_no_crash.reference | 0 ...4538_deep_nested_literal_clone_no_crash.sh | 13 ++ 5 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.reference create mode 100755 tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.sh diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 4c3e18b95eb0..6276bd58f068 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -12,6 +12,9 @@ #include #include #include +#include + +#include using namespace std::literals; @@ -31,6 +34,166 @@ extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +void Field::initEmptyContainer(Types::Which w) +{ + switch (w) + { + case Types::Array: new (&storage) Array(); break; + case Types::Tuple: new (&storage) Tuple(); break; + case Types::Map: new (&storage) Map(); break; + case Types::Object: new (&storage) Object(); break; + default: break; + } + which = w; +} + +void Field::createContainerIteratively(const Field & src) +{ + /// Build *this as a deep copy of `src`. Each pending entry is a (source, destination) + /// pair of same-typed container Fields whose destination is empty and still needs its + /// elements copied in. Container children are created empty and enqueued instead of + /// being copied recursively, so the copy runs with a bounded native stack. + initEmptyContainer(src.which); + + absl::InlinedVector, 16> pending; + + /// On a mid-copy allocation failure, tear down what was built so we neither leak the + /// partial container nor leave the storage in a half-constructed state (matches the + /// strong guarantee the recursive std::vector copy used to provide). + auto copy_level = [&pending](const Field & s, Field & d) + { + auto copy_vector = [&pending](const auto & sv, auto & dv) + { + dv.reserve(sv.size()); /// keep &dv.back() stable while we hand out pointers below + for (const Field & se : sv) + { + if (isContainer(se.which)) + { + dv.emplace_back(); + Field & de = dv.back(); + de.initEmptyContainer(se.which); + pending.emplace_back(&se, &de); + } + else + dv.push_back(se); /// leaf: a shallow copy, no recursion + } + }; + + switch (d.which) + { + case Types::Array: copy_vector(s.get(), d.get()); break; + case Types::Tuple: copy_vector(s.get(), d.get()); break; + case Types::Map: copy_vector(s.get(), d.get()); break; + case Types::Object: + { + /// std::map insertion never invalidates references to existing elements, + /// so the &de pointers we enqueue stay valid. + for (const auto & [key, se] : s.get()) + { + if (isContainer(se.which)) + { + Field & de = d.get().emplace(key, Field()).first->second; + de.initEmptyContainer(se.which); + pending.emplace_back(&se, &de); + } + else + d.get().emplace(key, se); + } + break; + } + default: break; + } + }; + + try + { + copy_level(src, *this); + while (!pending.empty()) + { + auto [s, d] = pending.back(); + pending.pop_back(); + copy_level(*s, *d); + } + } + catch (...) + { + destroy(); + throw; + } +} + +static bool containerIsEmpty(const Field & f) +{ + switch (f.getType()) + { + case Field::Types::Array: return f.safeGet().empty(); + case Field::Types::Tuple: return f.safeGet().empty(); + case Field::Types::Map: return f.safeGet().empty(); + case Field::Types::Object: return f.safeGet().empty(); + default: return true; + } +} + +void Field::destroyContainerIteratively(Types::Which old_which) noexcept +{ + /// Tear down a (possibly deeply nested) container without native recursion: move every + /// non-empty nested-container child into an explicit worklist so each vector/map + /// destructor only ever destroys leaf elements (and already-emptied containers, which are + /// trivial), keeping the native stack depth bounded regardless of the nesting depth. + /// + /// This runs from ~Field, so it must not throw. The worklist can allocate, and allocation + /// goes through the throwing operator new, so suppress the memory-limit exception for its + /// lifetime (memory is still tracked, so freeing the value being destroyed is still credited). + /// The worklist only holds the current frontier of nested containers, which for a deeply + /// nested value is narrow (a deep literal is query-size bounded, so it cannot also be wide); + /// the inline buffer keeps that common case allocation-free. + LockMemoryExceptionInThread block_memory_limit_exception; + absl::InlinedVector to_destroy; + + auto steal_children = [&to_destroy](Field & container, Types::Which w) + { + auto steal_from_vector = [&to_destroy](auto & vec) + { + for (Field & elem : vec) + if (isContainer(elem.which) && !containerIsEmpty(elem)) + to_destroy.push_back(std::move(elem)); + }; + + switch (w) + { + case Types::Array: steal_from_vector(container.get()); break; + case Types::Tuple: steal_from_vector(container.get()); break; + case Types::Map: steal_from_vector(container.get()); break; + case Types::Object: + for (auto & [key, elem] : container.get()) + if (isContainer(elem.which) && !containerIsEmpty(elem)) + to_destroy.push_back(std::move(elem)); + break; + default: break; + } + }; + + /// `which` is already Null here (set by destroy()), so drive off the saved `old_which`. + steal_children(*this, old_which); + switch (old_which) + { + case Types::Array: destroy(); break; + case Types::Tuple: destroy(); break; + case Types::Map: destroy(); break; + case Types::Object: destroy(); break; + default: break; + } + + while (!to_destroy.empty()) + { + Field cur = std::move(to_destroy.back()); + to_destroy.pop_back(); + /// Empty `cur`'s nested containers into the worklist first, so destroying `cur` at the + /// end of this scope stays shallow (its remaining children are leaves or emptied). + steal_children(cur, cur.which); + } +} + bool AggregateFunctionStateData::operator < (const AggregateFunctionStateData &) const { throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Operator < is not implemented for AggregateFunctionStateData."); diff --git a/src/Core/Field.h b/src/Core/Field.h index e77d2579fb3d..c3e863ca0f5f 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -587,11 +587,24 @@ class Field ptr->assign(std::move(str)); } + /// Array/Tuple/Map/Object nest Fields inside Fields, so a straightforward + /// (recursive) copy/destroy overflows the native stack for a deeply nested value. + /// These containers are handled by explicit-worklist iterative helpers instead. + static bool isContainer(Types::Which w) + { + return w == Types::Array || w == Types::Tuple || w == Types::Map || w == Types::Object; + } + void create(const Field & x) { - dispatch([this] (auto & value) { createConcrete(value); }, x); + if (isContainer(x.which)) + createContainerIteratively(x); + else + dispatch([this] (auto & value) { createConcrete(value); }, x); } + /// Moving a Field just steals the container buffer (no per-element recursion), so + /// the move paths only need the iterative teardown of the value being overwritten. void create(Field && x) { dispatch([this] (auto & value) { createConcrete(std::move(value)); }, x); @@ -599,12 +612,26 @@ class Field void assign(const Field & x) { - dispatch([this] (auto & value) { assignConcrete(value); }, x); + if (isContainer(x.which)) + { + /// A vector/map copy-assignment would recurse per nesting level; rebuild instead. + destroy(); + create(x); + } + else + dispatch([this] (auto & value) { assignConcrete(value); }, x); } void assign(Field && x) { - dispatch([this] (auto & value) { assignConcrete(std::move(value)); }, x); + if (isContainer(x.which)) + { + /// A vector/map move-assignment first destroys the old (possibly deep) value recursively. + destroy(); + create(std::move(x)); + } + else + dispatch([this] (auto & value) { assignConcrete(std::move(value)); }, x); } template @@ -631,16 +658,10 @@ class Field destroy(); break; case Types::Array: - destroy(); - break; case Types::Tuple: - destroy(); - break; case Types::Map: - destroy(); - break; case Types::Object: - destroy(); + destroyContainerIteratively(old_which); break; case Types::AggregateFunctionState: destroy(); @@ -659,6 +680,14 @@ class Field T * MAY_ALIAS ptr = reinterpret_cast(&storage); ptr->~T(); } + + /// Placement-construct an empty container of the given type into raw (or Null) storage. + void initEmptyContainer(Types::Which w); + + /// Copy/destroy a (possibly deeply nested) Array/Tuple/Map/Object value using an explicit + /// worklist so the native stack depth stays bounded regardless of the nesting depth. + void createContainerIteratively(const Field & src); + void destroyContainerIteratively(Types::Which old_which) noexcept; }; #undef DBMS_MIN_FIELD_SIZE diff --git a/src/Core/tests/gtest_field.cpp b/src/Core/tests/gtest_field.cpp index 7e778be95750..96964fe67b6c 100644 --- a/src/Core/tests/gtest_field.cpp +++ b/src/Core/tests/gtest_field.cpp @@ -54,3 +54,59 @@ GTEST_TEST(Field, Move) f = Array{String{"Hello, world (6)"}}; ASSERT_EQ(f.safeGet()[0].safeGet(), "Hello, world (6)"); } + + +/// Copying and destroying a deeply nested Field must not overflow the native stack, both when the +/// source is a Field and when a container lvalue is wrapped/assigned through the templated +/// constructor / assignment operator (which forward to createConcrete / assignConcrete, i.e. the +/// underlying container copy, whose elements are copied via the iterative Field copy). The depth is +/// far beyond what a recursive copy could survive. +GTEST_TEST(Field, DeeplyNestedCopyAndDestroyDoesNotOverflowStack) +{ + static constexpr size_t depth = 100000; + + /// Build the nested value iteratively (moving, never copying) so constructing the test input + /// is O(depth) and cannot overflow either. + auto make_deep_array = [] + { + Array a; + a.push_back(Field{UInt64{1}}); + for (size_t i = 0; i < depth; ++i) + { + Array next; + next.push_back(Field{std::move(a)}); + a = std::move(next); + } + return a; + }; + + /// Field(const Field &): the ASTLiteral::clone path. + { + Field src{make_deep_array()}; + Field copy = src; // NOLINT(performance-unnecessary-copy-initialization) + ASSERT_EQ(copy.getType(), Field::Types::Array); + } + + /// Field(T &&) with a container lvalue: createConcrete -> container copy -> per-element Field copy. + { + Array a = make_deep_array(); + Field from_lvalue{a}; // lvalue -> copy + ASSERT_EQ(from_lvalue.getType(), Field::Types::Array); + } + + /// operator=(T &&) with a container lvalue: assignConcrete / destroy+createConcrete. + { + Array a = make_deep_array(); + Field assigned; + assigned = a; // lvalue -> copy-assign + ASSERT_EQ(assigned.getType(), Field::Types::Array); + } + + /// The same for a value nested inside an Object (the std::map-backed container). + { + Object obj; + obj.emplace("k", Field{make_deep_array()}); + Field src{obj}; // Object lvalue -> copy + ASSERT_EQ(src.getType(), Field::Types::Object); + } +} diff --git a/tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.reference b/tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.sh b/tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.sh new file mode 100755 index 000000000000..77718362184d --- /dev/null +++ b/tests/queries/0_stateless/04538_deep_nested_literal_clone_no_crash.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# A deeply nested array literal parses into a single ASTLiteral whose Field nests one Array per +# bracket. Analysing it clones the AST (deep-copying that Field) and eventually tears the Field +# down; both the copy and the destructor used to recurse once per nesting level and overflow the +# native stack. They are now iterative, so an arbitrarily deep literal must be rejected cleanly +# with TOO_DEEP_RECURSION by a stack-guarded walk, never crash. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 -c "print('SELECT ' + '['*100000 + '1' + ']'*100000 + '; -- { serverError TOO_DEEP_RECURSION }')" \ + | $CLICKHOUSE_LOCAL --max_parser_depth=1000000000 --max_query_size=1000000000 From 81a90c8f2b26f8dffb90c018393803b3d2c1a24c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 17:38:28 +0000 Subject: [PATCH 11/86] Backport #109675 to 26.6: Fix segfault due to bad shutdown ordering on startup exception --- programs/server/Server.cpp | 128 +++++++++--------- tests/integration/helpers/cluster.py | 10 +- tests/integration/test_restart_server/test.py | 13 ++ 3 files changed, 86 insertions(+), 65 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9260a3c9d2dc..35e8cc701b8f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -3331,70 +3331,6 @@ try if (config().has("startup_scripts")) loadStartupScripts(config(), server_settings, global_context, log); - { - std::lock_guard lock(servers_lock); - - /// Restore the startup log level overrides before accepting connections, - /// so that no requests are served with an elevated (startup) log level. - /// This must happen before server.start() because the config reload callback - /// (ConfigReloader) reads from config() which includes the writable layer - /// where startup level overrides are stored. - if (should_restore_default_logger_level) - { - config().setString("logger.level", default_logger_level_config); - Loggers::updateLevels(config(), logger()); - LOG_INFO(log, "Restored default logger level to {}", default_logger_level_config); - } - - if (should_restore_console_log_level) - { - /// If the level was unset just remove the override so the default can be set via - /// Loggers::updateLevels again; otherwise restore the configured value. - if (console_log_level_was_set) - { - config().setString("logger.console_log_level", original_console_log_level_config); - Loggers::updateLevels(config(), logger()); - LOG_INFO(log, "Restored console logger level to {}", original_console_log_level_config); - } - else - { - config().remove("logger.console_log_level"); - Loggers::updateLevels(config(), logger()); - LOG_INFO(log, "Restored console logger level to logger.level"); - } - } - - for (auto & server : servers) - { - server.start(); - LOG_INFO(log, "Listening for {}", server.getDescription()); - } - - global_context->setServerCompletelyStarted(); - LOG_INFO(log, "Ready for connections."); - } - - startup_watch.stop(); - ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds()); - - CannotAllocateThreadFaultInjector::setFaultProbability(server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]); - - try - { - global_context->startClusterDiscovery(); - } - catch (...) - { - tryLogCurrentException(log, "Caught exception while starting cluster discovery"); - } - -#if defined(OS_LINUX) - /// Tell the service manager that service startup is finished. - /// NOTE: the parent clickhouse-watchdog process must do systemdNotify("MAINPID={}\n", child_pid); before - /// the child process notifies 'READY=1'. - systemdNotify("READY=1\n"); -#endif - auto stop_acme_instance = []{ #if USE_SSL /// Stop ACME tasks. @@ -3511,6 +3447,70 @@ try } }); + { + std::lock_guard lock(servers_lock); + + /// Restore the startup log level overrides before accepting connections, + /// so that no requests are served with an elevated (startup) log level. + /// This must happen before server.start() because the config reload callback + /// (ConfigReloader) reads from config() which includes the writable layer + /// where startup level overrides are stored. + if (should_restore_default_logger_level) + { + config().setString("logger.level", default_logger_level_config); + Loggers::updateLevels(config(), logger()); + LOG_INFO(log, "Restored default logger level to {}", default_logger_level_config); + } + + if (should_restore_console_log_level) + { + /// If the level was unset just remove the override so the default can be set via + /// Loggers::updateLevels again; otherwise restore the configured value. + if (console_log_level_was_set) + { + config().setString("logger.console_log_level", original_console_log_level_config); + Loggers::updateLevels(config(), logger()); + LOG_INFO(log, "Restored console logger level to {}", original_console_log_level_config); + } + else + { + config().remove("logger.console_log_level"); + Loggers::updateLevels(config(), logger()); + LOG_INFO(log, "Restored console logger level to logger.level"); + } + } + + for (auto & server : servers) + { + server.start(); + LOG_INFO(log, "Listening for {}", server.getDescription()); + } + + global_context->setServerCompletelyStarted(); + LOG_INFO(log, "Ready for connections."); + } + + startup_watch.stop(); + ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds()); + + CannotAllocateThreadFaultInjector::setFaultProbability(server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]); + + try + { + global_context->startClusterDiscovery(); + } + catch (...) + { + tryLogCurrentException(log, "Caught exception while starting cluster discovery"); + } + +#if defined(OS_LINUX) + /// Tell the service manager that service startup is finished. + /// NOTE: the parent clickhouse-watchdog process must do systemdNotify("MAINPID={}\n", child_pid); before + /// the child process notifies 'READY=1'. + systemdNotify("READY=1\n"); +#endif + std::vector> metrics_transmitters; for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) { diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 2c7c61bd0a91..715895b6b253 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -5416,7 +5416,12 @@ def stop_clickhouse(self, stop_wait_sec=30, kill=False): logging.warning(f"Stop ClickHouse raised an error {e}") def start_clickhouse( - self, start_wait_sec=60, retry_start=True, expected_to_fail=False + self, + start_wait_sec=60, + retry_start=True, + expected_to_fail=False, + environment=None, + wait_start=True, ): if not self.stay_alive: raise Exception( @@ -5438,7 +5443,10 @@ def start_clickhouse( detach=True, use_cli=False, get_exec_id=True, + environment=environment, ) + if not wait_start: + return exec_id if expected_to_fail: self.wait_start_failed(start_wait_sec + start_time - time.time()) return diff --git a/tests/integration/test_restart_server/test.py b/tests/integration/test_restart_server/test.py index 07368531adc2..225a966130c0 100755 --- a/tests/integration/test_restart_server/test.py +++ b/tests/integration/test_restart_server/test.py @@ -35,3 +35,16 @@ def test_flushes_async_insert_queue(): ) node.restart_clickhouse() assert node.query("SELECT * FROM flush_test") == "world\t23456\n" + + +def test_server_startup_notify_socket_exception(): + # Regression test. + # Make sure after an exception is thrown from systemdNotify during startup + # the server does not segfault on further queries. + node.stop_clickhouse() + node.start_clickhouse(environment={"NOTIFY_SOCKET": "bad"}, wait_start=False) + + node.wait_for_log_line("Shut down storages", timeout=180, look_behind_lines=1) + assert "Connection refused" in node.query_and_get_error("SELECT 1") + + node.start_clickhouse() From 99a509e9339e2d6b2dc178b575e9dac54494d848 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 18:30:23 +0000 Subject: [PATCH 12/86] Backport #110914 to 26.6: Fix use-after-free when using Iceberg tables in combination with temporary tables --- src/Interpreters/DatabaseCatalog.cpp | 20 ++++++++++++++----- src/Interpreters/DatabaseCatalog.h | 4 +++- .../Common/AvroForIcebergDeserializer.cpp | 4 ++-- .../Common/AvroForIcebergDeserializer.h | 1 - .../integration/test_database_iceberg/test.py | 15 +++++++++++++- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 3d0fe9a923ba..247e954b6476 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -140,7 +140,7 @@ class DatabaseNameHints : public IHints<> TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryTableHolder::Creator & creator, const ASTPtr & query) : WithContext(context_->getGlobalContext()) - , temporary_tables(DatabaseCatalog::instance().getDatabaseForTemporaryTables().get()) + , temporary_tables(DatabaseCatalog::instance().getDatabaseForTemporaryTables()) { ASTPtr original_create; ASTCreateQuery * create = dynamic_cast(query.get()); @@ -163,7 +163,7 @@ TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryT auto table_id = StorageID(DatabaseCatalog::TEMPORARY_DATABASE, global_name, id); auto table = creator(table_id); DatabaseCatalog::instance().addUUIDMapping(id); - temporary_tables->createTable(getContext(), global_name, table, original_create); + getDatabase()->createTable(getContext(), global_name, table, original_create); table->startup(); } @@ -190,7 +190,7 @@ TemporaryTableHolder::TemporaryTableHolder( } TemporaryTableHolder::TemporaryTableHolder(TemporaryTableHolder && rhs) noexcept - : WithContext(rhs.context), temporary_tables(rhs.temporary_tables), id(rhs.id), future_set(std::move(rhs.future_set)) + : WithContext(rhs.context), temporary_tables(std::move(rhs.temporary_tables)), id(rhs.id), future_set(std::move(rhs.future_set)) { rhs.id = UUIDHelpers::Nil; } @@ -210,7 +210,7 @@ TemporaryTableHolder::~TemporaryTableHolder() { auto table = getTable(); table->flushAndShutdown(/*is_drop=*/ true); - temporary_tables->dropTable(getContext(), "_tmp_" + toString(id)); + getDatabase()->dropTable(getContext(), "_tmp_" + toString(id)); } catch (...) { @@ -224,9 +224,19 @@ StorageID TemporaryTableHolder::getGlobalTableID() const return StorageID{DatabaseCatalog::TEMPORARY_DATABASE, "_tmp_" + toString(id), id}; } +std::shared_ptr TemporaryTableHolder::getDatabase() const +{ + auto database = temporary_tables.lock(); + if (!database) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Database for temporary tables is already destroyed, but TemporaryTableHolder for {} is still alive", + getGlobalTableID().getNameForLogs()); + return database; +} + StoragePtr TemporaryTableHolder::getTable() const { - auto table = temporary_tables->tryGetTable("_tmp_" + toString(id), getContext()); + auto table = getDatabase()->tryGetTable("_tmp_" + toString(id), getContext()); if (!table) throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary table {} not found", getGlobalTableID().getNameForLogs()); return table; diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index bbb5b0a88541..3b9ca19e54dc 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -72,9 +72,11 @@ struct TemporaryTableHolder : boost::noncopyable, WithContext StoragePtr getTable() const; + std::shared_ptr getDatabase() const; + operator bool () const { return id != UUIDHelpers::Nil; } /// NOLINT - IDatabase * temporary_tables = nullptr; + std::weak_ptr temporary_tables; UUID id = UUIDHelpers::Nil; FutureSetFromSubqueryPtr future_set; }; diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 4517834d0dec..653c0bb47d17 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -33,9 +33,9 @@ AvroForIcebergDeserializer::AvroForIcebergDeserializer( const IcebergPathFromMetadata & manifest_file_path_, const DB::FormatSettings & format_settings) try - : buffer(std::move(buffer_)) - , manifest_file_path(manifest_file_path_) + : manifest_file_path(manifest_file_path_) { + auto buffer = std::move(buffer_); auto manifest_file_reader = std::make_unique(std::make_unique(*buffer), MAX_AVRO_SCHEMA_DEPTH); diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h index b20aedac504d..ea2a54706b60 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h @@ -36,7 +36,6 @@ using ParsedManifestFileEntryPtr = std::shared_ptr buffer; Iceberg::IcebergPathFromMetadata manifest_file_path; DB::ColumnPtr parsed_column; std::shared_ptr parsed_column_data_type; diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 31ec8882a357..7dd38185688c 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -412,11 +412,24 @@ def test_select(started_cluster): ) assert num_rows == int( - node.query(f"SELECT count() FROM {CATALOG_NAME}.`{namespace}.{table_name}`") + node.query( + # Regression test: a session temp table used to be pinned by the query context + # captured in the S3 client refresher and cached with the manifest file in the + # global IcebergMetadataFilesCache, crashing the graceful restart below with a + # use-after-free. The SELECT * is required: it reads a manifest file (count() + # alone is served from the snapshot summary). All statements must stay in one + # node.query call = one session. + f"CREATE TEMPORARY TABLE pin_me (x UInt8) ENGINE = Memory;" + f"SELECT * FROM {CATALOG_NAME}.`{namespace}.{table_name}` FORMAT Null;" + f"SELECT count() FROM {CATALOG_NAME}.`{namespace}.{table_name}`" + ) ) assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE table = '{namespace}.{table_name}' and database = '{CATALOG_NAME}'").strip()) == 1 + # Replays the graceful shutdown; the teardown sanitizer check catches the UAF if it regresses. + node.restart_clickhouse() + def test_hide_sensitive_info(started_cluster): node = started_cluster.instances["node1"] From 1ccc86a3decfa04f07029cc994e222f0b09e8e81 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Jul 2026 20:55:15 +0000 Subject: [PATCH 13/86] Backport #109188 to 26.6: Fix text index on mapValues/mapKeys not used through a Distributed engine table --- src/Storages/StorageBuffer.cpp | 7 ++ src/Storages/StorageBuffer.h | 4 + src/Storages/StorageDistributed.h | 4 + src/Storages/StorageMaterializedView.h | 3 + src/Storages/StorageMerge.cpp | 5 + src/Storages/StorageMerge.h | 3 + src/Storages/StorageProxy.h | 3 + .../02346_text_index_bug108874.reference | 14 +++ .../02346_text_index_bug108874.sql | 116 ++++++++++++++++++ 9 files changed, 159 insertions(+) create mode 100644 tests/queries/0_stateless/02346_text_index_bug108874.reference create mode 100644 tests/queries/0_stateless/02346_text_index_bug108874.sql diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 3c5a2007e8f8..17c99a495b1b 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -912,6 +912,13 @@ bool StorageBuffer::supportsPrewhere() const return false; } +bool StorageBuffer::supportsOptimizationToSubcolumns() const +{ + if (auto destination = getDestinationTable()) + return destination->supportsOptimizationToSubcolumns(); + return false; +} + bool StorageBuffer::checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows, size_t additional_bytes) const { time_t time_passed = 0; diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index fde88d166b47..aad37e3badfd 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -121,6 +121,10 @@ friend class BufferSink; return true; } bool supportsPrewhere() const override; + /// read() forwards the already-analyzed query straight to the destination table, so the + /// initiator must not rewrite functions to subcolumns when the destination opts out (e.g. + /// Distributed). Fails closed like supportsPrewhere(): no destination means no rewrite. + bool supportsOptimizationToSubcolumns() const override; bool supportsFinal() const override { return true; } void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 56692156c5b9..5670e0c39909 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -76,6 +76,10 @@ class StorageDistributed final : public IStorage, WithContext bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } bool supportsSubcolumns() const override { return true; } + /// Distributed only serializes the query to shards; it never reads columns locally, so rewriting + /// functions to subcolumns brings no benefit and breaks shard-side skip-index analysis (a rewritten + /// subcolumn no longer matches an index defined on the original expression). Same as IStorageCluster. + bool supportsOptimizationToSubcolumns() const override { return false; } bool supportsColumnsWithDynamicStructure() const override { return true; } StoragePolicyPtr getStoragePolicy() const override; diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 05e3aab1a566..4711072bc27a 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -36,6 +36,9 @@ class StorageMaterializedView final : public StorageWithCommonVirtualColumns, Wi bool supportsFinal() const override { return getTargetTable()->supportsFinal(); } bool supportsParallelInsert() const override { return getTargetTable()->supportsParallelInsert(); } bool supportsSubcolumns() const override { return getTargetTable()->supportsSubcolumns(); } + /// readImpl forwards the already-analyzed query tree straight to the target table, so the + /// initiator must not rewrite functions to subcolumns when the target opts out (e.g. Distributed). + bool supportsOptimizationToSubcolumns() const override { return getTargetTable()->supportsOptimizationToSubcolumns(); } bool supportsColumnsWithDynamicStructure() const override; bool supportsTransactions() const override { return getTargetTable()->supportsTransactions(); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 8a7d6cfeeaca..7da726420895 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -315,6 +315,11 @@ bool StorageMerge::supportsPrewhere() const return traverseTablesUntil([](const auto & table) { return !table->supportsPrewhere(); }) == nullptr; } +bool StorageMerge::supportsOptimizationToSubcolumns() const +{ + return traverseTablesUntil([](const auto & table) { return !table->supportsOptimizationToSubcolumns(); }) == nullptr; +} + bool StorageMerge::canMoveConditionsToPrewhere() const { /// NOTE: This check and the above check are used during query analysis as condition for applying diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 8f8eb45103d2..096b9a98e98a 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -51,6 +51,9 @@ class StorageMerge final : public IStorage, WithContext bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } bool supportsSubcolumns() const override { return true; } + /// Fails closed: a Merge over a child that opts out (e.g. Distributed) must not let the + /// initiator rewrite functions to subcolumns, or a skip index on the shard would be missed. + bool supportsOptimizationToSubcolumns() const override; bool supportsColumnsWithDynamicStructure() const override { return true; } bool supportsPrewhere() const override; std::optional supportedPrewhereColumns() const override; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 77b1fce2d6ff..088af3249124 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -30,6 +30,9 @@ class StorageProxy : public IStorage bool noPushingToViewsOnInserts() const override { return getNested()->noPushingToViewsOnInserts(); } bool hasEvenlyDistributedRead() const override { return getNested()->hasEvenlyDistributedRead(); } bool supportsSubcolumns() const override { return getNested()->supportsSubcolumns(); } + /// The IStorage default ties this to supportsSubcolumns(); forward it so a proxy around a + /// storage that opts out of the rewrite (e.g. Distributed) does not re-advertise true. + bool supportsOptimizationToSubcolumns() const override { return getNested()->supportsOptimizationToSubcolumns(); } bool supportsColumnsWithDynamicStructure() const override { return getNested()->supportsColumnsWithDynamicStructure(); } ColumnSizeByName getColumnSizes() const override { return getNested()->getColumnSizes(); } diff --git a/tests/queries/0_stateless/02346_text_index_bug108874.reference b/tests/queries/0_stateless/02346_text_index_bug108874.reference new file mode 100644 index 000000000000..f72099076222 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug108874.reference @@ -0,0 +1,14 @@ +mapValues local 1 +mapValues cluster 2 +mapValues dist 2 +mapKeys local 1 +mapKeys cluster 2 +mapKeys dist 2 +mapValues mv-over-dist 2 +mapKeys mv-over-dist 2 +mapValues merge-over-dist 2 +mapKeys merge-over-dist 2 +mapValues buffer-over-dist 2 +mapKeys buffer-over-dist 2 +mapValues lazy-proxy dist 2 +mapKeys lazy-proxy dist 2 diff --git a/tests/queries/0_stateless/02346_text_index_bug108874.sql b/tests/queries/0_stateless/02346_text_index_bug108874.sql new file mode 100644 index 000000000000..3f27cce97c12 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug108874.sql @@ -0,0 +1,116 @@ +-- Tags: distributed, no-replicated-database + +-- Regression test for https://github.com/ClickHouse/ClickHouse/issues/108874 +-- A text index on mapValues(map)/mapKeys(map) was used for a local MergeTree table and via +-- cluster()/remote(), but silently NOT used when the same table was queried through a +-- Distributed engine table with the analyzer. The Distributed table has no secondary indices in +-- its own metadata, so FunctionToSubcolumnsPass rewrote mapValues(attributes) -> attributes.values +-- on the initiator and serialized that form to the shards, where it no longer matched the index. +-- force_data_skipping_indices makes the query throw INDEX_NOT_USED (code 277) if the index is not +-- used, so a successful query with the correct count proves the index is used through every path. + +SET enable_full_text_index = 1; +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; -- default; the setting that triggered the bug + +DROP TABLE IF EXISTS logs; +DROP TABLE IF EXISTS logs_dist; + +CREATE TABLE logs +( + attributes Map(String, String), + INDEX attributes_vals_idx mapValues(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1, + INDEX attributes_keys_idx mapKeys(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple(); + +CREATE TABLE logs_dist AS logs +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), logs, rand()); + +INSERT INTO logs VALUES ({'ip': '192.168.1.1'}); + +SELECT 'mapValues local', count() FROM logs WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapValues cluster', count() FROM cluster('test_cluster_two_shards_localhost', currentDatabase(), logs) WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapValues dist', count() FROM logs_dist WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; + +SELECT 'mapKeys local', count() FROM logs WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; +SELECT 'mapKeys cluster', count() FROM cluster('test_cluster_two_shards_localhost', currentDatabase(), logs) WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; +SELECT 'mapKeys dist', count() FROM logs_dist WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; + +-- Wrapper storages over a Distributed table. StorageDistributed opts out of +-- supportsOptimizationToSubcolumns(), but a wrapper that ties the capability to +-- supportsSubcolumns() (the IStorage default) re-enables the mapValues -> subcolumn rewrite on +-- the initiator, so the shard query loses the index again. StorageMaterializedView must forward +-- the capability to its target table, and StorageMerge must fail closed if any child opts out. + +-- Materialized view whose target is the Distributed table; query the view itself. +CREATE MATERIALIZED VIEW logs_mv TO logs_dist AS SELECT attributes FROM logs; +SELECT 'mapValues mv-over-dist', count() FROM logs_mv WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapKeys mv-over-dist', count() FROM logs_mv WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; + +-- Merge table over the Distributed child. +CREATE TABLE logs_merge AS logs ENGINE = Merge(currentDatabase(), '^logs_dist$'); +SELECT 'mapValues merge-over-dist', count() FROM logs_merge WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapKeys merge-over-dist', count() FROM logs_merge WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; + +-- Buffer table over the Distributed child. getQueryProcessingStage and read() forward the +-- already-analyzed query to the destination, so like Merge/MaterializedView the Buffer must fail +-- closed instead of inheriting the IStorage default (supportsSubcolumns() == true). The buffer is +-- empty (flushed), so the SELECT reads only the destination Distributed table -> shard MergeTree, +-- and force_data_skipping_indices governs those indexed shard reads. +CREATE TABLE logs_buffer (attributes Map(String, String)) +ENGINE = Buffer(currentDatabase(), logs_dist, 1, 10, 100, 10000, 1000000, 10000000, 100000000); +SELECT 'mapValues buffer-over-dist', count() FROM logs_buffer WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapKeys buffer-over-dist', count() FROM logs_buffer WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; + +DROP TABLE logs_buffer; +DROP TABLE logs_merge; +DROP TABLE logs_mv; +DROP TABLE logs_dist; +DROP TABLE logs; + +-- Lazy-proxy variant. When a database has lazy_load_tables = 1, each table is wrapped in a +-- StorageTableProxy that forwards to the real storage on first access. StorageProxy forwarded +-- supportsSubcolumns() but not supportsOptimizationToSubcolumns(), so a proxy around Distributed +-- fell back to the IStorage default (which ties it to supportsSubcolumns() == true) and re-enabled +-- the mapValues -> subcolumn rewrite on the initiator. The index was then missed again through a +-- lazy-loaded Distributed engine table, even after StorageDistributed itself opted out. + +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.logs +( + attributes Map(String, String), + INDEX attributes_vals_idx mapValues(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1, + INDEX attributes_keys_idx mapKeys(attributes) TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple(); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.logs_dist (attributes Map(String, String)) +ENGINE = Distributed('test_cluster_two_shards_localhost', {CLICKHOUSE_DATABASE_1:String}, logs, rand()); + +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.logs VALUES ({'ip': '192.168.1.1'}); + +-- Re-attach so the tables become StorageTableProxy instances. +DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; + +SELECT 'mapValues lazy-proxy dist', count() FROM {CLICKHOUSE_DATABASE_1:Identifier}.logs_dist WHERE has(mapValues(attributes), '192.168.1.1') + SETTINGS force_data_skipping_indices = 'attributes_vals_idx'; +SELECT 'mapKeys lazy-proxy dist', count() FROM {CLICKHOUSE_DATABASE_1:Identifier}.logs_dist WHERE has(mapKeys(attributes), 'ip') + SETTINGS force_data_skipping_indices = 'attributes_keys_idx'; + +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; From fa25fe91b1805386d04f21aa84329f4a01b46dc7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Jul 2026 00:40:08 +0000 Subject: [PATCH 14/86] Pin optimize_functions_to_subcolumns=0 in 04549 test on 26.6 26.6 does not have https://github.com/ClickHouse/ClickHouse/pull/109188, which makes `StorageMerge` and `StorageDistributed` opt out of the `optimize_functions_to_subcolumns` rewrite. When the test runner randomizes that setting to 1, `mapValues(attributes)` in PREWHERE is rewritten to the subcolumn `attributes.values`, which `Merge` rejects with `ILLEGAL_PREWHERE`. Pin the setting to 0 so the predicate keeps the function form that matches the text-index expression, as in the original repro. Same fix as on backport/26.4/110710 (PR #111095), where it was verified against the PR's arm_release binary: the unpinned test fails with `ILLEGAL_PREWHERE` under `optimize_functions_to_subcolumns = 1`, the pinned test matches the reference. Co-Authored-By: Claude Fable 5 --- ...text_index_prewhere_where_dup_over_merge_distributed.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql index 04143f1d1590..02e1520ea542 100644 --- a/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql +++ b/tests/queries/0_stateless/04549_text_index_prewhere_where_dup_over_merge_distributed.sql @@ -8,6 +8,12 @@ SET enable_analyzer = 1; SET allow_experimental_full_text_index = 1; +-- 26.6 does not have the fix from https://github.com/ClickHouse/ClickHouse/pull/109188 (StorageMerge +-- and StorageDistributed opting out of the rewrite of mapValues(attributes) to the subcolumn +-- attributes.values), so with optimize_functions_to_subcolumns randomized to 1 the PREWHERE predicate +-- would be rewritten to the subcolumn, which Merge rejects with ILLEGAL_PREWHERE. Pin it to 0 to keep +-- the predicate in the function form that matches the text-index expression, as in the original repro. +SET optimize_functions_to_subcolumns = 0; -- The double-registration is reached only when the whole Merge -> Distributed -> MergeTree plan is -- built and optimized on the initiator (local replica). SET prefer_localhost_replica = 1; From 55d288fe1fc91c05ee24c5095601e2e6017f1f68 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Jul 2026 06:55:03 +0000 Subject: [PATCH 15/86] Backport #107437 to 26.6: Fix hive partition validation when compatibility is 26.6 --- .../engines/table-engines/integrations/s3.md | 3 +- .../table-functions/azureBlobStorage.md | 7 ++- docs/en/sql-reference/table-functions/gcs.md | 7 +-- docs/en/sql-reference/table-functions/s3.md | 4 +- .../StorageObjectStorageConfiguration.cpp | 53 ++++++++++++------- .../StorageObjectStorageConfiguration.h | 8 +++ .../registerStorageObjectStorage.cpp | 9 +++- .../test_checking_s3_blobs_paranoid/test.py | 2 +- .../test.py | 2 +- .../test_storage_azure_blob_storage/test.py | 18 +++---- .../test_cluster.py | 2 +- tests/integration/test_storage_s3/test.py | 16 +++--- .../0_stateless/01944_insert_partition_by.sql | 2 + ...45_s3_support_read_nested_column.reference | 12 ++--- .../02245_s3_support_read_nested_column.sql | 12 ++--- .../02302_s3_file_pruning.reference | 2 +- .../0_stateless/02302_s3_file_pruning.sql | 2 +- .../02480_s3_support_wildcard.reference | 4 +- .../0_stateless/02480_s3_support_wildcard.sql | 4 +- ...02481_s3_throw_if_mismatch_files.reference | 2 +- .../02481_s3_throw_if_mismatch_files.sql | 2 +- .../0_stateless/02495_s3_filter_by_file.sql | 2 +- .../02496_storage_s3_profile_events.sql | 2 +- ...7_s3_write_to_globbed_partitioned_path.sql | 2 + ...528_s3_insert_partition_by_whitespaces.sql | 6 ++- .../03629_storage_s3_disallow_index_alter.sql | 2 +- ..._s3_table_function_small_file_prefetch.sql | 2 +- .../04004_s3_schema_hash.reference | 2 +- .../0_stateless/04004_s3_schema_hash.sql | 2 +- ...tion_id_compatibility_validation.reference | 4 ++ ..._partition_id_compatibility_validation.sql | 52 ++++++++++++++++++ 31 files changed, 174 insertions(+), 75 deletions(-) create mode 100644 tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference create mode 100644 tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 92934e5d8b79..926d68c0f1b7 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -165,7 +165,8 @@ ENGINE = S3( 'http://minio:10000/clickhouse//test_{_partition_id}.csv', 'minioadmin', 'minioadminpassword', - 'CSV') + 'CSV', + partition_strategy='wildcard') PARTITION BY column3 ``` diff --git a/docs/en/sql-reference/table-functions/azureBlobStorage.md b/docs/en/sql-reference/table-functions/azureBlobStorage.md index 14dce34b0e0b..c1ef2a9645e9 100644 --- a/docs/en/sql-reference/table-functions/azureBlobStorage.md +++ b/docs/en/sql-reference/table-functions/azureBlobStorage.md @@ -26,7 +26,7 @@ Provides a table-like interface to select/insert files in [Azure Blob Storage](h Credentials are embedded in the connection string, so no separate `account_name`/`account_key` is needed: ```sql -azureBlobStorage(connection_string, container_name, blobpath [, format, compression, structure]) +azureBlobStorage(connection_string, container_name, blobpath [, format, compression, partition_strategy, structure]) ``` @@ -35,7 +35,7 @@ azureBlobStorage(connection_string, container_name, blobpath [, format, compress Requires `account_name` and `account_key` as separate arguments: ```sql -azureBlobStorage(storage_account_url, container_name, blobpath, account_name, account_key [, format, compression, structure]) +azureBlobStorage(storage_account_url, container_name, blobpath, account_name, account_key [, format, compression, partition_strategy, structure]) ``` @@ -62,6 +62,7 @@ azureBlobStorage(named_collection[, option=value [,..]]) | `account_key` | Storage account key. **Required** when using `storage_account_url` without SAS; must **not** be passed when using `connection_string`. | | `format` | The [format](/sql-reference/formats) of the file. | | `compression` | Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension (same as setting to `auto`). | +| `partition_strategy` | Optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as file names and the file format as the extension. | | `structure` | Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. | | `partition_strategy` | Optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). | | `partition_columns_in_data_file` | Optional. Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. | @@ -81,6 +82,7 @@ Arguments can also be passed using [named collections](/operations/named-collect | `account_key` | No | Required when using `storage_account_url` | | `format` | No | File format. | | `compression` | No | Compression type. | +| `partition_strategy` | No | Partition strategy: `WILDCARD` or `HIVE`. | | `structure` | No | Table structure. | | `client_id` | No | Client ID for authentication. | | `tenant_id` | No | Tenant ID for authentication. | @@ -156,6 +158,7 @@ INSERT INTO TABLE FUNCTION azureBlobStorage( 'test_{_partition_id}.csv', 'CSV', 'auto', + 'wildcard', 'column1 UInt32, column2 UInt32, column3 UInt32' ) PARTITION BY column3 VALUES (1, 2, 3), (3, 2, 1), (78, 43, 3); diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md index e8a713798c6f..81d7ea02e3e8 100644 --- a/docs/en/sql-reference/table-functions/gcs.md +++ b/docs/en/sql-reference/table-functions/gcs.md @@ -18,7 +18,7 @@ If you have multiple replicas in your cluster, you can use the [s3Cluster functi ## Syntax {#syntax} ```sql -gcs(url [, NOSIGN | hmac_key, hmac_secret] [,format] [,structure] [,compression_method]) +gcs(url [, NOSIGN | hmac_key, hmac_secret] [,format] [,structure] [,compression_method] [,partition_strategy]) gcs(named_collection[, option=value [,..]]) ``` @@ -37,6 +37,7 @@ See the [Google interoperability docs]( https://cloud.google.com/storage/docs/in | `format` | The [format](/sql-reference/formats) of the file. | | `structure` | Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. | | `compression_method` | Parameter is optional. Supported values: `none`, `gzip` or `gz`, `brotli` or `br`, `xz` or `LZMA`, `zstd` or `zst`. By default, it will autodetect compression method by file extension. | +| `partition_strategy` | Parameter is optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as file names and the file format as the extension. | :::note GCS The GCS path is in this format as the endpoint for the Google XML API is different than the JSON API: @@ -197,7 +198,7 @@ If you specify `PARTITION BY` expression when inserting data into `GCS` table, a ```sql INSERT INTO TABLE FUNCTION - gcs('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32') + gcs('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32', partition_strategy='wildcard') PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); ``` As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`. @@ -206,7 +207,7 @@ As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, a ```sql INSERT INTO TABLE FUNCTION - gcs('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + gcs('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32', partition_strategy='wildcard') PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); ``` As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 100ec614c961..d4c152207974 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -272,7 +272,7 @@ SELECT _path, * FROM s3(s3_conn, filename='t_03363_function/**.parquet'); ```sql INSERT INTO TABLE FUNCTION - s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32') + s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32', partition_strategy='wildcard') PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); ``` As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`. @@ -281,7 +281,7 @@ As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, a ```sql INSERT INTO TABLE FUNCTION - s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32', partition_strategy='wildcard') PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); ``` As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp index e65d0eaa78af..7703a992cc52 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp @@ -127,15 +127,14 @@ void StorageObjectStorageConfiguration::initialize( throw Exception(ErrorCodes::BAD_ARGUMENTS, "The `partition_strategy` argument is incompatible with data lakes"); } } - else if (configuration_to_initialize.partition_strategy_type == PartitionStrategyFactory::StrategyType::NONE) + else if (configuration_to_initialize.partition_strategy_type == PartitionStrategyFactory::StrategyType::NONE + && configuration_to_initialize.getRawPath().hasPartitionWildcard() + && local_context->getSettingsRef()[Setting::file_like_engine_default_partition_strategy].value + == FileLikeEngineDefaultPartitionStrategy::WILDCARD) { - if (configuration_to_initialize.getRawPath().hasPartitionWildcard()) - { - // Promote to wildcard in case it is not data lake to make it backwards compatible - configuration_to_initialize.partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; - } + /// Backwards compatibility: promote to WILDCARD only when it is the effective default strategy. + configuration_to_initialize.partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; } - if (configuration_to_initialize.format == "auto") { if (configuration_to_initialize.isDataLakeConfiguration()) @@ -191,19 +190,37 @@ void StorageObjectStorageConfiguration::initPartitionStrategy(ASTPtr partition_b /// `partition_columns_in_data_file = 0` combined with strategy `none`) keep raising. if (partition_by && partition_strategy_type == PartitionStrategyFactory::StrategyType::NONE && !isDataLakeConfiguration()) { - switch (context->getSettingsRef()[Setting::file_like_engine_default_partition_strategy].value) + if (!is_create_query) { - case FileLikeEngineDefaultPartitionStrategy::WILDCARD: - { - /// Set the strategy unconditionally; `PartitionStrategyFactory::get` will raise - /// `BAD_ARGUMENTS` if the path is missing the `{_partition_id}` placeholder. - partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; - break; - } - case FileLikeEngineDefaultPartitionStrategy::HIVE: + /// Backward compatibility on ATTACH / server startup / RESTORE / replicated-DDL replay: + /// for a table loaded from existing metadata the implicit strategy is deterministically + /// recoverable from the path alone, because the two strategies are mutually exclusive on + /// path shape — wildcard REQUIRES `{_partition_id}` in the path, hive FORBIDS it. Consulting + /// the mutable `file_like_engine_default_partition_strategy` default here instead would + /// refuse to load legitimately created tables whenever the default has changed since + /// creation (pre-26.6 wildcard tables under the 26.6 `hive` default, or implicit-hive + /// tables loaded under a `wildcard` default after a downgrade), aborting server startup + /// and breaking upgrades. Only a user-issued `CREATE` applies the default. + partition_strategy_type = getRawPath().hasPartitionWildcard() + ? PartitionStrategyFactory::StrategyType::WILDCARD + : PartitionStrategyFactory::StrategyType::HIVE; + } + else + { + switch (context->getSettingsRef()[Setting::file_like_engine_default_partition_strategy].value) { - partition_strategy_type = PartitionStrategyFactory::StrategyType::HIVE; - break; + case FileLikeEngineDefaultPartitionStrategy::WILDCARD: + { + /// Set the strategy unconditionally; `PartitionStrategyFactory::get` will raise + /// `BAD_ARGUMENTS` if the path is missing the `{_partition_id}` placeholder. + partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; + break; + } + case FileLikeEngineDefaultPartitionStrategy::HIVE: + { + partition_strategy_type = PartitionStrategyFactory::StrategyType::HIVE; + break; + } } } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h index aebf4baff2c0..71542cf33072 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h @@ -324,6 +324,14 @@ class StorageObjectStorageConfiguration bool partition_columns_in_data_file_was_set = false; std::shared_ptr partition_strategy; + /// False when the storage is instantiated from anything other than a user-issued `CREATE` + /// (ATTACH, server startup, RESTORE, replicated-DDL replay). `initPartitionStrategy` must not + /// apply the `file_like_engine_default_partition_strategy` default to such tables: a pre-26.6 + /// table with a `{_partition_id}` path was created under the implicit wildcard strategy, and + /// re-deriving `hive` from the current default on ATTACH would refuse to load it and abort + /// server startup/upgrade. Defaults to true so table functions keep strict validation. + bool is_create_query = true; + protected: void initializeFromParsedArguments(const StorageParsedArguments & parsed_arguments); virtual void fromNamedCollection(const NamedCollection & collection, ContextPtr context) = 0; diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index b1ca240b9689..e52d484bd8cd 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -80,6 +80,12 @@ createStorageObjectStorage(const StorageFactory::Arguments & args, StorageObject ContextMutablePtr context_copy = Context::createCopy(args.getContext()); Settings settings_copy = args.getLocalContext()->getSettingsCopy(); context_copy->setSettings(settings_copy); + + /// Only a user-issued `CREATE` may apply the `file_like_engine_default_partition_strategy` + /// default; ATTACH / startup / RESTORE / replicated-DDL replay must load pre-existing + /// `{_partition_id}` tables as wildcard (see `initPartitionStrategy`). + configuration->is_create_query = args.mode == LoadingStrictnessLevel::CREATE; + return std::make_shared( configuration, // We only want to perform write actions (e.g. create a container in Azure) when the table is being created, @@ -416,7 +422,8 @@ ENGINE = S3( 'http://minio:10000/clickhouse//test_{_partition_id}.csv', 'minioadmin', 'minioadminpassword', - 'CSV') + 'CSV', + partition_strategy='wildcard') PARTITION BY column3 ``` diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index a06fd3d887f7..fab19dc51469 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -572,7 +572,7 @@ def test_when_s3_timeout_at_listing( TABLE FUNCTION s3( 'http://resolver:8083/root/data/test_when_s3_timeout_at_listing/{{_partition_id}}/file', 'minio', '{minio_secret_key}', - 'CSV', auto, 'none' + 'CSV', auto, 'none', partition_strategy='wildcard' ) PARTITION BY number SELECT diff --git a/tests/integration/test_s3_non_deterministic_partition_by/test.py b/tests/integration/test_s3_non_deterministic_partition_by/test.py index 8b9b7a05fb77..9410d05790d1 100644 --- a/tests/integration/test_s3_non_deterministic_partition_by/test.py +++ b/tests/integration/test_s3_non_deterministic_partition_by/test.py @@ -30,7 +30,7 @@ def test_s3_non_deterministic_partition_by(started_cluster): ( s String ) - ENGINE = S3('http://minio1:9001/{started_cluster.minio_bucket}/{{_partition_id}}.parquet', 'minio', 'ClickHouse_Minio_P@ssw0rd') + ENGINE = S3('http://minio1:9001/{started_cluster.minio_bucket}/{{_partition_id}}.parquet', 'minio', 'ClickHouse_Minio_P@ssw0rd', partition_strategy='wildcard') PARTITION BY concat(s, toString(now64(9))) """ ) diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 8cb01fe5d86c..a0a2a4e28eb8 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -237,7 +237,7 @@ def test_partition_by(cluster): azure_query( node, f"CREATE TABLE test_partitioned_write ({table_format}) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}'," - f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') " + f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', 'wildcard') " f"PARTITION BY {partition_by}", ) azure_query(node, f"INSERT INTO test_partitioned_write VALUES {values}") @@ -258,7 +258,7 @@ def test_partition_by_string_column(cluster): azure_query( node, f"CREATE TABLE test_partitioned_string_write ({table_format}) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}'," - f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') " + f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', 'wildcard') " f"PARTITION BY {partition_by}", ) azure_query(node, f"INSERT INTO test_partitioned_string_write VALUES {values}") @@ -280,7 +280,7 @@ def test_partition_by_const_column(cluster): azure_query( node, f"CREATE TABLE test_partitioned_const_write ({table_format}) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}'," - f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV')" + f" 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', 'wildcard')" f" PARTITION BY {partition_by}", ) azure_query(node, f"INSERT INTO test_partitioned_const_write VALUES {values}") @@ -821,7 +821,7 @@ def test_partition_by_tf(cluster): node, f"INSERT INTO TABLE FUNCTION azureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', " f"'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', " - f"'CSV', 'auto', '{table_format}') PARTITION BY {partition_by} VALUES {values}", + f"'CSV', 'auto', 'wildcard', '{table_format}') PARTITION BY {partition_by} VALUES {values}", settings={"azure_truncate_on_insert": 1}, ) @@ -841,7 +841,7 @@ def test_filter_using_file(cluster): node, f"INSERT INTO TABLE FUNCTION azureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', 'cont', '{filename}', " f"'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', " - f"'{table_format}') PARTITION BY {partition_by} VALUES {values}", + f"'wildcard', '{table_format}') PARTITION BY {partition_by} VALUES {values}", settings={"azure_truncate_on_insert": 1}, ) @@ -1585,7 +1585,7 @@ def test_write_to_globbed_partitioned_path(cluster): account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" error = azure_query( node, - f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_data_*_{{_partition_id}}', '{account_name}', '{account_key}', 'CSV', 'auto', 'x UInt64') partition by 42 select 42 SETTINGS azure_truncate_on_insert=1", + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_data_*_{{_partition_id}}', '{account_name}', '{account_key}', 'CSV', 'auto', 'wildcard', 'x UInt64') partition by 42 select 42 SETTINGS azure_truncate_on_insert=1", expect_error="true", ) @@ -1634,7 +1634,7 @@ def test_respect_object_existence_on_partitioned_write(cluster): error = azure_query( node, - f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}') partition by 42 select 42 settings azure_truncate_on_insert=0", + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}', 'CSV', 'auto', 'wildcard') partition by 42 select 42 settings azure_truncate_on_insert=0", expect_error="true", ) @@ -1642,7 +1642,7 @@ def test_respect_object_existence_on_partitioned_write(cluster): azure_query( node, - f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}') partition by 42 select 43 settings azure_truncate_on_insert=1", + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}', 'CSV', 'auto', 'wildcard') partition by 42 select 43 settings azure_truncate_on_insert=1", ) result = azure_query( @@ -1654,7 +1654,7 @@ def test_respect_object_existence_on_partitioned_write(cluster): azure_query( node, - f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}') partition by 42 select 44 settings azure_truncate_on_insert=0, azure_create_new_file_on_insert=1", + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_partitioned_write{{_partition_id}}.csv', '{account_name}', '{account_key}', 'CSV', 'auto', 'wildcard') partition by 42 select 44 settings azure_truncate_on_insert=0, azure_create_new_file_on_insert=1", ) result = azure_query( diff --git a/tests/integration/test_storage_azure_blob_storage/test_cluster.py b/tests/integration/test_storage_azure_blob_storage/test_cluster.py index 264c6d158b79..602af40f7c66 100644 --- a/tests/integration/test_storage_azure_blob_storage/test_cluster.py +++ b/tests/integration/test_storage_azure_blob_storage/test_cluster.py @@ -242,7 +242,7 @@ def test_partition_parallel_reading_with_cluster(cluster): azure_query( node, f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', '{filename}', 'devstoreaccount1', " - f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', '{table_format}') " + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto', 'wildcard', '{table_format}') " f"PARTITION BY {partition_by} VALUES {values}", settings={"azure_truncate_on_insert": 1}, ) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index b325daa89cf6..692fc235f9ee 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -242,7 +242,7 @@ def test_partition_by(started_cluster): values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)" filename = "test_{_partition_id}.csv" put_query = f"""INSERT INTO TABLE FUNCTION - s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}') + s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}', partition_strategy='wildcard') PARTITION BY {partition_by} VALUES {values}""" run_query(instance, put_query) @@ -255,7 +255,7 @@ def test_partition_by(started_cluster): filename = "test2_{_partition_id}.csv" instance.query( - f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV') partition by column3" + f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', partition_strategy='wildcard') partition by column3" ) instance.query(f"insert into p values {values}") assert "1,2,3\n" == get_s3_file_content( @@ -280,7 +280,7 @@ def test_partition_by_string_column(started_cluster): values = "(1, 'foo/bar'), (3, 'йцук'), (78, '你好')" filename = "test_{_partition_id}.csv" put_query = f"""INSERT INTO TABLE FUNCTION - s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}') + s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}', partition_strategy='wildcard') PARTITION BY {partition_by} VALUES {values}""" run_query(instance, put_query) @@ -306,7 +306,7 @@ def test_partition_by_const_column(started_cluster): values_csv = "1,2,3\n3,2,1\n78,43,45\n" filename = "test_{_partition_id}.csv" put_query = f"""INSERT INTO TABLE FUNCTION - s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}') + s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}', partition_strategy='wildcard') PARTITION BY {partition_by} VALUES {values}""" run_query(instance, put_query) @@ -2155,7 +2155,7 @@ def test_s3_list_objects_failure(started_cluster): put_query = f""" INSERT INTO TABLE FUNCTION - s3('http://resolver:8083/{bucket}/{filename}', 'CSV', 'c1 UInt32') + s3('http://resolver:8083/{bucket}/{filename}', 'CSV', 'c1 UInt32', partition_strategy='wildcard') PARTITION BY c1 % 20 SELECT number FROM numbers(100) SETTINGS s3_truncate_on_insert=1 @@ -2504,13 +2504,13 @@ def test_respect_object_existence_on_partitioned_write(started_cluster): assert int(result) == 42 error = instance.query_and_get_error( - f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64') partition by 42 select 42 settings s3_truncate_on_insert=0" + f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64', partition_strategy='wildcard') partition by 42 select 42 settings s3_truncate_on_insert=0" ) assert "BAD_ARGUMENTS" in error instance.query( - f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64') partition by 42 select 43 settings s3_truncate_on_insert=1" + f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64', partition_strategy='wildcard') partition by 42 select 43 settings s3_truncate_on_insert=1" ) result = instance.query( @@ -2520,7 +2520,7 @@ def test_respect_object_existence_on_partitioned_write(started_cluster): assert int(result) == 43 instance.query( - f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64') partition by 42 select 44 settings s3_truncate_on_insert=0, s3_create_new_file_on_insert=1" + f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_partitioned_write{{_partition_id}}.csv', 'CSV', 'x UInt64', partition_strategy='wildcard') partition by 42 select 44 settings s3_truncate_on_insert=0, s3_create_new_file_on_insert=1" ) result = instance.query( diff --git a/tests/queries/0_stateless/01944_insert_partition_by.sql b/tests/queries/0_stateless/01944_insert_partition_by.sql index 03bbd17b8ce7..a747c7a60344 100644 --- a/tests/queries/0_stateless/01944_insert_partition_by.sql +++ b/tests/queries/0_stateless/01944_insert_partition_by.sql @@ -1,6 +1,8 @@ -- Tags: no-fasttest -- Tag no-fasttest: needs s3 +SET file_like_engine_default_partition_strategy = 'wildcard'; + INSERT INTO TABLE FUNCTION s3('http://localhost:9001/foo/test_{_partition_id}.csv', 'admin', 'admin', 'CSV', 'id Int32, val String') PARTITION BY val VALUES (1, '\r\n'); -- { serverError CANNOT_PARSE_TEXT } INSERT INTO TABLE FUNCTION s3('http://localhost:9001/foo/test_{_partition_id}.csv', 'admin', 'admin', 'CSV', 'id Int32, val String') PARTITION BY val VALUES (1, 'abc\x00abc'); -- { serverError CANNOT_PARSE_TEXT } INSERT INTO TABLE FUNCTION s3('http://localhost:9001/foo/test_{_partition_id}.csv', 'admin', 'admin', 'CSV', 'id Int32, val String') PARTITION BY val VALUES (1, 'abc\xc3\x28abc'); -- { serverError CANNOT_PARSE_TEXT } diff --git a/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference b/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference index dca377143d07..1a3416a82ae1 100644 --- a/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference +++ b/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference @@ -3,33 +3,33 @@ drop table if exists test_02245_s3_nested_parquet1; drop table if exists test_02245_s3_nested_parquet2; set input_format_parquet_import_nested = 1; set s3_truncate_on_insert = 1; -create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet') partition by a; +create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_parquet1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_parquet1_*', format='Parquet'); 1 2 a -create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet') partition by a; +create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_parquet2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_parquet2_*', format='Parquet', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); 1 2 3 a drop table if exists test_02245_s3_nested_arrow1; drop table if exists test_02245_s3_nested_arrow2; set input_format_arrow_import_nested=1; -create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow') partition by a; +create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_arrow1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_arrow1_*', format='Arrow'); 1 2 a -create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow') partition by a; +create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_arrow2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_arrow2_*', format='Arrow', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); 1 2 3 a drop table if exists test_02245_s3_nested_orc1; drop table if exists test_02245_s3_nested_orc2; set input_format_orc_import_nested=1; -create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC') partition by a; +create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_orc1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_orc1_*', format='ORC'); 1 2 a -create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC') partition by a; +create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_orc2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_orc2_*', format='ORC', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); 1 2 3 a diff --git a/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql b/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql index 921f27a0a66e..fe8592d4b447 100644 --- a/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql +++ b/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql @@ -6,12 +6,12 @@ drop table if exists test_02245_s3_nested_parquet1; drop table if exists test_02245_s3_nested_parquet2; set input_format_parquet_import_nested = 1; set s3_truncate_on_insert = 1; -create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet') partition by a; +create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_parquet1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_parquet1_*', format='Parquet'); -create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet') partition by a; +create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_parquet2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_parquet2_*', format='Parquet', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); @@ -20,12 +20,12 @@ select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_parq drop table if exists test_02245_s3_nested_arrow1; drop table if exists test_02245_s3_nested_arrow2; set input_format_arrow_import_nested=1; -create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow') partition by a; +create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_arrow1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_arrow1_*', format='Arrow'); -create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow') partition by a; +create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_arrow2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_arrow2_*', format='Arrow', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); @@ -34,12 +34,12 @@ select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_arro drop table if exists test_02245_s3_nested_orc1; drop table if exists test_02245_s3_nested_orc2; set input_format_orc_import_nested=1; -create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC') partition by a; +create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_orc1 values (1, (2, 'a')); select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_orc1_*', format='ORC'); -create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC') partition by a; +create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC', partition_strategy='wildcard') partition by a; insert into test_02245_s3_nested_orc2 values (1, (2, (3, 'a'))); select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_orc2_*', format='ORC', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.reference b/tests/queries/0_stateless/02302_s3_file_pruning.reference index 52de703714ad..dc91ff073383 100644 --- a/tests/queries/0_stateless/02302_s3_file_pruning.reference +++ b/tests/queries/0_stateless/02302_s3_file_pruning.reference @@ -1,6 +1,6 @@ -- { echo } drop table if exists test_02302; -create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; insert into test_02302 select number from numbers(10) settings s3_truncate_on_insert=1; select * from test_02302; -- { serverError NOT_IMPLEMENTED } drop table test_02302; diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.sql b/tests/queries/0_stateless/02302_s3_file_pruning.sql index 58afb682face..73540b3949c0 100644 --- a/tests/queries/0_stateless/02302_s3_file_pruning.sql +++ b/tests/queries/0_stateless/02302_s3_file_pruning.sql @@ -5,7 +5,7 @@ SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injectio -- { echo } drop table if exists test_02302; -create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; insert into test_02302 select number from numbers(10) settings s3_truncate_on_insert=1; select * from test_02302; -- { serverError NOT_IMPLEMENTED } drop table test_02302; diff --git a/tests/queries/0_stateless/02480_s3_support_wildcard.reference b/tests/queries/0_stateless/02480_s3_support_wildcard.reference index a36b7cd2c200..1e2c1d1139c3 100644 --- a/tests/queries/0_stateless/02480_s3_support_wildcard.reference +++ b/tests/queries/0_stateless/02480_s3_support_wildcard.reference @@ -1,7 +1,7 @@ -- { echo } drop table if exists test_02480_support_wildcard_write; drop table if exists test_02480_support_wildcard_write2; -create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a; +create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c'); select a, b from s3(s3_conn, filename='test_02480_support_wildcard_*', format=Parquet) order by a; @@ -22,7 +22,7 @@ select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', fo 1 a 22 b 333 c -create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a; +create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g'); select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_*', format=Parquet) order by a; diff --git a/tests/queries/0_stateless/02480_s3_support_wildcard.sql b/tests/queries/0_stateless/02480_s3_support_wildcard.sql index 91d9dae2fd1f..450cfa3ba57b 100644 --- a/tests/queries/0_stateless/02480_s3_support_wildcard.sql +++ b/tests/queries/0_stateless/02480_s3_support_wildcard.sql @@ -4,7 +4,7 @@ -- { echo } drop table if exists test_02480_support_wildcard_write; drop table if exists test_02480_support_wildcard_write2; -create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a; +create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c'); @@ -15,7 +15,7 @@ select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?*?', format= select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1,333}', format=Parquet) order by a; select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', format=Parquet) order by a; -create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a; +create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g'); diff --git a/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.reference b/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.reference index a7096a686f51..87ff2b81f08d 100644 --- a/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.reference +++ b/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.reference @@ -1,6 +1,6 @@ -- { echo } drop table if exists test_02481_mismatch_files; -create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a; +create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c'); select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } diff --git a/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.sql b/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.sql index 7ec1d3ebd5f8..d90a049f4ada 100644 --- a/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.sql +++ b/tests/queries/0_stateless/02481_s3_throw_if_mismatch_files.sql @@ -3,7 +3,7 @@ -- { echo } drop table if exists test_02481_mismatch_files; -create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a; +create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet, partition_strategy='wildcard') partition by a; set s3_truncate_on_insert=1; insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c'); diff --git a/tests/queries/0_stateless/02495_s3_filter_by_file.sql b/tests/queries/0_stateless/02495_s3_filter_by_file.sql index 8d6d8a8a5a40..03e02cb9d5c0 100644 --- a/tests/queries/0_stateless/02495_s3_filter_by_file.sql +++ b/tests/queries/0_stateless/02495_s3_filter_by_file.sql @@ -3,7 +3,7 @@ DROP TABLE IF EXISTS t_s3_filter_02495; CREATE TABLE t_s3_filter_02495 (a UInt64) -ENGINE = S3(s3_conn, filename = 'test_02495_{_partition_id}', format = Parquet) +ENGINE = S3(s3_conn, filename = 'test_02495_{_partition_id}', format = Parquet, partition_strategy = 'wildcard') PARTITION BY a; INSERT INTO t_s3_filter_02495 SELECT number FROM numbers(10) SETTINGS s3_truncate_on_insert=1; diff --git a/tests/queries/0_stateless/02496_storage_s3_profile_events.sql b/tests/queries/0_stateless/02496_storage_s3_profile_events.sql index 26280921e28a..cb04c020c693 100644 --- a/tests/queries/0_stateless/02496_storage_s3_profile_events.sql +++ b/tests/queries/0_stateless/02496_storage_s3_profile_events.sql @@ -3,7 +3,7 @@ DROP TABLE IF EXISTS t_s3_events_02496; CREATE TABLE t_s3_events_02496 (a UInt64) -ENGINE = S3(s3_conn, filename = 'test_02496_{_partition_id}', format = Parquet) +ENGINE = S3(s3_conn, filename = 'test_02496_{_partition_id}', format = Parquet, partition_strategy = 'wildcard') PARTITION BY a; INSERT INTO t_s3_events_02496 SELECT number FROM numbers(10) SETTINGS s3_truncate_on_insert=1; diff --git a/tests/queries/0_stateless/03037_s3_write_to_globbed_partitioned_path.sql b/tests/queries/0_stateless/03037_s3_write_to_globbed_partitioned_path.sql index 1de89a593b06..78a2dc04dae4 100644 --- a/tests/queries/0_stateless/03037_s3_write_to_globbed_partitioned_path.sql +++ b/tests/queries/0_stateless/03037_s3_write_to_globbed_partitioned_path.sql @@ -1,4 +1,6 @@ -- Tags: no-fasttest +SET file_like_engine_default_partition_strategy = 'wildcard'; + insert into function s3('http://localhost:11111/test/data_*_{_partition_id}.csv') partition by number % 3 select * from numbers(10); -- {serverError DATABASE_ACCESS_DENIED} diff --git a/tests/queries/0_stateless/03528_s3_insert_partition_by_whitespaces.sql b/tests/queries/0_stateless/03528_s3_insert_partition_by_whitespaces.sql index ffb0ad860df3..8cae61f2d24d 100644 --- a/tests/queries/0_stateless/03528_s3_insert_partition_by_whitespaces.sql +++ b/tests/queries/0_stateless/03528_s3_insert_partition_by_whitespaces.sql @@ -5,7 +5,8 @@ INSERT INTO FUNCTION s3( s3_conn, filename = currentDatabase() || '/{_partition_id}/test.parquet', - format = Parquet + format = Parquet, + partition_strategy = 'wildcard' ) PARTITION BY 1 SELECT @@ -19,7 +20,8 @@ INSERT INTO FUNCTION s3( s3_conn, filename = currentDatabase() || '/{_partition_id}/test.parquet', - format = Parquet + format = Parquet, + partition_strategy = 'wildcard' ) PARTITION BY 2 SELECT * FROM system.numbers diff --git a/tests/queries/0_stateless/03629_storage_s3_disallow_index_alter.sql b/tests/queries/0_stateless/03629_storage_s3_disallow_index_alter.sql index fb0384aa8d39..4ec3c59deb3e 100644 --- a/tests/queries/0_stateless/03629_storage_s3_disallow_index_alter.sql +++ b/tests/queries/0_stateless/03629_storage_s3_disallow_index_alter.sql @@ -3,7 +3,7 @@ -- Issue: https://github.com/ClickHouse/ClickHouse/issues/87059 DROP TABLE IF EXISTS test_03629; -CREATE TABLE test_03629 (a UInt64) ENGINE = S3(s3_conn, filename='test_03629_{_partition_id}', format='Native') PARTITION BY a; +CREATE TABLE test_03629 (a UInt64) ENGINE = S3(s3_conn, filename='test_03629_{_partition_id}', format='Native', partition_strategy='wildcard') PARTITION BY a; ALTER TABLE test_03629 ADD INDEX a_idx a TYPE set(0); -- { serverError NOT_IMPLEMENTED } ALTER TABLE test_03629 ADD PROJECTION a_proj (SELECT a + 1 ORDER BY a); -- { serverError NOT_IMPLEMENTED } diff --git a/tests/queries/0_stateless/04000_s3_table_function_small_file_prefetch.sql b/tests/queries/0_stateless/04000_s3_table_function_small_file_prefetch.sql index fccfb0f519bf..f103bdf62818 100644 --- a/tests/queries/0_stateless/04000_s3_table_function_small_file_prefetch.sql +++ b/tests/queries/0_stateless/04000_s3_table_function_small_file_prefetch.sql @@ -10,7 +10,7 @@ -- Write 16 tiny files. Each is far below 2 * max_download_buffer_size, i.e. "object_too_small", -- so it takes the prefetch path. -INSERT INTO FUNCTION s3(s3_conn, filename='04000_prefetch_{_partition_id}.tsv', format='TSV') +INSERT INTO FUNCTION s3(s3_conn, filename='04000_prefetch_{_partition_id}.tsv', format='TSV', partition_strategy='wildcard') PARTITION BY (a % 16) SELECT number AS a, toString(number) AS b FROM numbers(16000) diff --git a/tests/queries/0_stateless/04004_s3_schema_hash.reference b/tests/queries/0_stateless/04004_s3_schema_hash.reference index 9de831829d66..4e23688d3802 100644 --- a/tests/queries/0_stateless/04004_s3_schema_hash.reference +++ b/tests/queries/0_stateless/04004_s3_schema_hash.reference @@ -20,7 +20,7 @@ select a, b from test_04004_hash_write order by a; 1 hello 2 world -- Test 3: Combined {_schema_hash} and {_partition_id} — write via table, read via s3() glob -create table test_04004_hash_partitioned (a UInt64, b String) engine = S3(s3_conn, filename='test_04004/{_schema_hash}/{_partition_id}/data.parquet', format=Parquet) partition by a; +create table test_04004_hash_partitioned (a UInt64, b String) engine = S3(s3_conn, filename='test_04004/{_schema_hash}/{_partition_id}/data.parquet', format=Parquet, partition_strategy='wildcard') partition by a; insert into test_04004_hash_partitioned values (1, 'foo'), (2, 'bar'), (3, 'baz'); select a, b from s3(s3_conn, filename='test_04004/*/*/data.parquet', format=Parquet) order by a; 1 foo diff --git a/tests/queries/0_stateless/04004_s3_schema_hash.sql b/tests/queries/0_stateless/04004_s3_schema_hash.sql index 9be7e47141fd..313baebff9b5 100644 --- a/tests/queries/0_stateless/04004_s3_schema_hash.sql +++ b/tests/queries/0_stateless/04004_s3_schema_hash.sql @@ -21,7 +21,7 @@ select round(x, 2) from test_04004_hash_write2 order by x; select a, b from test_04004_hash_write order by a; -- Test 3: Combined {_schema_hash} and {_partition_id} — write via table, read via s3() glob -create table test_04004_hash_partitioned (a UInt64, b String) engine = S3(s3_conn, filename='test_04004/{_schema_hash}/{_partition_id}/data.parquet', format=Parquet) partition by a; +create table test_04004_hash_partitioned (a UInt64, b String) engine = S3(s3_conn, filename='test_04004/{_schema_hash}/{_partition_id}/data.parquet', format=Parquet, partition_strategy='wildcard') partition by a; insert into test_04004_hash_partitioned values (1, 'foo'), (2, 'bar'), (3, 'baz'); select a, b from s3(s3_conn, filename='test_04004/*/*/data.parquet', format=Parquet) order by a; diff --git a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference new file mode 100644 index 000000000000..d24715a06873 --- /dev/null +++ b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference @@ -0,0 +1,4 @@ +1 +1 +2 +3 diff --git a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql new file mode 100644 index 000000000000..32dc5efd2bf6 --- /dev/null +++ b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql @@ -0,0 +1,52 @@ +-- Tags: no-fasttest, no-random-settings +-- Tag no-fasttest: Depends on S3 + +SET compatibility = '26.6'; +CREATE TABLE old_export (d Date, x UInt64) +ENGINE = S3('s3://bucket/export/data_{_partition_id}.parquet', 'Parquet') +PARTITION BY d; -- {serverError BAD_ARGUMENTS} + +SET compatibility = '26.5'; +CREATE TABLE old_export_compat_265 (d Date, x UInt64) +ENGINE = S3('s3://bucket/export/data_{_partition_id}.parquet', 'Parquet') +PARTITION BY d; +SELECT 1; + +SET compatibility = '26.6'; +SET file_like_engine_default_partition_strategy = 'wildcard'; +CREATE TABLE old_export2 (d Date, x UInt64) +ENGINE = S3('s3://bucket/export/data_{_partition_id}.parquet', 'Parquet') +PARTITION BY d; +SELECT 1; + +-- Backward compatibility: a pre-26.6 table with a `{_partition_id}` path (implicit wildcard) +-- must still load via ATTACH under the 26.6 `hive` default — the same code path the server +-- takes for every such table at startup and during upgrades. Before the fix this threw +-- `BAD_ARGUMENTS` and aborted server startup. +-- The explicit `hive` below is required: `SET compatibility` does not override the +-- explicitly-set `file_like_engine_default_partition_strategy = 'wildcard'` above, and the +-- ATTACH must run with the `hive` default in effect to be a real regression test. +SET compatibility = '26.6'; +SET file_like_engine_default_partition_strategy = 'hive'; +DETACH TABLE old_export_compat_265; +ATTACH TABLE old_export_compat_265; +SELECT 2; + +DROP TABLE IF EXISTS old_export; -- never created: the first CREATE above is expected to throw +DROP TABLE old_export_compat_265; +DROP TABLE old_export2; + +-- Mirror case: an implicit-`hive` table (created under the 26.6 default, no `{_partition_id}` +-- in the path) must still load via ATTACH when the effective default is `wildcard` +-- (e.g. a downgrade or a `compatibility = '26.5'` session). The strategy on load is derived +-- from the path shape, never from the mutable default. +SET compatibility = '26.6'; +SET file_like_engine_default_partition_strategy = 'hive'; +CREATE TABLE hive_export (d Date, x UInt64) +ENGINE = S3('s3://bucket/export2', 'Parquet') +PARTITION BY d; +SET file_like_engine_default_partition_strategy = 'wildcard'; +DETACH TABLE hive_export; +ATTACH TABLE hive_export; +SELECT 3; +DROP TABLE hive_export; From 9c43abf855811e4dcc77e94d8cfa5bcaf7527755 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Jul 2026 13:33:12 +0000 Subject: [PATCH 16/86] Backport #109886 to 26.6: Optimize text index analysis --- src/Common/SipHash.h | 11 ++- .../optimizeDirectReadFromTextIndex.cpp | 10 +-- .../MergeTree/MergeTreeIndexConditionText.cpp | 66 +++++++++------ .../MergeTree/MergeTreeIndexConditionText.h | 27 +++++- src/Storages/MergeTree/MergeTreeIndexText.cpp | 12 +-- .../MergeTree/MergeTreeReaderTextIndex.cpp | 72 ++++++++-------- .../MergeTree/MergeTreeReaderTextIndex.h | 2 +- src/Storages/MergeTree/TextIndexAnalyzer.cpp | 36 ++++---- src/Storages/MergeTree/TextIndexAnalyzer.h | 4 +- .../02346_text_index_bug90778.reference | 4 +- .../02346_text_index_direct_read.reference | 36 ++++---- ...2346_text_index_duplicate_tokens.reference | 4 +- .../02346_text_index_hint.reference | 8 +- .../02346_text_index_hint_map.reference | 24 +++--- ...02346_text_index_materialization.reference | 6 +- ...02346_text_index_on_lower_column.reference | 8 +- ...2346_text_index_prewhere_support.reference | 84 +++++++++---------- ...tokenizer_partially_materialized.reference | 4 +- ...093_text_index_separate_analysis.reference | 6 +- .../04102_text_index_hasAny_hasAll.reference | 6 +- 20 files changed, 236 insertions(+), 194 deletions(-) diff --git a/src/Common/SipHash.h b/src/Common/SipHash.h index 0adf54b7c23d..40780fffa653 100644 --- a/src/Common/SipHash.h +++ b/src/Common/SipHash.h @@ -228,12 +228,12 @@ inline CityHash_v1_0_2::uint128 getSipHash128AsPair(SipHash & sip_hash) return result; } -inline String getSipHash128AsHexString(SipHash & sip_hash) +inline String getSipHash128AsHexString(const UInt128 & hash) { String result; - const auto hash_data = getSipHash128AsArray(sip_hash); - const auto hash_size = hash_data.size(); + const auto * hash_data = reinterpret_cast(&hash); + const auto hash_size = sizeof(hash); result.resize(hash_size * 2); for (size_t i = 0; i < hash_size; ++i) { @@ -245,6 +245,11 @@ inline String getSipHash128AsHexString(SipHash & sip_hash) return result; } +inline String getSipHash128AsHexString(SipHash & sip_hash) +{ + return getSipHash128AsHexString(sip_hash.get128()); +} + inline UInt128 sipHash128Keyed(UInt64 key0, UInt64 key1, const char * data, const size_t size) { SipHash hash(key0, key1); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index d2c725219646..f8c2eb5104ac 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -464,7 +464,7 @@ class TextIndexDAGReplacer continue; /// For None mode, the condition is still needed for preprocessing (tokenizer/preprocessor injection). - if (search_query->direct_read_mode == TextIndexDirectReadMode::None) + if (search_query->getDirectReadMode() == TextIndexDirectReadMode::None) { selected_conditions.emplace_back(search_query, index_name, String{}, &info); used_index_columns.insert(index_header.begin()->name); @@ -727,7 +727,7 @@ class TextIndexDAGReplacer std::vector selected_conditions; for (const auto & condition : all_conditions) { - if (condition.search_query->direct_read_mode != TextIndexDirectReadMode::None) + if (condition.search_query->getDirectReadMode() != TextIndexDirectReadMode::None) selected_conditions.push_back(condition); } if (selected_conditions.empty()) @@ -739,7 +739,7 @@ class TextIndexDAGReplacer for (const auto & condition : selected_conditions) { has_materialized_index |= condition.info->is_materialized; - has_exact_search |= condition.search_query->direct_read_mode == TextIndexDirectReadMode::Exact; + has_exact_search |= condition.search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact; } /// It doesn't make sense to optimize if index is not materialized in any data part. @@ -756,10 +756,10 @@ class TextIndexDAGReplacer /// It will be executed by merge tree reader when index is not materialized in the data part. ASTPtr default_expression; - if (condition.search_query->direct_read_mode == TextIndexDirectReadMode::Exact) + if (condition.search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact) default_expression = convertNodeToAST(function_node); /// Do not execute the default expression for hint mode, because it will be executed anyway in the original predicate. - else if (condition.search_query->direct_read_mode == TextIndexDirectReadMode::Hint) + else if (condition.search_query->getDirectReadMode() == TextIndexDirectReadMode::Hint) default_expression = make_intrusive(Field(1)); VirtualColumnDescription virtual_column(condition.virtual_column_name, std::make_shared(), /*codec=*/ nullptr, condition.index_name, VirtualsKind::Ephemeral, VirtualsMaterializationPlace::Reader); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 6d9995920735..b4e6f021425e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -54,38 +54,46 @@ namespace Setting extern const SettingsBool reject_expensive_hyperscan_regexps; } -TextSearchQuery::TextSearchQuery(String function_name_, TextSearchMode search_mode_, TextIndexDirectReadMode direct_read_mode_, VectorWithMemoryTracking tokens_, std::vector patterns_) +TextSearchQuery::TextSearchQuery( + String function_name_, + TextSearchMode search_mode_, + TextIndexDirectReadMode direct_read_mode_, + VectorWithMemoryTracking tokens_, + std::vector patterns_, + VectorWithMemoryTracking phrase_tokens_) : function_name(std::move(function_name_)) , search_mode(search_mode_) , direct_read_mode(direct_read_mode_) , tokens(std::move(tokens_)) , patterns(std::move(patterns_)) + , phrase_tokens(std::move(phrase_tokens_)) { std::sort(tokens.begin(), tokens.end()); + initializeHash(); } -SipHash TextSearchQuery::getHash() const +void TextSearchQuery::initializeHash() { - SipHash hash; - hash.update(function_name); - hash.update(search_mode); - hash.update(direct_read_mode); + SipHash hash_state; + hash_state.update(function_name); + hash_state.update(search_mode); + hash_state.update(direct_read_mode); - hash.update(tokens.size()); + hash_state.update(tokens.size()); for (const auto & token : tokens) { - hash.update(token.size()); - hash.update(token); + hash_state.update(token.size()); + hash_state.update(token); } if (!patterns.empty()) { - hash.update(patterns.size()); + hash_state.update(patterns.size()); for (const auto & pattern : patterns) { if (const auto & re2 = pattern.getRE2()) { - hash.update(re2->pattern()); + hash_state.update(re2->pattern()); } else { @@ -93,24 +101,24 @@ SipHash TextSearchQuery::getHash() const bool is_trivial = false; bool required_substring_is_prefix = false; pattern.getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix); - hash.update(required_substring); - hash.update(is_trivial); - hash.update(required_substring_is_prefix); + hash_state.update(required_substring); + hash_state.update(is_trivial); + hash_state.update(required_substring_is_prefix); } } } if (!phrase_tokens.empty()) { - hash.update(phrase_tokens.size()); + hash_state.update(phrase_tokens.size()); for (const auto & token : phrase_tokens) { - hash.update(token.size()); - hash.update(token); + hash_state.update(token.size()); + hash_state.update(token); } } - return hash; + hash = hash_state.get128(); } MergeTreeIndexConditionText::MergeTreeIndexConditionText( @@ -175,8 +183,8 @@ MergeTreeIndexConditionText::MergeTreeIndexConditionText( { for (const auto & search_query : element.text_search_queries) { - all_search_tokens_set.insert(search_query->tokens.begin(), search_query->tokens.end()); - all_search_queries[search_query->getHash().get128()] = search_query; + all_search_tokens_set.insert(search_query->getTokens().begin(), search_query->getTokens().end()); + all_search_queries[search_query->getHash()] = search_query; } if (requiresReadingAllTokens(element)) @@ -312,17 +320,17 @@ TextSearchQueryPtr MergeTreeIndexConditionText::createTextSearchQuery(const Acti std::optional MergeTreeIndexConditionText::replaceToVirtualColumn(const TextSearchQuery & query, const String & index_name) { - if (query.tokens.empty() && query.patterns.empty() && query.direct_read_mode == TextIndexDirectReadMode::Hint) + if (query.getTokens().empty() && query.getPatterns().empty() && query.getDirectReadMode() == TextIndexDirectReadMode::Hint) return std::nullopt; auto query_hash = query.getHash(); - auto it = all_search_queries.find(query_hash.get128()); + auto it = all_search_queries.find(query_hash); if (it == all_search_queries.end()) return std::nullopt; auto hash_str = getSipHash128AsHexString(query_hash); - String virtual_column_name = fmt::format("{}{}_{}_{}", TEXT_INDEX_VIRTUAL_COLUMN_PREFIX, index_name, query.function_name, hash_str); + String virtual_column_name = fmt::format("{}{}_{}_{}", TEXT_INDEX_VIRTUAL_COLUMN_PREFIX, index_name, query.getFunctionName(), hash_str); virtual_column_to_search_query[virtual_column_name] = it->second; return virtual_column_name; @@ -487,7 +495,7 @@ std::string MergeTreeIndexConditionText::getDescription() const bool MergeTreeIndexConditionText::hasSearchPatterns() const { - return std::ranges::any_of(all_search_queries, [](const auto & query) { return !query.second->patterns.empty(); }); + return std::ranges::any_of(all_search_queries, [](const auto & query) { return !query.second->getPatterns().empty(); }); } bool MergeTreeIndexConditionText::traverseAtomNode(const RPNBuilderTreeNode & node, RPNElement & out) const @@ -1074,8 +1082,14 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( std::sort(unique_tokens.begin(), unique_tokens.end()); } - auto query = std::make_shared(function_name, TextSearchMode::Phrase, direct_read_mode, std::move(unique_tokens)); - query->phrase_tokens = std::move(phrase_tokens); + auto query = std::make_shared( + function_name, + TextSearchMode::Phrase, + direct_read_mode, + std::move(unique_tokens), + std::vector{}, + std::move(phrase_tokens)); + out.function = RPNElement::FUNCTION_HAS_PHRASE; out.text_search_queries.emplace_back(std::move(query)); return true; diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.h b/src/Storages/MergeTree/MergeTreeIndexConditionText.h index 9e6a118b7508..b39347aaa51b 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.h @@ -43,17 +43,36 @@ enum class TextIndexDirectReadMode : uint8_t /// Represents a single text-search function struct TextSearchQuery { - TextSearchQuery(String function_name_, TextSearchMode search_mode_, TextIndexDirectReadMode direct_read_mode_, VectorWithMemoryTracking tokens_, std::vector patterns_ = {}); + TextSearchQuery( + String function_name_, + TextSearchMode search_mode_, + TextIndexDirectReadMode direct_read_mode_, + VectorWithMemoryTracking tokens_, + std::vector patterns_ = {}, + VectorWithMemoryTracking phrase_tokens_ = {}); + + const String & getFunctionName() const { return function_name; } + TextSearchMode getSearchMode() const { return search_mode; } + TextIndexDirectReadMode getDirectReadMode() const { return direct_read_mode; } + const VectorWithMemoryTracking & getTokens() const { return tokens; } + const std::vector & getPatterns() const { return patterns; } + const VectorWithMemoryTracking & getPhraseTokens() const { return phrase_tokens; } + UInt128 getHash() const { return hash; } +private: + void initializeHash(); + + /// Fields are immutable after construction, otherwise the precomputed hash becomes stale. String function_name; TextSearchMode search_mode; TextIndexDirectReadMode direct_read_mode; + /// Sorted in the constructor. VectorWithMemoryTracking tokens; std::vector patterns; - /// not sorted, not deduplicated + /// Not sorted, not deduplicated. VectorWithMemoryTracking phrase_tokens; - - SipHash getHash() const; + /// Precomputed in the constructor because getHash is called on hot paths. + UInt128 hash{}; }; using TextSearchQueryPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeIndexText.cpp b/src/Storages/MergeTree/MergeTreeIndexText.cpp index 1fce12141170..b1e745250099 100644 --- a/src/Storages/MergeTree/MergeTreeIndexText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexText.cpp @@ -724,7 +724,7 @@ size_t MergeTreeIndexGranuleText::memoryUsageBytes() const bool MergeTreeIndexGranuleText::hasAnyQueryTokens(const TextSearchQuery & query) const { - if (query.tokens.empty()) + if (query.getTokens().empty()) return false; return hasAnyTokensImpl(query); @@ -732,7 +732,7 @@ bool MergeTreeIndexGranuleText::hasAnyQueryTokens(const TextSearchQuery & query) bool MergeTreeIndexGranuleText::hasAnyQueryPatterns(const TextSearchQuery & query) const { - if (query.patterns.empty()) + if (query.getPatterns().empty()) return false; return hasAnyTokensImpl(query); @@ -747,7 +747,7 @@ bool MergeTreeIndexGranuleText::hasAnyTokensImpl(const TextSearchQuery & query) return false; /// Pattern bypass means analysis is incomplete, so conservatively return true. - if (query_builder.is_bypassed && !query.patterns.empty()) + if (query_builder.is_bypassed && !query.getPatterns().empty()) return true; if (!current_range.has_value()) @@ -774,7 +774,7 @@ bool MergeTreeIndexGranuleText::hasAnyTokensImpl(const TextSearchQuery & query) bool MergeTreeIndexGranuleText::hasAllQueryTokens(const TextSearchQuery & query) const { - if (query.tokens.empty()) + if (query.getTokens().empty()) return false; return hasAllQueryTokensOrEmpty(query); @@ -782,7 +782,7 @@ bool MergeTreeIndexGranuleText::hasAllQueryTokens(const TextSearchQuery & query) bool MergeTreeIndexGranuleText::hasAllQueryTokensOrEmpty(const TextSearchQuery & query) const { - if (query.tokens.empty()) + if (query.getTokens().empty()) return true; const auto & query_builder = analyzer->getQueryBuilder(query); @@ -792,7 +792,7 @@ bool MergeTreeIndexGranuleText::hasAllQueryTokensOrEmpty(const TextSearchQuery & return false; /// Pattern bypass means analysis is incomplete, so conservatively return true. - if (query_builder.is_bypassed && !query.patterns.empty()) + if (query_builder.is_bypassed && !query.getPatterns().empty()) return true; if (!current_range.has_value()) diff --git a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp index 79f971857f88..aabdc399a716 100644 --- a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp @@ -142,8 +142,8 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * [&](const auto & column) { const auto search_query = condition_text.getSearchQueryForVirtualColumn(column.name); - return search_query && search_query->search_mode == TextSearchMode::Phrase - && search_query->direct_read_mode == TextIndexDirectReadMode::Exact; + return search_query && search_query->getSearchMode() == TextSearchMode::Phrase + && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact; }); if (!has_fallback_candidates) @@ -172,8 +172,8 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * if (!search_query) continue; - bool needs_fallback = !search_query->patterns.empty() - || (search_query->search_mode == TextSearchMode::Phrase && search_query->direct_read_mode == TextIndexDirectReadMode::Exact); + bool needs_fallback = !search_query->getPatterns().empty() + || (search_query->getSearchMode() == TextSearchMode::Phrase && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact); if (!needs_fallback) continue; @@ -283,14 +283,14 @@ void MergeTreeReaderTextIndex::classifyVirtualColumns() auto search_query = condition_text.getSearchQueryForVirtualColumn(column.name); const auto & query_builder = analyzer.getQueryBuilder(*search_query); - if (search_query->tokens.empty() && search_query->patterns.empty()) + if (search_query->getTokens().empty() && search_query->getPatterns().empty()) { /// Token and phrase searches with no search tokens never match (row-level returns 0, e.g. when a /// postprocessor maps every needle token to empty). Encode this as an explicit no-match so direct /// read agrees with the row-scan path; otherwise an always-true virtual column would wrongly keep /// all rows once granule pruning cannot mask it (e.g. under OR). - if (search_query->function_name == "hasAnyTokens" || search_query->function_name == "hasAllTokens" - || search_query->search_mode == TextSearchMode::Phrase) + if (search_query->getFunctionName() == "hasAnyTokens" || search_query->getFunctionName() == "hasAllTokens" + || search_query->getSearchMode() == TextSearchMode::Phrase) continue; /// Always return true for empty needles. @@ -303,7 +303,7 @@ void MergeTreeReaderTextIndex::classifyVirtualColumns() } else if (query_builder.is_bypassed) { - if (search_query->direct_read_mode == TextIndexDirectReadMode::Hint) + if (search_query->getDirectReadMode() == TextIndexDirectReadMode::Hint) { is_always_true[i] = true; } @@ -319,8 +319,8 @@ void MergeTreeReaderTextIndex::classifyVirtualColumns() } } else if ( - search_query->search_mode == TextSearchMode::Phrase - && search_query->direct_read_mode == TextIndexDirectReadMode::Exact + search_query->getSearchMode() == TextSearchMode::Phrase + && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact && fallback_reader && fallback_expressions.contains(column.name)) { /// For phrase queries with positions, check selectivity before reading positional data. @@ -333,16 +333,16 @@ void MergeTreeReaderTextIndex::classifyVirtualColumns() /// Cardinalities (granule) and num_rows_in_part (part) share scale - a text index has whole-part granularity. const size_t num_rows_in_part = data_part_info_for_read->getRowCount(); - const bool all_tokens_present = ((num_rows_in_part > 0) && std::ranges::all_of(search_query->tokens, + const bool all_tokens_present = ((num_rows_in_part > 0) && std::ranges::all_of(search_query->getTokens(), [&](const auto & token) { return all_token_infos.find(token) != all_token_infos.end(); })); if (all_tokens_present) { double log_cardinality = 0.0; - for (const auto & token : search_query->tokens) + for (const auto & token : search_query->getTokens()) log_cardinality += std::log(static_cast(all_token_infos.find(token)->second->cardinality)); - log_cardinality -= static_cast(search_query->tokens.size() - 1) * std::log(static_cast(num_rows_in_part)); + log_cardinality -= static_cast(search_query->getTokens().size() - 1) * std::log(static_cast(num_rows_in_part)); if (std::exp(log_cardinality) > static_cast(num_rows_in_part) * selectivity_threshold) use_fallback[i] = true; } @@ -453,7 +453,7 @@ size_t MergeTreeReaderTextIndex::readRows( } size_t read_rows = 0; - createEmptyColumns(res_columns); + createEmptyColumns(res_columns, max_rows_to_read); size_t total_marks = data_part_info_for_read->getIndexGranularity().getMarksCountWithoutFinal(); if (!is_initialized && max_rows_to_read > 0) @@ -522,7 +522,7 @@ size_t MergeTreeReaderTextIndex::readRows( rows_to_read); } else if (auto search_query = condition_text.getSearchQueryForVirtualColumn(columns_to_read[i].name); - search_query && search_query->search_mode == TextSearchMode::Phrase) + search_query && search_query->getSearchMode() == TextSearchMode::Phrase) { /// Phrase queries are resolved from positional data (.pos), not per-mark posting lists. applyPostingsPhrase(column_mutable, search_query, from_row, rows_to_read); @@ -554,12 +554,16 @@ size_t MergeTreeReaderTextIndex::readRows( return read_rows; } -void MergeTreeReaderTextIndex::createEmptyColumns(Columns & columns) const +void MergeTreeReaderTextIndex::createEmptyColumns(Columns & columns, size_t max_rows_to_read) const { for (size_t i = 0; i < columns.size(); ++i) { if (columns[i] == nullptr) - columns[i] = columns_to_read[i].type->createColumn(*serializations[i]); + { + auto column = columns_to_read[i].type->createColumn(*serializations[i]); + column->reserve(max_rows_to_read); + columns[i] = std::move(column); + } } } @@ -610,12 +614,12 @@ std::vector MergeTreeReaderTextIndex::buildPostingsForMark(size_t m continue; auto search_query = condition_text.getSearchQueryForVirtualColumn(columns_to_read[i].name); - if (search_query->tokens.empty() && search_query->patterns.empty()) + if (search_query->getTokens().empty() && search_query->getPatterns().empty()) continue; /// Phrase queries are resolved from positional data (.pos) in applyPostingsPhrase, /// not from per-mark posting lists. - if (search_query->search_mode == TextSearchMode::Phrase) + if (search_query->getSearchMode() == TextSearchMode::Phrase) continue; result[i] = buildPostingsForQuery(*search_query, analyzer, *effective_range, range_posting); @@ -649,7 +653,7 @@ PostingList MergeTreeReaderTextIndex::buildPostingsForQuery( auto read_blocks = readPostingsBlocksForToken(token, *token_info, range); if (read_blocks.empty()) { - if (query.search_mode == TextSearchMode::All) + if (query.getSearchMode() == TextSearchMode::All) return {}; else continue; @@ -661,12 +665,12 @@ PostingList MergeTreeReaderTextIndex::buildPostingsForQuery( if (!result) result = std::move(large_postings); - else if (query.search_mode == TextSearchMode::All) + else if (query.getSearchMode() == TextSearchMode::All) *result &= large_postings; - else if (query.search_mode == TextSearchMode::Any) + else if (query.getSearchMode() == TextSearchMode::Any) *result |= large_postings; - if (query.search_mode == TextSearchMode::All && result && result->cardinality() == 0) + if (query.getSearchMode() == TextSearchMode::All && result && result->cardinality() == 0) return {}; } @@ -756,9 +760,9 @@ void MergeTreeReaderTextIndex::fillColumnLazy(IColumn & column, const String & c const auto & condition_text = assert_cast(*index.condition); auto search_query = condition_text.getSearchQueryForVirtualColumn(column_name); - chassert(search_query->patterns.empty()); + chassert(search_query->getPatterns().empty()); - if (search_query->tokens.empty()) + if (search_query->getTokens().empty()) { /// hasAnyTokens / hasAllTokens whose needle tokens were all dropped (e.g. by a postprocessor): no /// match, so fill zeros for every row read, matching fillColumn and the row-scan path. @@ -805,12 +809,12 @@ void MergeTreeReaderTextIndex::fillColumnLazy(IColumn & column, const String & c { cursors.push_back(it->second); } - else if (query_builder.postings->cardinality() > 0) + else if (!query_builder.postings->isEmpty()) { /// If there are no cursors for large postings, fill the column directly from the postings. if (cursors.empty()) { - if (range_posting.cardinality() == 0) + if (range_posting.isEmpty()) { requireRowOffsetRepresentable(row_offset); auto range_end = static_cast(std::min(row_offset + num_rows - 1, std::numeric_limits::max())); @@ -843,12 +847,12 @@ void MergeTreeReaderTextIndex::fillColumnLazy(IColumn & column, const String & c if (cursors.empty()) return; - if (search_query->search_mode == TextSearchMode::Any) + if (search_query->getSearchMode() == TextSearchMode::Any) lazyUnionPostingLists(column, cursors, old_size, row_offset, num_rows); - else if (search_query->search_mode == TextSearchMode::All) + else if (search_query->getSearchMode() == TextSearchMode::All) lazyIntersectPostingLists(column, cursors, old_size, row_offset, num_rows, lazy_density_threshold); else - throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid search mode: {}", search_query->search_mode); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid search mode: {}", search_query->getSearchMode()); } void MergeTreeReaderTextIndex::applyPostingsPhrase( @@ -861,10 +865,10 @@ void MergeTreeReaderTextIndex::applyPostingsPhrase( size_t column_offset = column_data.size(); column_data.resize_fill(column_offset + num_rows, 0); - if (!positions_stream || search_query->phrase_tokens.empty()) + if (!positions_stream || search_query->getPhraseTokens().empty()) return; - auto cache_key = search_query->getHash().get128(); + auto cache_key = search_query->getHash(); auto doc_ids_it = phrase_search_doc_ids.find(cache_key); if (doc_ids_it == phrase_search_doc_ids.end()) @@ -879,8 +883,8 @@ void MergeTreeReaderTextIndex::applyPostingsPhrase( const auto & all_token_infos = granule->getAnalyzer().getAllTokenInfos(); std::vector position_offsets; - position_offsets.reserve(search_query->phrase_tokens.size()); - for (const auto & token : search_query->phrase_tokens) + position_offsets.reserve(search_query->getPhraseTokens().size()); + for (const auto & token : search_query->getPhraseTokens()) { auto it = all_token_infos.find(token); if (it == all_token_infos.end() || !(it->second->header & PostingsSerialization::Flags::HasPositions)) diff --git a/src/Storages/MergeTree/MergeTreeReaderTextIndex.h b/src/Storages/MergeTree/MergeTreeReaderTextIndex.h index c53933f31d79..e18b006d868f 100644 --- a/src/Storages/MergeTree/MergeTreeReaderTextIndex.h +++ b/src/Storages/MergeTree/MergeTreeReaderTextIndex.h @@ -55,7 +55,7 @@ class MergeTreeReaderTextIndex : public IMergeTreeReader private: void setIndexGranule(MergeTreeIndexGranulePtr index_granule); void initializeFallbackReader(const IMergeTreeReader * main_reader); - void createEmptyColumns(Columns & columns) const; + void createEmptyColumns(Columns & columns, size_t max_rows_to_read) const; std::unique_ptr makeTextIndexStream(const MergeTreeIndexSubstream & substream) const; /// Returns combined postings per column for the given mark, clipped to `slice_range` diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.cpp b/src/Storages/MergeTree/TextIndexAnalyzer.cpp index 8a0726fb6cd3..bd230cec5574 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.cpp +++ b/src/Storages/MergeTree/TextIndexAnalyzer.cpp @@ -33,7 +33,7 @@ void TextIndexAnalyzer::QueryBuilder::markBypassed() void TextIndexAnalyzer::QueryBuilder::addMissingToken() { - if (query->search_mode == TextSearchMode::All || query->search_mode == TextSearchMode::Phrase) + if (query->getSearchMode() == TextSearchMode::All || query->getSearchMode() == TextSearchMode::Phrase) markFailed(); } @@ -64,11 +64,11 @@ void TextIndexAnalyzer::QueryBuilder::addRowsRange(RowsRange token_rows_range) { rows_range = token_rows_range; } - else if (query->search_mode == TextSearchMode::Any) + else if (query->getSearchMode() == TextSearchMode::Any) { rows_range = rows_range->unionWith(token_rows_range); } - else if (query->search_mode == TextSearchMode::All || query->search_mode == TextSearchMode::Phrase) + else if (query->getSearchMode() == TextSearchMode::All || query->getSearchMode() == TextSearchMode::Phrase) { rows_range = rows_range->intersectWith(token_rows_range); @@ -88,7 +88,7 @@ void TextIndexAnalyzer::QueryBuilder::addPostings(PostingListPtr token_postings) { postings = *token_postings; } - else if (query->search_mode == TextSearchMode::Any) + else if (query->getSearchMode() == TextSearchMode::Any) { *postings |= *token_postings; } @@ -109,21 +109,21 @@ TextIndexAnalyzer::TextIndexAnalyzer(const MergeTreeIndexConditionText & conditi { query_builders[hash].query = query; - for (const auto & token : query->tokens) + for (const auto & token : query->getTokens()) queries_by_token[token].insert(hash); - for (const auto & pattern : query->patterns) + for (const auto & pattern : query->getPatterns()) queries_by_pattern[&pattern].insert(hash); } } const TextIndexAnalyzer::QueryBuilder & TextIndexAnalyzer::getQueryBuilder(const TextSearchQuery & query) const { - auto hash = query.getHash().get128(); + auto hash = query.getHash(); auto it = query_builders.find(hash); if (it == query_builders.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Query builder not found for text search query with function '{}'", query.function_name); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query builder not found for text search query with function '{}'", query.getFunctionName()); return it->second; } @@ -211,10 +211,10 @@ void TextIndexAnalyzer::bypassPatternQueries() double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_builder, size_t total_rows) const { const auto & query = *query_builder.query; - chassert(!query.tokens.empty()); + chassert(!query.getTokens().empty()); const double n = static_cast(total_rows); - switch (query.search_mode) + switch (query.getSearchMode()) { case TextSearchMode::All: /// A phrase requires all its tokens to be present. @@ -228,7 +228,7 @@ double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_bu : std::log(n); size_t num_unread = 0; - for (const auto & token : query.tokens) + for (const auto & token : query.getTokens()) { auto it = query_builder.tokens.find(token); if (it == query_builder.tokens.end()) @@ -251,7 +251,7 @@ double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_bu ? 1.0 - static_cast(query_builder.postings->cardinality()) / n : 1.0; - for (const auto & token : query.tokens) + for (const auto & token : query.getTokens()) { auto it = query_builder.tokens.find(token); if (it != query_builder.tokens.end() && hasReadPostings(token)) @@ -285,13 +285,13 @@ void TextIndexAnalyzer::analyzeCardinalitiesAndBypassHints(double selectivity_th continue; const auto & query = *query_builder.query; - if (query.direct_read_mode != TextIndexDirectReadMode::Hint) + if (query.getDirectReadMode() != TextIndexDirectReadMode::Hint) continue; /// Pure-pattern queries have no declared tokens at parse time; their tokens are /// discovered dynamically during dictionary scan. Skip the cardinality check in /// that case — it would have no inputs to work with. - if (query.tokens.empty()) + if (query.getTokens().empty()) continue; double estimated_cardinality = estimateQueryCardinality(query_builder, total_rows); @@ -307,8 +307,8 @@ void TextIndexAnalyzer::analyzeCardinalitiesAndBypassHints(double selectivity_th query_builder.markBypassed(); ProfileEvents::increment(ProfileEvents::TextIndexDiscardHint); - auto hash = query.getHash().get128(); - for (const auto & query_token : query.tokens) + auto hash = query.getHash(); + for (const auto & query_token : query.getTokens()) queries_by_token[query_token].erase(hash); for (const auto & [query_token, _] : query_builder.tokens) @@ -339,10 +339,10 @@ void TextIndexAnalyzer::processTokenOperation(std::string_view token, Operation always_false = true; /// Erase the failed query for the full declared token set so yet-unseen tokens stop passing isTokenNeeded. - for (const auto & query_token : query_builder.query->tokens) + for (const auto & query_token : query_builder.query->getTokens()) queries_by_token[query_token].erase(query_hash); - /// Also erase for already-discovered dynamic pattern tokens (not in query->tokens). + /// Also erase for already-discovered dynamic pattern tokens (not in `query->getTokens`). for (const auto & [query_token, _] : query_builder.tokens) queries_by_token[query_token].erase(query_hash); } diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.h b/src/Storages/MergeTree/TextIndexAnalyzer.h index b3736cf6cf2a..d96416a42e12 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.h +++ b/src/Storages/MergeTree/TextIndexAnalyzer.h @@ -20,9 +20,9 @@ class TextIndexAnalyzer TextSearchQueryPtr query; /// Tokens this query has observed so far (declared + pattern-discovered). TokenToPostingsInfosMap tokens; - /// Row range folded across observed tokens by `query->search_mode` (intersect for `All`, union for `Any`). + /// Row range folded across observed tokens by the query search mode (intersect for `All`, union for `Any`). std::optional rows_range; - /// Posting list folded across materialized tokens by `query->search_mode`. + /// Posting list folded across materialized tokens by the query search mode. std::optional postings; /// Query can never match (e.g. missing token in `All` mode, empty intersection). diff --git a/tests/queries/0_stateless/02346_text_index_bug90778.reference b/tests/queries/0_stateless/02346_text_index_bug90778.reference index d7e164a197ff..be66339bd1b8 100644 --- a/tests/queries/0_stateless/02346_text_index_bug90778.reference +++ b/tests/queries/0_stateless/02346_text_index_bug90778.reference @@ -1,4 +1,4 @@ 1 -Actions: INPUT : 0 -> __text_index_idx_equals_a8795ef08bf8680d4058b0b633446b5d UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_equals_55ee63b12fbf5cb6a7510c6b2df99282 UInt8 : 0 1 -Actions: INPUT : 0 -> __text_index_idx_hasToken_64e1720f37357ea1ffbb211db4a54a6c UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasToken_0884a8aaaf5dac89f2f31ccbb118ab20 UInt8 : 0 diff --git a/tests/queries/0_stateless/02346_text_index_direct_read.reference b/tests/queries/0_stateless/02346_text_index_direct_read.reference index f3fd8ebe6795..76b83e4eff9f 100644 --- a/tests/queries/0_stateless/02346_text_index_direct_read.reference +++ b/tests/queries/0_stateless/02346_text_index_direct_read.reference @@ -7,22 +7,22 @@ Test select text + hasAnyTokens: Alick a01 Test hasToken and hasToken: 0 Test hasAnyTokens or hasToken: 2 Test NOT hasAllTokens: 1 -Added: [__text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97], Removed: [text] -Added: [__text_index_idx_hasAllTokens_3064868a108351f77214bdde8fe5fde4], Removed: [text] -Added: [__text_index_idx_hasAnyTokens_f9382267d4ac05648238a1fcb88f480c], Removed: [text] -Added: [__text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97] -Added: [__text_index_idx_hasAnyTokens_f9382267d4ac05648238a1fcb88f480c] -Added: [__text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97, __text_index_idx_hasToken_e333eae7ebcce5db7ff3bfc355ea5644], Removed: [text] -Added: [__text_index_idx_hasAnyTokens_08ba86dd30b46b1972011bb54e74e868, __text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97], Removed: [text] -Added: [__text_index_idx_hasAllTokens_85143c8dacb6c12b5cdffb4d22ebb367], Removed: [text] +Added: [__text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a], Removed: [text] +Added: [__text_index_idx_hasAllTokens_5fa51b2c623f1a486d9249d7757ec915], Removed: [text] +Added: [__text_index_idx_hasAnyTokens_1f4e7655ba9b30e7a282c4ab86df4899], Removed: [text] +Added: [__text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a] +Added: [__text_index_idx_hasAnyTokens_1f4e7655ba9b30e7a282c4ab86df4899] +Added: [__text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a, __text_index_idx_hasToken_d0c9a08c3b0871a03e9eec9b8f2d9b10], Removed: [text] +Added: [__text_index_idx_hasAnyTokens_0c8fc52ed37da16d57305d1bb45c0abb, __text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a], Removed: [text] +Added: [__text_index_idx_hasAllTokens_062af83dff7b50ae332d4792a157bd19], Removed: [text] - Test direct read optimization with EXPLAIN -Actions: INPUT : 0 -> __text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97 UInt8 : 0 -Actions: INPUT : 0 -> __text_index_idx_hasAllTokens_3064868a108351f77214bdde8fe5fde4 UInt8 : 0 -Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_f9382267d4ac05648238a1fcb88f480c UInt8 : 0 -INPUT : 1 -> __text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97 UInt8 : 2 -Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_f9382267d4ac05648238a1fcb88f480c UInt8 : 0 -Actions: INPUT : 0 -> __text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97 UInt8 : 0 -INPUT : 1 -> __text_index_idx_hasToken_e333eae7ebcce5db7ff3bfc355ea5644 UInt8 : 1 -Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_08ba86dd30b46b1972011bb54e74e868 UInt8 : 0 -INPUT : 1 -> __text_index_idx_hasToken_05af67caeb1dac7fe8822d89615c3a97 UInt8 : 1 -Actions: INPUT : 0 -> __text_index_idx_hasAllTokens_85143c8dacb6c12b5cdffb4d22ebb367 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasAllTokens_5fa51b2c623f1a486d9249d7757ec915 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_1f4e7655ba9b30e7a282c4ab86df4899 UInt8 : 0 +INPUT : 1 -> __text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a UInt8 : 2 +Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_1f4e7655ba9b30e7a282c4ab86df4899 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a UInt8 : 0 +INPUT : 1 -> __text_index_idx_hasToken_d0c9a08c3b0871a03e9eec9b8f2d9b10 UInt8 : 1 +Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_0c8fc52ed37da16d57305d1bb45c0abb UInt8 : 0 +INPUT : 1 -> __text_index_idx_hasToken_6162cbd2939ffaca63bd4e3b1c784a2a UInt8 : 1 +Actions: INPUT : 0 -> __text_index_idx_hasAllTokens_062af83dff7b50ae332d4792a157bd19 UInt8 : 0 diff --git a/tests/queries/0_stateless/02346_text_index_duplicate_tokens.reference b/tests/queries/0_stateless/02346_text_index_duplicate_tokens.reference index 5b718b91924a..76c7eab5d48d 100644 --- a/tests/queries/0_stateless/02346_text_index_duplicate_tokens.reference +++ b/tests/queries/0_stateless/02346_text_index_duplicate_tokens.reference @@ -1,5 +1,5 @@ 1 -INPUT : 1 -> __text_index_idx_s_like_c0b70db11b78502b054fdfa69d0792fe UInt8 : 3 +INPUT : 1 -> __text_index_idx_s_like_814f3833ee52630b8efd657170e55dfc UInt8 : 3 1 1 -INPUT : 1 -> __text_index_idx_s_startsWith_3eedf4e7343bca6ef5e0be755776a4a4 UInt8 : 3 +INPUT : 1 -> __text_index_idx_s_startsWith_2f100bf43b67b357066cc202fa1d8895 UInt8 : 3 diff --git a/tests/queries/0_stateless/02346_text_index_hint.reference b/tests/queries/0_stateless/02346_text_index_hint.reference index 7621610b4ede..71ff9186ae42 100644 --- a/tests/queries/0_stateless/02346_text_index_hint.reference +++ b/tests/queries/0_stateless/02346_text_index_hint.reference @@ -1,13 +1,13 @@ -- splitByNonAlpha 1 -INPUT : 1 -> __text_index_idx_equals_bd435c844cedc4f9bd02f62fd38c35ef UInt8 : 2 +INPUT : 1 -> __text_index_idx_equals_1c7afcced8817e9e6a05c70cbf682510 UInt8 : 2 19 -- array 1 -Actions: INPUT : 0 -> __text_index_idx_equals_30d21ca41ace507c1ea8201ea2f25a15 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_equals_cd9afffe64cb7813239c748155997a5c UInt8 : 0 19 -- ngrams(3) 1 -INPUT : 1 -> __text_index_idx_equals_6315d1ed5df88795d0beba701d2b3f7a UInt8 : 2 +INPUT : 1 -> __text_index_idx_equals_13f86b01f2b39398e82a6529931945c7 UInt8 : 2 19 -INPUT : 1 -> __text_index_idx_like_e4e4354e8a0ca24be44af5f1cbf9914b UInt8 : 2 +INPUT : 1 -> __text_index_idx_like_15aa1ca239fe93e2c84859d019458984 UInt8 : 2 diff --git a/tests/queries/0_stateless/02346_text_index_hint_map.reference b/tests/queries/0_stateless/02346_text_index_hint_map.reference index 0d6e3aabe8b3..5db80e63ad82 100644 --- a/tests/queries/0_stateless/02346_text_index_hint_map.reference +++ b/tests/queries/0_stateless/02346_text_index_hint_map.reference @@ -1,50 +1,50 @@ SELECT count() FROM tab WHERE has(mapKeys(m), 'k18') 36663 -Actions: INPUT : 0 -> __text_index_idx_mk_has_e93f2d2d6e394ffcfa6e8e8309cf2b01 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_mk_has_3b9d3bdf42a83839ec93507f0ae93868 UInt8 : 0 Name: idx_mk SELECT count() FROM tab WHERE has(m, 'k18') 36663 -Actions: INPUT : 0 -> __text_index_idx_mk_has_e93f2d2d6e394ffcfa6e8e8309cf2b01 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_mk_has_3b9d3bdf42a83839ec93507f0ae93868 UInt8 : 0 Name: idx_mk SELECT count() FROM tab WHERE mapContains(m, 'k18') 36663 -Actions: INPUT : 0 -> __text_index_idx_mk_mapContainsKey_d0e94dda9bea463c5120234c37b5800a UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_mk_mapContainsKey_d0f3d56af0d91d84a6b291769067b08c UInt8 : 0 Name: idx_mk SELECT count() FROM tab WHERE mapContainsKey(m, 'k18') 36663 -Actions: INPUT : 0 -> __text_index_idx_mk_mapContainsKey_d0e94dda9bea463c5120234c37b5800a UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_mk_mapContainsKey_d0f3d56af0d91d84a6b291769067b08c UInt8 : 0 Name: idx_mk SELECT count() FROM tab WHERE mapContainsValue(m, 'v18') 36663 -Actions: INPUT : 0 -> __text_index_idx_mv_mapContainsValue_555121f90bedd7ca05f9ab33fc33b0a7 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_mv_mapContainsValue_1c45423c1f61a85319da84b5bafdd459 UInt8 : 0 Name: idx_mv SELECT count() FROM tab WHERE m['k18'] = 'v18' 36663 -INPUT : 1 -> __text_index_idx_mk_mapContainsKey_f5275bc830debcb9f2b25eeec30a5191 UInt8 : 3 -INPUT : 2 -> __text_index_idx_mv_equals_15aaa95d4862fe797f6e421459a7e7f0 UInt8 : 4 +INPUT : 1 -> __text_index_idx_mk_mapContainsKey_cc36bddb1d58aff0475c735d390e0f67 UInt8 : 3 +INPUT : 2 -> __text_index_idx_mv_equals_8a6b117058a019f23a2a860c72a6aa22 UInt8 : 4 Name: idx_mk Name: idx_mv SELECT count() FROM tab WHERE m['k18'] LIKE '%v18%' 36663 -INPUT : 1 -> __text_index_idx_mk_mapContainsKey_f5275bc830debcb9f2b25eeec30a5191 UInt8 : 3 +INPUT : 1 -> __text_index_idx_mk_mapContainsKey_cc36bddb1d58aff0475c735d390e0f67 UInt8 : 3 Name: idx_mk SELECT count() FROM tab WHERE notEmpty(m['k18']) 36663 -INPUT : 1 -> __text_index_idx_mk_mapContainsKey_f5275bc830debcb9f2b25eeec30a5191 UInt8 : 2 +INPUT : 1 -> __text_index_idx_mk_mapContainsKey_cc36bddb1d58aff0475c735d390e0f67 UInt8 : 2 Name: idx_mk SELECT count() FROM tab WHERE empty(m['k18']) 63337 SELECT count() FROM tab WHERE toUInt64OrZero(extract(m['k18'], '[0-9]+')) = 18 36663 -INPUT : 1 -> __text_index_idx_mk_mapContainsKey_f5275bc830debcb9f2b25eeec30a5191 UInt8 : 4 +INPUT : 1 -> __text_index_idx_mk_mapContainsKey_cc36bddb1d58aff0475c735d390e0f67 UInt8 : 4 Name: idx_mk SELECT count() FROM tab WHERE toUInt64OrZero(extract(m['k18'], '[0-9]+')) = 0 63337 SELECT count() FROM tab WHERE mapContainsKeyLike(m, '%k18%') 36663 -INPUT : 1 -> __text_index_idx_mk_mapContainsKeyLike_416cd780bc3d7d1edb6650dcb0c74512 UInt8 : 2 +INPUT : 1 -> __text_index_idx_mk_mapContainsKeyLike_030c33a9d9b7ca155db85bd1c159a428 UInt8 : 2 Name: idx_mk SELECT count() FROM tab WHERE mapContainsValueLike(m, '%v18%') 36663 -INPUT : 1 -> __text_index_idx_mv_mapContainsValueLike_4e182887b279f7224d6b1a307743e5bc UInt8 : 2 +INPUT : 1 -> __text_index_idx_mv_mapContainsValueLike_7c701f001386dc46d39b2c90502a0d50 UInt8 : 2 Name: idx_mv diff --git a/tests/queries/0_stateless/02346_text_index_materialization.reference b/tests/queries/0_stateless/02346_text_index_materialization.reference index 1d1e4481fb5d..ffeb8ec5d7d1 100644 --- a/tests/queries/0_stateless/02346_text_index_materialization.reference +++ b/tests/queries/0_stateless/02346_text_index_materialization.reference @@ -2,14 +2,14 @@ Before OPTIMIZE FINAL 0 1 111 -Prewhere filter column: and(__text_index_idx_text_like_2f9b1745b930d9f8b662f2fe9c02d2d9, like(__table1.text, \'%v322%\'_String)) (removed) +Prewhere filter column: and(__text_index_idx_text_like_a9e15b748aea95bcebed600944a256c8, like(__table1.text, \'%v322%\'_String)) (removed) Granules: 196/196 Granules: 98/196 1 After OPTIMIZE FINAL 1 111 -Prewhere filter column: and(__text_index_idx_text_like_2f9b1745b930d9f8b662f2fe9c02d2d9, like(__table1.text, \'%v322%\'_String)) (removed) +Prewhere filter column: and(__text_index_idx_text_like_a9e15b748aea95bcebed600944a256c8, like(__table1.text, \'%v322%\'_String)) (removed) Granules: 196/196 Granules: 4/196 1 @@ -23,7 +23,7 @@ Granules: 196/196 After MATERIALIZE INDEX idx_text 1 111 -Prewhere filter column: and(__text_index_idx_text_like_2f9b1745b930d9f8b662f2fe9c02d2d9, like(__table1.text, \'%v322%\'_String)) (removed) +Prewhere filter column: and(__text_index_idx_text_like_a9e15b748aea95bcebed600944a256c8, like(__table1.text, \'%v322%\'_String)) (removed) Granules: 196/196 Granules: 4/196 1 diff --git a/tests/queries/0_stateless/02346_text_index_on_lower_column.reference b/tests/queries/0_stateless/02346_text_index_on_lower_column.reference index 28f242052785..104612d67010 100644 --- a/tests/queries/0_stateless/02346_text_index_on_lower_column.reference +++ b/tests/queries/0_stateless/02346_text_index_on_lower_column.reference @@ -1,14 +1,14 @@ 1 -Actions: INPUT : 0 -> __text_index_idx_text_hasToken_425d21a9e64c3b652fe967bc06aca4fc UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_text_hasToken_8c0986e6342faaa0d69505e07e03f371 UInt8 : 0 Name: idx_text 1 1 -Actions: INPUT : 0 -> __text_index_idx_text_hasAllTokens_7a7a737e1ade44463de66430e2c21507 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_text_hasAllTokens_9452de3cdb1b5a4013c7bfce6bcc52d3 UInt8 : 0 Name: idx_text 1 1 -Actions: INPUT : 0 -> __text_index_idx_text_hasToken_b62d561255328e2b7a6fbfbceec36f9b UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_text_hasToken_0542cddf60add5dacb445029df3e714a UInt8 : 0 Name: idx_text 1 -Actions: INPUT : 0 -> __text_index_idx_text_hasAllTokens_458a6f0c95f141515db966345d9a0f2e UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_text_hasAllTokens_948c8f1dbcee91a860e600f098427077 UInt8 : 0 Name: idx_text diff --git a/tests/queries/0_stateless/02346_text_index_prewhere_support.reference b/tests/queries/0_stateless/02346_text_index_prewhere_support.reference index e19a0b6ccaa4..bc756553b6bd 100644 --- a/tests/queries/0_stateless/02346_text_index_prewhere_support.reference +++ b/tests/queries/0_stateless/02346_text_index_prewhere_support.reference @@ -1,166 +1,166 @@ SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') 9 9 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 0 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAnyTokens(text1, ['clickhouse', 'database']) 12 12 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_12c047f4b3dd26289deedcb3914ac50f UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_3e549fbe3daa4f675999fd0d0d3d00e7 UInt8 : 0 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAllTokens(text1, ['column', 'store']) 4 4 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 0 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text2, 'fastio') 3 3 -Actions: INPUT : 0 -> __text_index_inv_idx2_hasToken_bd0a73d0c30edda02a64e2343897fa8a UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx2_hasToken_b28756beafa5218c9cdb6cae90f914df UInt8 : 0 Name: inv_idx2 SELECT count() FROM tab PREWHERE hasAnyTokens(text2, ['search', 'index']) 8 8 -Actions: INPUT : 0 -> __text_index_inv_idx2_hasAnyTokens_30ca92657d5a26f7d718ae98d687d947 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx2_hasAnyTokens_eec5bb834b8a2231794521db5a61de15 UInt8 : 0 Name: inv_idx2 SELECT count() FROM tab PREWHERE hasAllTokens(text2, ['merge', 'tree']) 2 2 -Actions: INPUT : 0 -> __text_index_inv_idx2_hasAllTokens_518cab706843eae694663ae410afb96a UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx2_hasAllTokens_4443311e45452959f342e0a9e321eca4 UInt8 : 0 Name: inv_idx2 SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') AND a > 0 9 9 -INPUT : 1 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 2 +INPUT : 1 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 2 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAnyTokens(text1, ['clickhouse', 'database']) AND id BETWEEN 10 AND 100 9 9 -INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_12c047f4b3dd26289deedcb3914ac50f UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_3e549fbe3daa4f675999fd0d0d3d00e7 UInt8 : 3 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAllTokens(text1, ['column', 'store']) AND (a % 2) = 0 4 4 -INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 3 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text1, 'analytics') OR a < 0 4 4 -INPUT : 1 -> __text_index_inv_idx1_hasToken_204a3f4264c08cddce6e739e7cc2ac5e UInt8 : 2 +INPUT : 1 -> __text_index_inv_idx1_hasToken_1935c344dfaff6eef309b89b9aa3eabe UInt8 : 2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (hasAnyTokens(text1, ['log', 'event']) OR hasToken(text2, 'error')) AND id > 0 5 5 -INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_1b6ddaa1516ebfd4d0bbd70a7fbf0955 UInt8 : 2 -INPUT : 2 -> __text_index_inv_idx2_hasToken_80aec043c4d3cd5c1fd62ec9f28fc548 UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_f43789623cf9632b91e90938bc248480 UInt8 : 2 +INPUT : 2 -> __text_index_inv_idx2_hasToken_aed188f6bb6d4e75a7294a53c0e8aae9 UInt8 : 3 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (a > 100 AND hasToken(text1, 'hot')) OR (a <= 100 AND hasAnyTokens(text2, ['cold', 'warm'])) 0 0 -INPUT : 1 -> __text_index_inv_idx1_hasToken_fa43ee0e78013d1beccc5ea73a314542 UInt8 : 2 -INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_f72df8660da2ff9863231cc88793beec UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasToken_db7c050d4f539215f55ef221b172161b UInt8 : 2 +INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_003957539a363239798bb5f2c1752e59 UInt8 : 3 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') AND hasAnyTokens(text1, ['clickhouse', 'database']) 9 9 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_12c047f4b3dd26289deedcb3914ac50f UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_3e549fbe3daa4f675999fd0d0d3d00e7 UInt8 : 1 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAnyTokens(text1, ['clickhouse', 'database']) AND NOT hasToken(text1, 'mysql') 12 12 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_12c047f4b3dd26289deedcb3914ac50f UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx1_hasToken_3684479f77734e8e477ada41a51ca47a UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_3e549fbe3daa4f675999fd0d0d3d00e7 UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx1_hasToken_586878f8cf4c2e8e9c86bb6060300664 UInt8 : 1 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAllTokens(text1, ['column', 'store']) OR hasAnyTokens(text1, ['olap', 'analytics']) 8 8 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_778991886a3005f3baf85723d2df46a2 UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_52db9f289bbe63edc12ed62144d5d077 UInt8 : 1 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') AND hasToken(text2, 'fastio') 3 3 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx2_hasToken_bd0a73d0c30edda02a64e2343897fa8a UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx2_hasToken_b28756beafa5218c9cdb6cae90f914df UInt8 : 1 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAnyTokens(text1, ['search', 'query']) OR hasAllTokens(text2, ['index', 'optimize']) 14 14 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_d545aaf68c1f9146a8180f9e86a0335c UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx2_hasAllTokens_680ff84c43352c06274f4f9525af9933 UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAnyTokens_6dbc958a3b8b19eb0a4ad5c57a1ae8b4 UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx2_hasAllTokens_8452ebd9d927f0161c158a972850ea03 UInt8 : 1 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (hasToken(text1, 'user') AND hasAnyTokens(text2, ['login', 'logout'])) OR (a >= 0 AND a <= 10) 12 12 -INPUT : 1 -> __text_index_inv_idx1_hasToken_85f3d1efe3123ac9bb86f9aae3dd003d UInt8 : 3 -INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_de94e14627f7cac2310406565b3f6b14 UInt8 : 4 +INPUT : 1 -> __text_index_inv_idx1_hasToken_e06bd6ab174efc6d7ae7cf7f192c1270 UInt8 : 3 +INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_3db9cb869e2d940b49eeface4f178f01 UInt8 : 4 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE NOT hasToken(text1, 'debug') AND hasAnyTokens(text2, ['info', 'warn', 'error']) 2 2 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_0e7394b6b4305035b4f2769a86ce79cf UInt8 : 0 -INPUT : 1 -> __text_index_inv_idx2_hasAnyTokens_2a7d7afe87cda6ccdfe6fb9190379413 UInt8 : 1 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_a0f03023317a7e61eaf2cd192f917a6f UInt8 : 0 +INPUT : 1 -> __text_index_inv_idx2_hasAnyTokens_a6d0e6888f753467c452b61983552d68 UInt8 : 1 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE NOT hasAllTokens(text1, ['internal', 'test']) AND a > 0 AND hasToken(text2, 'prod') 2 2 -INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_1e46f6f1fffc22edbb5eb5c456409089 UInt8 : 2 -INPUT : 2 -> __text_index_inv_idx2_hasToken_fa1607a88660ff1938d7de6128c971f8 UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_328c7d5df8c1e6bec8ceca3b99f6e192 UInt8 : 2 +INPUT : 2 -> __text_index_inv_idx2_hasToken_a0cdab879033370a66aac627e06a5786 UInt8 : 3 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE NOT (hasAnyTokens(text1, ['drop', 'truncate']) OR hasToken(text2, 'danger')) AND id IN (1, 2, 3, 4) 4 4 -INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_152811574e204e114891a456fa3549eb UInt8 : 2 -INPUT : 2 -> __text_index_inv_idx2_hasToken_c9480586a627a4e19f67999f849c5bbf UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasAnyTokens_cba40c56c3ce3fb9f91fe0263a91f211 UInt8 : 2 +INPUT : 2 -> __text_index_inv_idx2_hasToken_f966111c12115482b475d109b67023ef UInt8 : 3 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (a > 10 AND hasToken(text1, 'clickhouse')) OR (a <= 10 AND hasAnyTokens(text2, ['mysql', 'postgres'])) 8 8 -INPUT : 1 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 2 -INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_28c81820cbc8fc809ac01ed7a94b9f27 UInt8 : 3 +INPUT : 1 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 2 +INPUT : 2 -> __text_index_inv_idx2_hasAnyTokens_6c2096cd7194c83c90112848e488e5ba UInt8 : 3 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (hasAllTokens(text1, ['column', 'store']) AND a BETWEEN 1 AND 100) OR (hasAnyTokens(text2, ['lake', 'warehouse']) AND id > 1000) 2 2 -INPUT : 2 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 5 -INPUT : 3 -> __text_index_inv_idx2_hasAnyTokens_4ab65505a9827c6f159c723110afe4ab UInt8 : 6 +INPUT : 2 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 5 +INPUT : 3 -> __text_index_inv_idx2_hasAnyTokens_5021d33a8f75c33c7c1526d35692308b UInt8 : 6 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE (NOT hasToken(text1, 'archived') AND a > 0) OR (hasAnyTokens(text2, ['active', 'online']) AND id % 5 = 0) 128 128 -INPUT : 2 -> __text_index_inv_idx1_hasToken_eb33dfc4c4a7b6c38aa8ac853b4c155f UInt8 : 4 -INPUT : 3 -> __text_index_inv_idx2_hasAnyTokens_3048c776cfda056f6a2e9f7c7b18e6c4 UInt8 : 5 +INPUT : 2 -> __text_index_inv_idx1_hasToken_180d432dc3e2d9afdab90f73d4cab495 UInt8 : 4 +INPUT : 3 -> __text_index_inv_idx2_hasAnyTokens_50452afe6f96bf94d129371db673fa7b UInt8 : 5 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') WHERE a > 0 9 9 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 0 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAllTokens(text1, ['column', 'store']) WHERE a NOT IN (150, 160) AND id != 15 4 4 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 0 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasToken(text1, 'clickhouse') WHERE hasToken(text2, 'fastio') 3 3 -Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_41322c936a4537f9edab965eabbbc82b UInt8 : 0 +Actions: INPUT : 0 -> __text_index_inv_idx1_hasToken_c385787e31a273521d626dbfc1b8df99 UInt8 : 0 Name: inv_idx2 Name: inv_idx1 SELECT count() FROM tab PREWHERE hasAllTokens(text1, ['column', 'store']) WHERE hasAnyTokens(text1, ['olap', 'analytics']) 1 1 -INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_67ce908c110da22e8dfabdc5b2461e7e UInt8 : 1 +INPUT : 1 -> __text_index_inv_idx1_hasAllTokens_0da059602ca3ea50bb0df5c5afe742b6 UInt8 : 1 Name: inv_idx1 diff --git a/tests/queries/0_stateless/02346_text_index_tokenizer_partially_materialized.reference b/tests/queries/0_stateless/02346_text_index_tokenizer_partially_materialized.reference index 876d9363f10e..63eca266da6d 100644 --- a/tests/queries/0_stateless/02346_text_index_tokenizer_partially_materialized.reference +++ b/tests/queries/0_stateless/02346_text_index_tokenizer_partially_materialized.reference @@ -1,5 +1,5 @@ Fully materialized -Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_aec84632c5c510ddd2ec7f581b35e985 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_54e32b9689dc9670d100ead91d552336 UInt8 : 0 20000 20000 20000 @@ -9,7 +9,7 @@ Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_aec84632c5c510ddd2ec7f581b35 20000 20000 Partially materialized -Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_aec84632c5c510ddd2ec7f581b35e985 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_hasAnyTokens_54e32b9689dc9670d100ead91d552336 UInt8 : 0 20000 20000 20000 diff --git a/tests/queries/0_stateless/04093_text_index_separate_analysis.reference b/tests/queries/0_stateless/04093_text_index_separate_analysis.reference index b99f1ca46fc2..0cc11a28d54b 100644 --- a/tests/queries/0_stateless/04093_text_index_separate_analysis.reference +++ b/tests/queries/0_stateless/04093_text_index_separate_analysis.reference @@ -4,7 +4,7 @@ 0 1 --- Part 2: EXPLAIN indexes shows granule filtering (use_skip_indexes_on_data_read = 0) -Actions: INPUT : 0 -> __text_index_idx_message_hasAllTokens_0f7345644adb0992857049f7ce3d0420 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_idx_message_hasAllTokens_b79ad6ad12798e01a370b4598f91b250 UInt8 : 0 Parts: 1 Granules: 1 Parts: 1/1 @@ -16,8 +16,8 @@ Granules: 1/5 --- Part 3: Both indexes filter (use_skip_indexes_on_data_read = 0) Parts: 1 Granules: 1 -Actions: INPUT : 0 -> __text_index_idx_category_hasAllTokens_5825496a1e8288284e664f128adddff3 UInt8 : 0 -INPUT : 1 -> __text_index_idx_message_hasAllTokens_0f7345644adb0992857049f7ce3d0420 UInt8 : 1 +Actions: INPUT : 0 -> __text_index_idx_category_hasAllTokens_9c785b046c4985d788babb3b2ad90630 UInt8 : 0 +INPUT : 1 -> __text_index_idx_message_hasAllTokens_b79ad6ad12798e01a370b4598f91b250 UInt8 : 1 Parts: 1/1 Granules: 5/5 Name: idx_category diff --git a/tests/queries/0_stateless/04102_text_index_hasAny_hasAll.reference b/tests/queries/0_stateless/04102_text_index_hasAny_hasAll.reference index 805c457f016f..15723126d725 100644 --- a/tests/queries/0_stateless/04102_text_index_hasAny_hasAll.reference +++ b/tests/queries/0_stateless/04102_text_index_hasAny_hasAll.reference @@ -8,12 +8,12 @@ 0 4 -- array tokenizer: index is used -Actions: INPUT : 0 -> __text_index_arr_idx_hasAny_07e61fa0a9933fb0ccf40b67f8b57cd3 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_arr_idx_hasAny_a0642df28adf277f8f9914169374c509 UInt8 : 0 Granules: 3 Granules: 4/4 Name: arr_idx Granules: 3/4 -Actions: INPUT : 0 -> __text_index_arr_idx_hasAll_4b4788174638fe6be1c1d794abd7bdb8 UInt8 : 0 +Actions: INPUT : 0 -> __text_index_arr_idx_hasAll_6d61e90be1041a946fbae78dacec7339 UInt8 : 0 Granules: 1 Granules: 4/4 Name: arr_idx @@ -33,7 +33,7 @@ Granules: 4/4 Name: arr_idx Granules: 2/4 Granules: 1 -INPUT : 1 -> __text_index_arr_idx_hasAll_a9f8838c2835dbe60c454853d72b330b UInt8 : 2 +INPUT : 1 -> __text_index_arr_idx_hasAll_f043b7a64c5345481039a7a1fc760c43 UInt8 : 2 Granules: 4/4 Name: arr_idx Granules: 1/4 From c75f1f16056a8175c5ac7d75de2e813345b833eb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Jul 2026 16:57:16 +0000 Subject: [PATCH 17/86] Backport #111101 to 26.6: Accept empty nested Arrow List/Map with a 0-byte offsets buffer --- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 20 +++ ...04613_arrow_empty_nested_offsets.reference | 8 ++ .../04613_arrow_empty_nested_offsets.sh | 114 ++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 tests/queries/0_stateless/04613_arrow_empty_nested_offsets.reference create mode 100755 tests/queries/0_stateless/04613_arrow_empty_nested_offsets.sh diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 1d799af552b0..187dcd85edb4 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1276,6 +1276,11 @@ static ColumnPtr readOffsetsFromArrowListColumn(const std::shared_ptrnum_chunks(); chunk_i < num_chunks; ++chunk_i) { auto & list_chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); + /// A zero-length list chunk accesses no offsets (the loop below is skipped), so no bytes + /// are required. Skip before checkedCast to accept the 0-byte offsets buffer that Apache + /// Arrow Java < 19.0.0 emits for an empty nested List/Map (see checkBinaryOffsetsBuffer). + if (list_chunk.length() == 0) + continue; auto arrow_offsets_array = list_chunk.offsets(); /// The offsets array is a numeric Int32/Int64 array, validate its buffer before Value() calls. using OffsetArray = typename ArrowOffsetArray::type; @@ -1462,6 +1467,21 @@ static std::shared_ptr getNestedArrowColumn(const std::shar /// Flatten calls IsValid on the parent list array which reads buffers[0]. checkValidityBitmap(list_chunk, column_name); + /// A zero-length list chunk may carry a 0-byte offsets buffer (Apache Arrow Java < 19.0.0 + /// emits one for an empty nested List/Map). Arrow's Flatten() would read offset[0] from + /// that missing buffer and return a slice with a garbage offset; instead push an empty + /// slice of the values array, which preserves the child type with zero rows. + if (list_chunk.length() == 0) + { + const auto & values = list_chunk.values(); + if (!values) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Arrow List chunk has no values array for column '{}'", column_name); + array_vector.emplace_back(values->Slice(0, 0)); + continue; + } + /// Validate the offsets buffer before Flatten() reads it: Flatten() iterates /// over offset[0..length] to slice the values array, so it needs (length+1) entries. /// We also validate monotonicity here, before Flatten(), because Flatten() slices the diff --git a/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.reference b/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.reference new file mode 100644 index 000000000000..322bb0d64c62 --- /dev/null +++ b/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.reference @@ -0,0 +1,8 @@ +1 [] +2 [] +1 [] +2 [] +1 {} +2 {} +1 [] +2 [] diff --git a/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.sh b/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.sh new file mode 100755 index 000000000000..a405dd59f598 --- /dev/null +++ b/tests/queries/0_stateless/04613_arrow_empty_nested_offsets.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Regression test for accepting empty nested List/Map containers whose offsets +# buffer is 0 bytes. Apache Arrow Java < 19.0.0 (bundled with Apache Spark) +# emits a 0-byte offsets buffer for any variable-width vector with valueCount==0. +# When every outer collection in a batch is empty, the inner List/Map vector has +# zero elements and a 0-byte offsets buffer. Every other Arrow implementation +# (arrow-cpp, pyarrow, arrow-rs) accepts this; ClickHouse must too. The fix: +# when the list chunk length is 0, no offsets are ever accessed, so 0 bytes are +# required; skip the offsets-buffer validation before the cast. +# +# Follow-up to the flat String/Binary fix; here the element/value type is +# irrelevant (Array(Array(Int32)) fails just like the String cases), so this +# exercises the List/Map offsets path, not the String/Binary path. +# +# Covers four shapes where every outer collection is empty: +# (a) Array(Array(Int32)) -> inner list child length=0 (no String) +# (b) Array(Array(String)) -> inner list child length=0 +# (c) Map(String, Array(String)) -> map value (list) child length=0 +# (d) Array(Map(String, Int32)) -> inner map child length=0 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TMP_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "$TMP_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +python3 - "$TMP_DIR" <<'PYEOF' +import io, sys +import pyarrow as pa +import pyarrow.ipc as ipc + +out = sys.argv[1] + +def write_arrow(tbl): + buf = io.BytesIO() + with ipc.new_file(buf, tbl.schema) as w: + w.write_table(tbl) + return buf.getvalue() + +def zero_byte_string_array(): + """String array with length=0 and a 0-byte offsets buffer.""" + return pa.Array.from_buffers( + pa.string(), 0, + [None, pa.py_buffer(b""), pa.py_buffer(b"")] + ) + +def zero_byte_list_array(child): + """List array with length=0 and a 0-byte offsets buffer, as Apache Arrow + Java < 19.0.0 produces for an empty nested collection.""" + return pa.Array.from_buffers( + pa.list_(child.type), 0, + [None, pa.py_buffer(b"")], children=[child] + ) + +def outer_empty(inner): + """Two empty outer lists wrapping the given (empty) inner array.""" + return pa.ListArray.from_arrays(pa.array([0, 0, 0], type=pa.int32()), inner) + +ids = pa.array([1, 2], type=pa.int32()) + +# (a) Array(Array(Int32)) with two empty outer arrays -> inner list child length=0. +inner_int = zero_byte_list_array(pa.array([], type=pa.int32())) +tbl_a = pa.table({"id": ids, "a": outer_empty(inner_int)}) +open(f"{out}/nested_int.arrow", "wb").write(write_arrow(tbl_a)) + +# (b) Array(Array(String)) with two empty outer arrays -> inner list child length=0. +inner_str = zero_byte_list_array(zero_byte_string_array()) +tbl_b = pa.table({"id": ids, "a": outer_empty(inner_str)}) +open(f"{out}/nested_string.arrow", "wb").write(write_arrow(tbl_b)) + +# (c) Map(String, Array(String)) with two empty maps -> map value (list) child length=0. +map_col = pa.MapArray.from_arrays( + pa.array([0, 0, 0], type=pa.int32()), + zero_byte_string_array(), + zero_byte_list_array(zero_byte_string_array()), +) +tbl_c = pa.table({"id": ids, "m": map_col}) +open(f"{out}/nested_map_value.arrow", "wb").write(write_arrow(tbl_c)) + +# (d) Array(Map(String, Int32)) with two empty outer arrays -> inner map child length=0. +inner_map = pa.Array.from_buffers( + pa.map_(pa.string(), pa.int32()), 0, + [None, pa.py_buffer(b"")], + children=[pa.StructArray.from_arrays( + [zero_byte_string_array(), pa.array([], type=pa.int32())], + names=["key", "value"])], +) +tbl_d = pa.table({"id": ids, "a": outer_empty(inner_map)}) +open(f"{out}/nested_array_map.arrow", "wb").write(write_arrow(tbl_d)) +PYEOF + +# This release ships only the Apache Arrow library reader +# (ArrowColumnToCHColumn); the native ClickHouse Arrow reader does not exist +# here, so there is nothing to switch and the queries run against the library +# reader directly. + +# (a) Array(Array(Int32)): two empty outer arrays. +$CLICKHOUSE_LOCAL --query \ + "SELECT id, a FROM file('${TMP_DIR}/nested_int.arrow', Arrow) ORDER BY id" + +# (b) Array(Array(String)): two empty outer arrays. +$CLICKHOUSE_LOCAL --query \ + "SELECT id, a FROM file('${TMP_DIR}/nested_string.arrow', Arrow) ORDER BY id" + +# (c) Map(String, Array(String)): two empty maps. +$CLICKHOUSE_LOCAL --query \ + "SELECT id, m FROM file('${TMP_DIR}/nested_map_value.arrow', Arrow) ORDER BY id" + +# (d) Array(Map(String, Int32)): two empty outer arrays. +$CLICKHOUSE_LOCAL --query \ + "SELECT id, a FROM file('${TMP_DIR}/nested_array_map.arrow', Arrow) ORDER BY id" From 84ddc40b31dd7b6d1222d5de938c6b5329a4811b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Jul 2026 20:18:13 +0000 Subject: [PATCH 18/86] Backport #111039 to 26.6: Fix LOGICAL_ERROR on DROP COLUMN with ALIAS column using IN --- src/Interpreters/MutationsInterpreter.cpp | 7 +-- src/Interpreters/inplaceBlockConversions.cpp | 4 +- src/Planner/CollectTableExpressionData.cpp | 12 ++++ src/Planner/CollectTableExpressionData.h | 9 +++ src/Storages/AlterCommands.cpp | 2 +- ...2_alter_drop_column_alias_in_set.reference | 10 ++++ .../04612_alter_drop_column_alias_in_set.sql | 57 +++++++++++++++++++ 7 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.reference create mode 100644 tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.sql diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 11e30ec78fa1..2b409729e519 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -1641,8 +1640,7 @@ void MutationsInterpreter::prepareMutationStages(std::vector & prepared_s auto planner_context = std::make_shared( execution_context, global_planner_context, SelectQueryOptions{}); - collectSourceColumns(expression, planner_context, /*keep_alias_columns=*/true); - collectSets(expression, *planner_context); + collectSetsAndSourceColumns(expression, planner_context, /*keep_alias_columns=*/true); /// 3. Build input columns from all available columns plus any /// virtual columns actually referenced by the expression @@ -1788,8 +1786,7 @@ void MutationsInterpreter::prepareMutationStages(std::vector & prepared_s auto update_tree = buildQueryTree(update_expr_list, execution_context); QueryAnalyzer update_analyzer(/*only_analyze=*/!execute_scalar_subqueries); update_analyzer.resolve(update_tree, table_node, execution_context); - collectSourceColumns(update_tree, planner_context, true); - collectSets(update_tree, *planner_context); + collectSetsAndSourceColumns(update_tree, planner_context, true); auto update_actions = std::make_shared(); update_actions->dag = ActionsDAG(available_columns_for_step); diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 9effe3095f27..fc6751d06621 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -240,8 +239,7 @@ std::optional createExpressionsAnalyzer( GlobalPlannerContextPtr global_planner_context = std::make_shared(nullptr, nullptr, nullptr, FiltersForTableExpressionMap{}); auto planner_context = std::make_shared(execution_context, global_planner_context, SelectQueryOptions{}); - collectSourceColumns(expression, planner_context, true /*keep_alias_columns*/); - collectSets(expression, *planner_context); + collectSetsAndSourceColumns(expression, planner_context, true /*keep_alias_columns*/); auto actions = buildActionsDAGFromExpressionNode(expression, header.getColumnsWithTypeAndName(), planner_context, {}).first; chassert(expression->getChildren().size() == actions.getOutputs().size()); diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index b98ee4a33f8c..bc52e24499cc 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -108,6 +109,11 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextgetGlobalPlannerContext()->createColumnIdentifier(node); + /// The ALIAS column may be referenced from inside a subquery, which collectSets + /// never descends into (it skips QUERY and UNION children), so register the sets + /// of the ALIAS expression here before building actions over it. + collectSets(column_node->getExpression(), *planner_context); + ActionsDAG alias_column_actions_dag; ColumnNodePtrWithHashSet empty_correlated_columns_set; PlannerActionsVisitor actions_visitor(planner_context, empty_correlated_columns_set, false); @@ -465,4 +471,10 @@ void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr collect_source_columns_visitor.visit(expression_node); } +void collectSetsAndSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context, bool keep_alias_columns) +{ + collectSets(expression_node, *planner_context); + collectSourceColumns(expression_node, planner_context, keep_alias_columns); +} + } diff --git a/src/Planner/CollectTableExpressionData.h b/src/Planner/CollectTableExpressionData.h index b0cebc156825..b48c59a74285 100644 --- a/src/Planner/CollectTableExpressionData.h +++ b/src/Planner/CollectTableExpressionData.h @@ -21,4 +21,13 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr */ void collectSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context, bool keep_alias_columns = true); +/** Register sets, then collect source columns for expression node, in that order. + * + * The order is required: collectSourceColumns expands ALIAS column expressions through + * PlannerActionsVisitor, which resolves IN operators via PlannerContext::getPreparedSets; + * the sets must therefore be registered by collectSets beforehand, otherwise an IN inside + * an ALIAS expression throws "No set is registered for key". + */ +void collectSetsAndSourceColumns(QueryTreeNodePtr & expression_node, PlannerContextPtr & planner_context, bool keep_alias_columns = true); + } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 28af0bee46c2..0526d66d378c 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1832,7 +1832,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const analyzer.resolve(expression, fake_table_expression, execution_context); GlobalPlannerContextPtr global_planner_context = std::make_shared(nullptr, nullptr, nullptr, FiltersForTableExpressionMap{}); auto planner_context = std::make_shared(execution_context, global_planner_context, SelectQueryOptions{}); - collectSourceColumns(expression, planner_context); + collectSetsAndSourceColumns(expression, planner_context); if (const auto * table_expression = planner_context->getTableExpressionDataOrNull(fake_table_expression)) { for (const auto & selected_column : table_expression->getSelectedColumnsNames()) diff --git a/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.reference b/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.reference new file mode 100644 index 000000000000..557c4c99cb91 --- /dev/null +++ b/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.reference @@ -0,0 +1,10 @@ +after drop 1 1 YES +after drop 2 0 NO +after drop 3 1 YES +subquery select 1 0 +subquery select 2 1 +subquery select 3 0 +subquery where 1 +after delete 2 0 NO +after update 2 1 YES +after lightweight delete 0 diff --git a/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.sql b/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.sql new file mode 100644 index 000000000000..233e52f5934d --- /dev/null +++ b/tests/queries/0_stateless/04612_alter_drop_column_alias_in_set.sql @@ -0,0 +1,57 @@ +-- Tags: no-old-analyzer +-- The bug and its fix are in the Analyzer's Planner (`collectSets`), and the old analyzer cannot +-- execute these mutations at all (the ALIAS -> ALIAS -> IN chain in a mutation predicate throws +-- UNKNOWN_IDENTIFIER). Background mutations run with server-default settings, so a session-level +-- `SET enable_analyzer = 1` is not enough and the test must be skipped in old-analyzer runs. + +-- ALTER validation and mutations must not abort with LOGICAL_ERROR "No set is registered +-- for key" when the table has an ALIAS column referencing another ALIAS column that uses an +-- IN expression. Expanding such ALIAS expressions runs PlannerActionsVisitor, which resolves +-- IN via the prepared sets, so the sets must be registered (collectSets) before the columns +-- are collected. Covers the DROP COLUMN validator and the DELETE/UPDATE mutation paths. + +SET mutations_sync = 2; +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_alias_in_set; + +CREATE TABLE t_alias_in_set +( + id UInt64, + category String, + extra UInt32 DEFAULT 0, + is_special UInt8 ALIAS category IN ('electronics', 'clothing', 'food'), + label String ALIAS if(is_special, 'YES', 'NO') +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO t_alias_in_set (id, category) VALUES (1, 'electronics'), (2, 'other'), (3, 'food'); + +-- DROP COLUMN validation expands ALIAS expressions (AlterCommands::validate). +ALTER TABLE t_alias_in_set DROP COLUMN extra; +SELECT 'after drop', id, is_special, label FROM t_alias_in_set ORDER BY id; + +-- Correlated scalar subquery referencing the ALIAS chain (found by the AST fuzzer on this PR). +-- collectSets does not descend into subqueries, so the ALIAS expansion must register +-- the sets of the ALIAS expression itself. +SELECT 'subquery select', id, 'YES' IS DISTINCT FROM (SELECT label) FROM t_alias_in_set ORDER BY id; +SELECT 'subquery where', count() FROM t_alias_in_set WHERE 'YES' IS DISTINCT FROM (SELECT label); +-- In a mutation filter a correlated subquery is rejected with a regular error, not a LOGICAL_ERROR. +ALTER TABLE t_alias_in_set DELETE WHERE 'YES' IS DISTINCT FROM (SELECT label); -- { serverError NOT_IMPLEMENTED } + +-- Mutation predicate references ALIAS -> ALIAS -> IN (MutationsInterpreter). +ALTER TABLE t_alias_in_set DELETE WHERE label = 'YES'; +SELECT 'after delete', id, is_special, label FROM t_alias_in_set ORDER BY id; + +-- Mutation update value references ALIAS -> ALIAS -> IN (MutationsInterpreter). +ALTER TABLE t_alias_in_set UPDATE category = if(is_special, 'clothing', 'food') WHERE id = 2; +SELECT 'after update', id, is_special, label FROM t_alias_in_set ORDER BY id; + +-- Lightweight DELETE goes through the same mutation preparation. +DELETE FROM t_alias_in_set WHERE label = 'YES'; +SELECT 'after lightweight delete', count() FROM t_alias_in_set; + +ALTER TABLE t_alias_in_set DROP COLUMN IF EXISTS nonexistent_col; + +DROP TABLE t_alias_in_set; From 3a9a5474822e2e743b7664f7b8c7f8338a6d9029 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 06:58:19 +0000 Subject: [PATCH 19/86] Backport #111108 to 26.6: Fix bug with `data_kind = Preprocessed` failing the whole async batch --- src/Interpreters/AsynchronousInsertQueue.cpp | 43 +++++++++++++------ ...sert_block_batch_error_isolation.reference | 6 +++ ...sync_insert_block_batch_error_isolation.sh | 38 ++++++++++++++++ 3 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.reference create mode 100755 tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.sh diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 8c96831d070e..88207be6ba60 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -1090,12 +1090,13 @@ try auto add_entry_to_asynchronous_insert_log = [&, query_by_format = NameToNameMap{}]( const InsertData::EntryPtr & entry, - const String & parsing_exception, + const String & exception, size_t num_rows, - size_t num_bytes) mutable + size_t num_bytes, + bool is_flush_error = false) mutable { /// Track per-entry stats for reporting back to clients on success. - if (parsing_exception.empty()) + if (exception.empty()) per_entry_progress_results[entry.get()] = ResultProgress{num_rows, num_bytes}; if (!async_insert_log) @@ -1110,7 +1111,7 @@ try elem.query_id = entry->query_id; elem.bytes = num_bytes; elem.rows = num_rows; - elem.exception = parsing_exception; + elem.exception = exception; elem.data_kind = entry->chunk.getDataKind(); elem.timeout_milliseconds = data->timeout_ms.count(); elem.flush_query_id = insert_query_id; @@ -1132,10 +1133,12 @@ try else elem.query_for_logging = get_query_by_format(entry->format); - /// If there was a parsing error, - /// the entry won't be flushed anyway, - /// so add the log element immediately. - if (!elem.exception.empty()) + if (is_flush_error) + { + /// Per-entry conversion failure at flush: log immediately as `FlushError` with a real `flush_time`. + appendElementsToLogSafe(*async_insert_log, {std::move(elem)}, std::chrono::system_clock::now(), exception); + } + else if (!elem.exception.empty()) { elem.status = AsynchronousInsertLogElement::ParsingError; async_insert_log->add(std::move(elem)); @@ -1186,7 +1189,7 @@ try if (async_insert_log) { for (const auto & entry : data->entries) - add_entry_to_asynchronous_insert_log(entry, /*parsing_exception=*/ "", /*num_rows=*/ 0, entry->chunk.byteSize()); + add_entry_to_asynchronous_insert_log(entry, /*exception=*/ "", /*num_rows=*/ 0, entry->chunk.byteSize()); auto exception = getCurrentExceptionMessage(false); auto flush_time = std::chrono::system_clock::now(); @@ -1387,13 +1390,27 @@ Chunk AsynchronousInsertQueue::processPreprocessedEntries( Block block_to_insert = *block; if (block_to_insert.rows() == 0) { - add_to_async_insert_log(entry, /*parsing_exception=*/ "", block_to_insert.rows(), block_to_insert.bytes()); + add_to_async_insert_log(entry, /*exception=*/ "", block_to_insert.rows(), block_to_insert.bytes()); entry->resetChunk(); continue; } - if (!isCompatibleHeader(block_to_insert, header)) - convertBlockToHeader(block_to_insert, header, context_); + try + { + if (!isCompatibleHeader(block_to_insert, header)) + convertBlockToHeader(block_to_insert, header, context_); + } + catch (...) + { + /// Per-entry isolation: log as `FlushError` (not `ParsingError`) with a real + /// `flush_time`, via the `is_flush_error` path in `add_to_async_insert_log`. + const auto exception_msg = getCurrentExceptionMessage(/*with_stacktrace=*/ false); + LOG_ERROR(logger, "Failed conversion for insert query id {}. {}", entry->query_id, exception_msg); + add_to_async_insert_log(entry, exception_msg, /*num_rows=*/ 0, block->bytes(), /*is_flush_error=*/ true); + entry->finish(std::current_exception()); + entry->resetChunk(); + continue; + } auto columns = block_to_insert.getColumns(); for (size_t i = 0, s = columns.size(); i < s; ++i) @@ -1403,7 +1420,7 @@ Chunk AsynchronousInsertQueue::processPreprocessedEntries( deduplication_info->setUserToken(entry->async_dedup_token, block_to_insert.rows()); - add_to_async_insert_log(entry, /*parsing_exception=*/ "", block_to_insert.rows(), block_to_insert.bytes()); + add_to_async_insert_log(entry, /*exception=*/ "", block_to_insert.rows(), block_to_insert.bytes()); entry->resetChunk(); } diff --git a/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.reference b/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.reference new file mode 100644 index 000000000000..dcc486337308 --- /dev/null +++ b/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.reference @@ -0,0 +1,6 @@ +=== VALUES (Parsed): bad entry is isolated, good entry survives === +1 100 +=== native block (Preprocessed): bad entry is isolated, good entry survives === +1 100 +Preprocessed Ok 1 +Preprocessed FlushError 1 diff --git a/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.sh b/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.sh new file mode 100755 index 000000000000..d8c003fa65b8 --- /dev/null +++ b/tests/queries/0_stateless/04612_async_insert_block_batch_error_isolation.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Per-entry error isolation on the async insert queue flush. +# Two async entries with the same query share one bucket; a concurrent `MODIFY COLUMN` makes +# only one unconvertible at flush. Both the parsing path (`INSERT ... VALUES`, `data_kind = +# Parsed`, isolated via `on_error` in `processEntriesWithParsing`) and the block path (native +# block, `data_kind = Preprocessed`, isolated in `processPreprocessedEntries`) must isolate +# the bad entry so the valid sibling (id=1, '100' -> 100) still lands. +# Before the block-path fix the unconvertible block failed the whole batch. + +async=(--async_insert=1 --wait_for_async_insert=0 --async_insert_busy_timeout_max_ms=300000 --async_insert_busy_timeout_min_ms=300000 --async_insert_use_adaptive_busy_timeout=0) + +echo "=== VALUES (Parsed): bad entry is isolated, good entry survives ===" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_async_iso_values" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_async_iso_values (id Int64, amount String) ENGINE = MergeTree ORDER BY id" +$CLICKHOUSE_CLIENT "${async[@]}" -q "INSERT INTO t_async_iso_values VALUES (1, '100')" +$CLICKHOUSE_CLIENT "${async[@]}" -q "INSERT INTO t_async_iso_values VALUES (2, 'hello')" +$CLICKHOUSE_CLIENT -q "ALTER TABLE t_async_iso_values MODIFY COLUMN amount Int64" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE t_async_iso_values" +$CLICKHOUSE_CLIENT -q "SELECT id, amount FROM t_async_iso_values ORDER BY id" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_async_iso_values" + +echo "=== native block (Preprocessed): bad entry is isolated, good entry survives ===" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_async_iso_block" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_async_iso_block (id Int64, amount String) ENGINE = MergeTree ORDER BY id" +$CLICKHOUSE_CLIENT -q "SELECT toInt64(1) AS id, '100'::String AS amount FORMAT Native" | $CLICKHOUSE_CLIENT "${async[@]}" -q "INSERT INTO t_async_iso_block FORMAT Native" +$CLICKHOUSE_CLIENT -q "SELECT toInt64(2) AS id, 'hello'::String AS amount FORMAT Native" | $CLICKHOUSE_CLIENT "${async[@]}" -q "INSERT INTO t_async_iso_block FORMAT Native" +$CLICKHOUSE_CLIENT -q "ALTER TABLE t_async_iso_block MODIFY COLUMN amount Int64" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE t_async_iso_block" +$CLICKHOUSE_CLIENT -q "SELECT id, amount FROM t_async_iso_block ORDER BY id" +# Verify the bad entry is recorded as a failure in the log, not silently dropped. +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS asynchronous_insert_log" +$CLICKHOUSE_CLIENT -q "SELECT data_kind, status, flush_time_microseconds > 0 AS has_flush_time FROM system.asynchronous_insert_log WHERE database = currentDatabase() AND table = 't_async_iso_block' AND event_date >= yesterday() AND event_time >= now() - 600 ORDER BY status" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_async_iso_block" From 2cedfd2a242ef7cf0fb13a821fc554cbadfcb6cd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 07:57:25 +0000 Subject: [PATCH 20/86] Backport #108043 to 26.6: Fix Bad cast for a qualified asterisk over a JOIN USING key nested under a PASTE/CROSS join --- src/Analyzer/Resolve/IdentifierResolveScope.h | 3 + src/Analyzer/Resolve/QueryAnalyzer.cpp | 86 +++++++--- ...asterisk_join_using_nullable_key.reference | 14 ++ ...ified_asterisk_join_using_nullable_key.sql | 147 +++++++++++++++++- 4 files changed, 221 insertions(+), 29 deletions(-) diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.h b/src/Analyzer/Resolve/IdentifierResolveScope.h index 3b6afa1a9415..d98860443e55 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.h +++ b/src/Analyzer/Resolve/IdentifierResolveScope.h @@ -201,6 +201,9 @@ struct IdentifierResolveScope /// JOINs count size_t joins_count = 0; + /// JOIN USING count (joins whose keys can retype a matched column) + size_t using_joins_count = 0; + /// Subquery depth size_t subquery_depth = 0; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 353e6d381388..62a3dedc7c8a 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1791,14 +1791,20 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::getMatchedColumnNodesWithN } -/// Columns that resolved from matcher can also match columns from JOIN USING. -/// In that case we update type to type of column in USING section. +/// Columns resolved from a matcher can also be JOIN USING keys, whose type the join changes +/// (the supertype of both sides, and Nullable when an OUTER side makes the data Nullable). /// -/// Unqualified matcher (`*`): the matched column IS the merged USING key, so it takes the -/// key type as-is. Qualified matcher (`t.*`): the matched column is `t`'s own column, so its -/// type must equal what the explicit reference `t.col` resolves to. The merged key's type is -/// not correct here: in a nested JOIN it reflects the outer join's other side, while `t.col` -/// only follows the joins `t` participates in. +/// Unqualified matcher (`*`): the matched column IS the merged USING key of the top join, so it +/// takes that key's type directly. +/// +/// Qualified matcher (`t.*`): the matched column is `t`'s own column, so its type must equal what +/// the explicit reference `t.col` resolves to. Rather than inspect the USING joins here, resolve +/// `t.col` through the normal identifier flow and adopt its type. That flow +/// (IdentifierResolver::tryResolveIdentifierFromJoin) follows only the joins `t` participates in, +/// wherever the USING join sits in the tree, applies the same type correction, and registers the +/// changed type in `scope.join_columns_with_changed_types`. So a USING key that `t.col` matches +/// only by name in a join `t` does not take part in is naturally not applied, and no separate +/// participation check or registration is needed. void QueryAnalyzer::updateMatchedColumnsFromJoinUsing( QueryTreeNodesWithNames & result_matched_column_nodes_with_names, bool is_qualified_matcher, @@ -1816,6 +1822,47 @@ void QueryAnalyzer::updateMatchedColumnsFromJoinUsing( scope.scope_node->formatASTForErrorMessage()); } + if (is_qualified_matcher) + { + /// Only a JOIN USING key can change a matched column's type here; `join_use_nulls` + /// nullability for the matcher is applied later in resolveMatcher. With no USING join in + /// scope there is nothing to correct, so skip the per-column identifier resolution. + /// The count lives on the query scope whose join tree is inspected above, not on the + /// current scope: a matcher in a lambda body (`arrayMap(x -> t.*, ...)`) resolves through + /// a fresh child scope whose counter is zero, but still expands `t.*` from the parent query. + if (nearest_query_scope->using_joins_count == 0) + return; + + for (auto & [matched_column_node, _] : result_matched_column_nodes_with_names) + { + auto & matched_column_node_typed = matched_column_node->as(); + + Identifier explicit_identifier = matched_qualified_identifier; + explicit_identifier.push_back(matched_column_node_typed.getColumnName()); + auto explicit_lookup = IdentifierLookup{explicit_identifier, IdentifierLookupContext::EXPRESSION}; + IdentifierResolveContext explicit_resolve_settings; + explicit_resolve_settings.allow_to_check_cte = false; + explicit_resolve_settings.allow_to_check_database_catalog = false; + auto explicit_resolve_result = tryResolveIdentifier(explicit_lookup, scope, explicit_resolve_settings); + if (!explicit_resolve_result.resolved_identifier) + continue; + + auto resolved_type = explicit_resolve_result.resolved_identifier->getResultType(); + if (resolved_type->equals(*matched_column_node_typed.getColumnType())) + continue; + + auto it = node_to_projection_name.find(matched_column_node); + matched_column_node = matched_column_node->clone(); + if (it != node_to_projection_name.end()) + node_to_projection_name.emplace(matched_column_node, it->second); + + matched_column_node->as().setColumnType(resolved_type); + correctColumnExpressionType(matched_column_node->as(), scope.context); + } + + return; + } + const auto & join_tree = nearest_query_scope_query_node->getJoinTree(); const auto * join_node = join_tree->as(); @@ -1855,24 +1902,6 @@ void QueryAnalyzer::updateMatchedColumnsFromJoinUsing( auto using_column_type = join_using_column_node.getResultType(); - /// Qualified matcher: the matched column is `t.col`, NOT the merged USING key. - /// Resolve the explicit identifier `t.col` and adopt its type, exactly matching - /// how an explicit reference is typed (IdentifierResolver::tryResolveIdentifierFromJoin). - /// The merged key's type is wrong here: in a nested JOIN it reflects the OUTER - /// join's siblings, while `t.col` only follows the joins `t` participates in. - if (is_qualified_matcher) - { - Identifier explicit_identifier = matched_qualified_identifier; - explicit_identifier.push_back(matched_column_name); - auto explicit_lookup = IdentifierLookup{explicit_identifier, IdentifierLookupContext::EXPRESSION}; - IdentifierResolveContext explicit_resolve_settings; - explicit_resolve_settings.allow_to_check_cte = false; - explicit_resolve_settings.allow_to_check_database_catalog = false; - auto explicit_resolve_result = tryResolveIdentifier(explicit_lookup, scope, explicit_resolve_settings); - if (explicit_resolve_result.resolved_identifier) - using_column_type = explicit_resolve_result.resolved_identifier->getResultType(); - } - auto it = node_to_projection_name.find(matched_column_node); matched_column_node = matched_column_node->clone(); if (it != node_to_projection_name.end()) @@ -4164,6 +4193,8 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod join_tree_node_ptrs_to_process_queue.push_back(&join.getRightTableExpression()); scope.table_expressions_in_resolve_process.insert(current_join_tree_node.get()); ++scope.joins_count; + if (join.isUsingJoinExpression()) + ++scope.using_joins_count; break; } default: @@ -5207,6 +5238,11 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS join_node_typed.getJoinExpression() = std::make_shared(std::move(using_nodes)); join_node_typed.setUsingJoinExpression(); + + /// initializeQueryJoinTreeNode counts USING joins before this NATURAL -> USING conversion, + /// when the join still has no USING expression. Count it here so the qualified-matcher guard + /// in updateMatchedColumnsFromJoinUsing sees the synthesized key and corrects the matched type. + ++scope.using_joins_count; } if (join_node_typed.isOnJoinExpression()) diff --git a/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.reference b/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.reference index 7a28c4fbad9c..7b2b23a4528c 100644 --- a/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.reference +++ b/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.reference @@ -11,3 +11,17 @@ Int64 1 Nullable(UInt64) 1 +1 +1 +1 +1 +1 +5 five +1 one +5 five +5 five +1 +Nullable(UInt64) +1 +1 +1 diff --git a/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.sql b/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.sql index c47da4bde6ba..b17b72360cba 100644 --- a/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.sql +++ b/tests/queries/0_stateless/04329_qualified_asterisk_join_using_nullable_key.sql @@ -1,6 +1,8 @@ +-- Tags: long + -- A qualified matcher (`t.*`) over a JOIN USING key must keep the matched column's own -- type, not the merged key's. Otherwise it can wrongly become Nullable (join_use_nulls = 0, --- outer JOIN against a Nullable key) and an aggregate over it aborts with +-- outer JOIN against a Nullable key) and an aggregate over it throws the exception -- "Bad cast from type DB::IColumn const* to DB::ColumnNullable const*". DROP TABLE IF EXISTS t_jn1; @@ -17,7 +19,7 @@ INSERT INTO t_jn3 VALUES (0, 'g'), (2, 'h'), (4, 'i'); SET enable_analyzer = 1; --- The aggregation over the qualified matcher used to crash the server. +-- The aggregation over the qualified matcher used to throw a Bad cast exception. SELECT count() FROM ( SELECT anyHeavy(sipHash64(t2.*)) @@ -70,7 +72,7 @@ LIMIT 1 SETTINGS join_use_nulls = 0; -- Aggregating over a qualified matcher whose USING key both widens (UInt8 -> Int64) and --- gains nullability on the other side must not crash with join_use_nulls = 0. +-- gains nullability on the other side must not throw the Bad cast exception with join_use_nulls = 0. SELECT count() FROM ( SELECT anyHeavy(sipHash64(t1.*)) @@ -82,7 +84,7 @@ SETTINGS join_use_nulls = 0; -- USING key. When a sibling USING table is Nullable, the inner-merged key (and the runtime -- column) is Nullable, so the matcher type must be Nullable too, matching the explicit -- reference. The matcher used to wrongly take the matched column's own (non-Nullable) type, --- and an `-OrNull` aggregate over it aborted with the opposite Bad cast +-- and an `-OrNull` aggregate over it threw the opposite Bad cast exception -- "from type DB::ColumnNullable to DB::ColumnVector". DROP TABLE IF EXISTS t_jn1_nullable; DROP TABLE IF EXISTS t_jn3_nullable; @@ -108,9 +110,146 @@ SELECT count() FROM ) SETTINGS join_use_nulls = 0; +-- The USING join `t2` participates in can sit below a non-USING top join (PASTE/CROSS/comma +-- join, or an outer ON join). The qualified matcher must then still adopt the type of the +-- explicit reference, since inspecting only the top node would skip the type correction and +-- leave the matched column with its non-Nullable table type while the runtime column is +-- Nullable, throwing a Bad cast exception from an aggregate over it. The type equality must hold +-- and the aggregate must not throw the exception for every wrapping join shape. + +-- PASTE JOIN wrapping the USING join. +SELECT toTypeName(sipHash64(t2.*)) = toTypeName(sipHash64(t2.id, t2.value)) +FROM t_jn2 AS t2 RIGHT JOIN t_jn3_nullable USING (id) PASTE JOIN numbers(2) AS n +LIMIT 1 +SETTINGS join_use_nulls = 0; + +SELECT count() FROM +( + SELECT anyHeavy(sipHash64(t2.*)) + FROM t_jn2 AS t2 RIGHT JOIN t_jn3_nullable USING (id) PASTE JOIN numbers(2) AS n +) +SETTINGS join_use_nulls = 0; + +-- CROSS JOIN wrapping the USING join. +SELECT count() FROM +( + SELECT anyHeavy(sipHash64(t2.*)) + FROM t_jn2 AS t2 RIGHT JOIN t_jn3_nullable USING (id) CROSS JOIN numbers(2) AS n +) +SETTINGS join_use_nulls = 0; + +-- Comma join wrapping the USING join. +SELECT count() FROM +( + SELECT anyHeavy(sipHash64(t2.*)) + FROM t_jn2 AS t2 RIGHT JOIN t_jn3_nullable USING (id), numbers(2) AS n +) +SETTINGS join_use_nulls = 0; + +-- Outer ON join wrapping the USING join. +SELECT count() FROM +( + SELECT anyHeavy(sipHash64(t2.*)) + FROM t_jn2 AS t2 RIGHT JOIN t_jn3_nullable USING (id) INNER JOIN t_jn1 ON t2.value = t_jn1.value +) +SETTINGS join_use_nulls = 0; + DROP TABLE t_jn1_nullable; DROP TABLE t_jn3_nullable; +-- Examining every USING join in the tree for a qualified matcher must not pick up a USING join +-- the matched table does not take part in. With `(t_a JOIN t_b USING(id)) CROSS JOIN t_c`, +-- resolving `t_c.*` reaches the inner `USING(id)` only because the key name `id` coincides with +-- `t_c.id`. The matched column must not be retyped against, nor registered as changed-type from, +-- that unrelated key: such a registration rewrites `t_c.id` to `t_a.id` during PREWHERE +-- replacement (and treats a later unqualified `id` as equal to it instead of ambiguous), giving +-- wrong PREWHERE results. The column types here are identical, so the only observable effect is +-- the wrong rewrite; PREWHERE must filter on `t_c.id`, matching the equivalent WHERE. +DROP TABLE IF EXISTS t_a; +DROP TABLE IF EXISTS t_b; +DROP TABLE IF EXISTS t_c; +CREATE TABLE t_a (id Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_b (id UInt32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_c (id UInt64, v String) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_a VALUES (1); +INSERT INTO t_b VALUES (1); +INSERT INTO t_c VALUES (1, 'one'), (2, 'two'), (5, 'five'); + +-- PREWHERE on the non-participating table's key must filter that table, not the unrelated USING key. +SELECT t_c.* FROM t_a INNER JOIN t_b USING (id) CROSS JOIN t_c PREWHERE t_c.id = 5 SETTINGS join_use_nulls = 0; +SELECT t_c.* FROM t_a INNER JOIN t_b USING (id) CROSS JOIN t_c PREWHERE t_c.id = 1 SETTINGS join_use_nulls = 0; +-- Same with a comma join wrapping the USING join. +SELECT t_c.* FROM t_a INNER JOIN t_b USING (id), t_c PREWHERE t_c.id = 5 SETTINGS join_use_nulls = 0; +-- The rewrite must match the equivalent WHERE (which is not subject to the changed-type replacement). +SELECT t_c.* FROM t_a INNER JOIN t_b USING (id) CROSS JOIN t_c WHERE t_c.id = 5 SETTINGS join_use_nulls = 0; + +DROP TABLE t_a; +DROP TABLE t_b; +DROP TABLE t_c; + +-- A NATURAL JOIN synthesizes its USING key (the common column names) only while resolving the +-- join, after the join-tree walk that counts USING joins has already run. The qualified matcher +-- must still adopt the type the synthesized key gives the explicit reference; otherwise `t1.*` +-- keeps the table (non-Nullable) type while the runtime column is Nullable, and an aggregate over +-- it throws the same Bad cast exception. The type equality must hold and the aggregate must not throw it. +DROP TABLE IF EXISTS nt1; +DROP TABLE IF EXISTS nt2; +CREATE TABLE nt1 (id UInt64, x String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE nt2 (id Nullable(UInt64), y String) ENGINE = MergeTree ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO nt1 VALUES (0, 'a'), (1, 'b'), (2, 'c'); +INSERT INTO nt2 VALUES (0, 'g'), (2, 'h'), (4, 'i'); + +SELECT toTypeName(sipHash64(nt1.*)) = toTypeName(sipHash64(nt1.id, nt1.x)) +FROM nt1 NATURAL RIGHT JOIN nt2 +LIMIT 1 +SETTINGS join_use_nulls = 0; + +SELECT toTypeName(nt1.id) +FROM nt1 NATURAL RIGHT JOIN nt2 +LIMIT 1 +SETTINGS join_use_nulls = 0; + +SELECT count() FROM +( + SELECT anyHeavy(sipHash64(nt1.*)) + FROM nt1 NATURAL RIGHT JOIN nt2 +) +SETTINGS join_use_nulls = 0; + +DROP TABLE nt1; +DROP TABLE nt2; + +-- A qualified matcher inside a lambda body (`arrayMap(x -> t.*, ...)`) resolves through a fresh +-- child scope, but still expands `t.*` from the parent query's join tree. The USING-join presence +-- check must look at that parent query scope, not the lambda's (whose counter is zero); otherwise +-- the matcher keeps the table (non-Nullable) type while the explicit reference is Nullable, and an +-- aggregate over it throws the same Bad cast exception. +DROP TABLE IF EXISTS lt2; +DROP TABLE IF EXISTS lt1_nullable; +DROP TABLE IF EXISTS lt3_nullable; +CREATE TABLE lt2 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE lt1_nullable (id Nullable(UInt64), value String) ENGINE = MergeTree ORDER BY tuple() SETTINGS allow_nullable_key = 1; +CREATE TABLE lt3_nullable (id Nullable(UInt64), value String) ENGINE = MergeTree ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO lt2 VALUES (0, 'd'), (1, 'e'), (3, 'f'); +INSERT INTO lt1_nullable VALUES (0, 'a'), (1, 'b'), (2, 'c'); +INSERT INTO lt3_nullable VALUES (0, 'g'), (2, 'h'), (4, 'i'); + +SELECT toTypeName(arrayMap(x -> sipHash64(t2.*), [1])) = toTypeName(arrayMap(x -> sipHash64(t2.id, t2.value), [1])) +FROM lt1_nullable LEFT JOIN lt2 AS t2 USING (id) RIGHT JOIN lt3_nullable USING (id) +LIMIT 1 +SETTINGS join_use_nulls = 0; + +SELECT count() FROM +( + SELECT anyHeavyOrNull(arrayMap(x -> sipHash64(t2.*), [1])) + FROM lt1_nullable LEFT JOIN lt2 AS t2 USING (id) GLOBAL RIGHT JOIN lt3_nullable USING (id) +) +SETTINGS join_use_nulls = 0; + +DROP TABLE lt2; +DROP TABLE lt1_nullable; +DROP TABLE lt3_nullable; + DROP TABLE t_jn1; DROP TABLE t_jn2; DROP TABLE t_jn3; From c1e6eadcd9b2b5a7797d88d0d04552530af4df5a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 10:51:25 +0000 Subject: [PATCH 21/86] Backport #111049 to 26.6: Fix async-insert deduplication token bleed across partitions --- src/Interpreters/InsertDeduplication.cpp | 32 +++++++++ src/Interpreters/InsertDeduplication.h | 3 + .../MergeTree/MergeTreeDataWriter.cpp | 11 +++- src/Storages/MergeTree/MergeTreeDataWriter.h | 10 ++- src/Storages/MergeTree/MergeTreeSink.cpp | 11 +++- .../MergeTree/ReplicatedMergeTreeSink.cpp | 11 +++- ...sert_dedup_token_partition_bleed.reference | 3 + ...sync_insert_dedup_token_partition_bleed.sh | 66 +++++++++++++++++++ 8 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.reference create mode 100755 tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.sh diff --git a/src/Interpreters/InsertDeduplication.cpp b/src/Interpreters/InsertDeduplication.cpp index 96bae6cddc7f..342653274469 100644 --- a/src/Interpreters/InsertDeduplication.cpp +++ b/src/Interpreters/InsertDeduplication.cpp @@ -151,6 +151,38 @@ DeduplicationInfo::FilterResult DeduplicationInfo::deduplicateSelf(bool deduplic } +DeduplicationInfo::Ptr DeduplicationInfo::filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index) const +{ + /// An empty selector means the block was not split (single partition); with dedup off or a + /// single token there is nothing to attribute. Every token then belongs to this partition. + if (disabled || row_to_partition.empty() || getCount() <= 1) + return cloneSelf(); + + /// Keep only tokens that have at least one row in this partition. + std::set absent_offsets; + for (size_t i = 0; i < offsets.size(); ++i) + { + bool present = false; + for (size_t row = getTokenBegin(i); row < getTokenEnd(i); ++row) + { + if (row_to_partition[row] == partition_index) + { + present = true; + break; + } + } + if (!present) + absent_offsets.insert(i); + } + + if (absent_offsets.empty()) + return cloneSelf(); + + /// filterImpl drops the absent tokens and keeps the block/offsets consistent. + return filterImpl(absent_offsets).deduplication_info; +} + + DeduplicationInfo::FilterResult DeduplicationInfo::recalculateBlock(DeduplicationInfo::FilterResult && filtered, const std::string & partition_id, ContextPtr context) const { if (filtered.removed_rows == 0) diff --git a/src/Interpreters/InsertDeduplication.h b/src/Interpreters/InsertDeduplication.h index 26cb79464cee..0bf5196de708 100644 --- a/src/Interpreters/InsertDeduplication.h +++ b/src/Interpreters/InsertDeduplication.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -95,6 +96,8 @@ class DeduplicationInfo : public ChunkInfo FilterResult deduplicateSelf(bool deduplication_enabled, const std::string & partition_id, ContextPtr context) const; FilterResult deduplicateBlock(const std::vector & existing_block_ids, const std::string & partition_id, ContextPtr context) const; + Ptr filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index) const; + std::vector getDeduplicationHashes(const std::string & partition_id, bool deduplication_enabled) const; size_t getCount() const; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 9a2d8d75cadd..5a91d5dd6338 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -475,8 +475,13 @@ void MergeTreeTemporaryPart::prewarmCaches() } BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts( - Block && block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) + Block && block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, IColumn::Selector * out_selector) { + /// out_selector is left empty when the block is not split (a single resulting partition); + /// the caller then knows every row belongs to the only partition. + if (out_selector) + out_selector->clear(); + BlocksWithPartition result; if (block.empty() || !block.rows()) { @@ -546,6 +551,10 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts( for (auto & item : result) item.partition_id = item.partition.getID(metadata_snapshot->getPartitionKey().sample_block); + /// Hand the row -> partition-index mapping to the caller (deduplication uses it). + if (out_selector) + *out_selector = std::move(selector); + return result; } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index f452a0e80221..e48dbe3fa0f6 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -70,8 +70,16 @@ class MergeTreeDataWriter /** Split the block to blocks, each of them must be written as separate part. * (split rows by partition) * Works deterministically: if same block was passed, function will return same result in same order. + * When out_selector is set, it receives the row -> partition-index mapping (empty when the block + * is not split, i.e. a single resulting partition). Deduplication needs it to attribute each + * source row to the partition it landed in. */ - static BlocksWithPartition splitBlockIntoParts(Block && block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); + static BlocksWithPartition splitBlockIntoParts( + Block && block, + size_t max_parts, + const StorageMetadataPtr & metadata_snapshot, + ContextPtr context, + IColumn::Selector * out_selector = nullptr); /// This structure contains not completely written temporary part. /// Some writes may happen asynchronously, e.g. for blob storages. diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 318879678876..0bb700b7a8b2 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -103,7 +103,8 @@ void MergeTreeSink::consume(Chunk & chunk) auto block = getHeader().cloneWithColumns(chunk.getColumns()); auto deduplication_info = chunk.getChunkInfos().getSafe(); - auto part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context); + IColumn::Selector partition_selector; + auto part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context, &partition_selector); using DelayedPartitions = std::vector; DelayedPartitions partitions; @@ -115,12 +116,16 @@ void MergeTreeSink::consume(Chunk & chunk) std::vector all_partwriter_hashes; all_partwriter_hashes.reserve(part_blocks.size()); - for (auto & current_block : part_blocks) + for (size_t part_index = 0; part_index < part_blocks.size(); ++part_index) { + auto & current_block = part_blocks[part_index]; + ProfileEvents::Counters part_counters; auto partition_scope = std::make_unique(&part_counters); - auto current_deduplication_info = deduplication_info->cloneSelf(); + /// Keep only the tokens whose own rows landed in this partition, so a coalesced async + /// insert does not register a token in partitions it never wrote to. + auto current_deduplication_info = deduplication_info->filterToPartition(partition_selector, part_index); { ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 8808f54f936b..9ca653027115 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -305,7 +305,8 @@ void ReplicatedMergeTreeSink::consume(Chunk & chunk) auto deduplication_info = chunk.getChunkInfos().getSafe(); - BlocksWithPartition part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context); + IColumn::Selector partition_selector; + BlocksWithPartition part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context, &partition_selector); decltype(delayed_parts) current_parts; @@ -314,14 +315,18 @@ void ReplicatedMergeTreeSink::consume(Chunk & chunk) std::vector all_partitions_block_ids; - for (auto & current_block : part_blocks) + for (size_t part_index = 0; part_index < part_blocks.size(); ++part_index) { + auto & current_block = part_blocks[part_index]; + Stopwatch watch; ProfileEvents::Counters part_counters; auto profile_events_scope = std::make_unique(&part_counters); - auto current_deduplication_info = deduplication_info->cloneSelf(); + /// Keep only the tokens whose own rows landed in this partition, so a coalesced async + /// insert does not register a token in partitions it never wrote to. + auto current_deduplication_info = deduplication_info->filterToPartition(partition_selector, part_index); { ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); diff --git a/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.reference b/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.reference new file mode 100644 index 000000000000..c7bccb76b6ab --- /dev/null +++ b/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.reference @@ -0,0 +1,3 @@ +MergeTree [(0,10),(1,20),(1,30)] 3 +ReplicatedMergeTree [(0,10),(1,20),(1,30)] 3 +single-entry-multi-partition [(0,1),(1,2)] 2 diff --git a/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.sh b/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.sh new file mode 100755 index 000000000000..5e14747d13ae --- /dev/null +++ b/tests/queries/0_stateless/03662_async_insert_dedup_token_partition_bleed.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the async insert queue to coalesce several tokens into one flush. + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/111031 +# When several async-insert entries with distinct insert_deduplication_token values are +# coalesced into one flush, each token used to be registered in the dedup log of EVERY +# partition the flush touched, not only the partition its own rows landed in. A later +# legitimately-distinct insert reusing one of those tokens in a partition it never wrote +# to was then silently deduplicated away (silent data loss). Affects MergeTree and +# ReplicatedMergeTree (shared sink split logic). + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Common async settings: long busy timeout so nothing auto-fires; one explicit flush +# coalesces the queued entries into a single batch. insert_deduplication_token is excluded +# from the async queue key, so entries with distinct tokens land in ONE flush. +async_insert=(--async_insert=1 --wait_for_async_insert=0 + --async_insert_busy_timeout_min_ms=600000 --async_insert_busy_timeout_max_ms=600000 + --async_insert_use_adaptive_busy_timeout=0 --insert_deduplicate=1) + +run_engine() { + local label=$1 engine=$2 table=$3 + + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS $table" + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $table (p UInt8, x UInt64) + ENGINE = $engine PARTITION BY p ORDER BY x + SETTINGS non_replicated_deduplication_window = 1000, deduplicate_merge_projection_mode = 'drop'" + + # Coalesce two entries: token A lands only in p=0, token B lands only in p=1. + $CLICKHOUSE_CLIENT "${async_insert[@]}" --insert_deduplication_token='A' -q "INSERT INTO $table VALUES (0, 10)" + $CLICKHOUSE_CLIENT "${async_insert[@]}" --insert_deduplication_token='B' -q "INSERT INTO $table VALUES (1, 20)" + $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE $table" + + # Token A was never used in p=1, so this row must be accepted (was dropped before the fix). + $CLICKHOUSE_CLIENT "${async_insert[@]}" --insert_deduplication_token='A' -q "INSERT INTO $table VALUES (1, 30)" + $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE $table" + + # Expected: [(0,10),(1,20),(1,30)]. Before the fix (1,30) was silently dropped -> count 2. + # enable_parallel_replicas=0: this check verifies inserted data, not the read path; parallel + # replicas need a matching cluster topology that this test does not set up. + $CLICKHOUSE_CLIENT -q "SELECT '$label', groupArray((p, x)), count() FROM (SELECT p, x FROM $table ORDER BY p, x) SETTINGS enable_parallel_replicas = 0" + + $CLICKHOUSE_CLIENT -q "DROP TABLE $table" +} + +run_engine 'MergeTree' 'MergeTree' 'bleed_mt' +run_engine 'ReplicatedMergeTree' "ReplicatedMergeTree('/clickhouse/tables/{database}/bleed_rmt', 'r1')" 'bleed_rmt' + +# A single async entry whose one token legitimately spans two partitions must still +# deduplicate on a full re-insert (the token belongs in both partitions it wrote to). +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS single_multi" +$CLICKHOUSE_CLIENT -q " +CREATE TABLE single_multi (p UInt8, x UInt64) +ENGINE = MergeTree PARTITION BY p ORDER BY x +SETTINGS non_replicated_deduplication_window = 1000" +$CLICKHOUSE_CLIENT "${async_insert[@]}" --insert_deduplication_token='T' -q "INSERT INTO single_multi VALUES (0, 1), (1, 2)" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE single_multi" +$CLICKHOUSE_CLIENT "${async_insert[@]}" --insert_deduplication_token='T' -q "INSERT INTO single_multi VALUES (0, 1), (1, 2)" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH ASYNC INSERT QUEUE single_multi" +# Expected: [(0,1),(1,2)] count 2 (fully deduplicated, not doubled). +$CLICKHOUSE_CLIENT -q "SELECT 'single-entry-multi-partition', groupArray((p, x)), count() FROM (SELECT p, x FROM single_multi ORDER BY p, x) SETTINGS enable_parallel_replicas = 0" +$CLICKHOUSE_CLIENT -q "DROP TABLE single_multi" From 7853871ad847621b249b1ca4b9133b2b3a439a9a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 18:52:06 +0000 Subject: [PATCH 22/86] =?UTF-8?q?Backport=20#111150=20to=2026.6:=20perf:?= =?UTF-8?q?=20pre-warm=20async=20insert=20dedup=20hashes=20before=20partit?= =?UTF-8?q?ion=20loop=20to=20fix=20O(P=C3=97N)=20recomputation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Interpreters/InsertDeduplication.cpp | 39 ++++++++++++- src/Interpreters/InsertDeduplication.h | 4 ++ src/Storages/MergeTree/MergeTreeSink.cpp | 17 ++++++ .../MergeTree/ReplicatedMergeTreeSink.cpp | 14 +++++ .../MergeTree/tests/gtest_async_inserts.cpp | 56 +++++++++++++++++++ ...ync_insert_dedup_multi_partition.reference | 11 ++++ ...603_async_insert_dedup_multi_partition.sql | 34 +++++++++++ ..._dedup_identical_multi_partition.reference | 3 + ...insert_dedup_identical_multi_partition.sql | 32 +++++++++++ 9 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.reference create mode 100644 tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.sql create mode 100644 tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.reference create mode 100644 tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.sql diff --git a/src/Interpreters/InsertDeduplication.cpp b/src/Interpreters/InsertDeduplication.cpp index 342653274469..ad04ece0e6e4 100644 --- a/src/Interpreters/InsertDeduplication.cpp +++ b/src/Interpreters/InsertDeduplication.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -208,12 +209,10 @@ std::set DeduplicationInfo::filterSelf(const String & partition_id) cons if (getCount() <= 1) return {}; - auto block_id_to_offsets = buildBlockIdToOffsetsMap(partition_id); - std::set fitered_offsets; /// fitered_offsets will contain all but first offsets for each block id /// so that only first occurrence of each block id will remain - for (auto & [_, block_offsets] : block_id_to_offsets) + for (const auto & [_, block_offsets] : buildOffsetsMapImpl(partition_id)) { if (block_offsets.size() > 1) fitered_offsets.insert(block_offsets.begin() + 1, block_offsets.end()); @@ -452,6 +451,26 @@ DeduplicationHash DeduplicationInfo::getBlockUnifiedHash(size_t offset, const st } +std::vector>> DeduplicationInfo::buildOffsetsMapImpl(const std::string & partition_id) const +{ + /// (hash, offset) pairs sorted by hash, then offset; runs of equal hashes are the groups. + std::vector> sorted; + sorted.reserve(offsets.size()); + for (size_t offset = 0; offset < offsets.size(); ++offset) + sorted.emplace_back(getBlockUnifiedHash(offset, partition_id).hash, offset); + std::sort(sorted.begin(), sorted.end()); + + std::vector>> result; + for (const auto & [hash, offset] : sorted) + { + if (result.empty() || result.back().first != hash) + result.emplace_back(hash, std::vector{}); + result.back().second.push_back(offset); + } + return result; +} + + DeduplicationHash DeduplicationInfo::getBlockHash(size_t offset, const std::string & partition_id) const { // if user token is empty we calculate by_data_hash @@ -567,6 +586,20 @@ std::vector DeduplicationInfo::getDeduplicationHashes(const s } +void DeduplicationInfo::prewarmDataHashes() const +{ + if (!original_block || !original_block->rows()) + return; + + for (size_t i = 0; i < tokens.size(); ++i) + { + if (!tokens[i].by_user.empty()) + continue; + calculateDataHashColumnWise(i, *original_block); + } +} + + size_t DeduplicationInfo::getCount() const { return offsets.size(); diff --git a/src/Interpreters/InsertDeduplication.h b/src/Interpreters/InsertDeduplication.h index 0bf5196de708..89c6ef290057 100644 --- a/src/Interpreters/InsertDeduplication.h +++ b/src/Interpreters/InsertDeduplication.h @@ -69,6 +69,7 @@ class DeduplicationInfo : public ChunkInfo /// src/Storages/MergeTree/tests/gtest_async_inserts.cpp friend std::vector testSelfDeduplicate(std::vector data, std::vector offsets, std::vector hashes); friend std::vector testSelfDeduplicateStrings(std::vector data, std::vector offsets, std::vector hashes); + friend std::vector testPrewarmDataHashes(std::vector data, std::vector offsets); public: using Ptr = std::shared_ptr; @@ -100,6 +101,8 @@ class DeduplicationInfo : public ChunkInfo std::vector getDeduplicationHashes(const std::string & partition_id, bool deduplication_enabled) const; + void prewarmDataHashes() const; + size_t getCount() const; size_t getRows() const; @@ -157,6 +160,7 @@ class DeduplicationInfo : public ChunkInfo size_t getTokenEnd(size_t pos) const; size_t getTokenRows(size_t pos) const; + std::vector>> buildOffsetsMapImpl(const std::string & partition_id) const; std::unordered_map> buildBlockIdToOffsetsMap(const std::string & partition_id) const; enum class Level diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 0bb700b7a8b2..5dfacc1b1a3c 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -116,6 +116,23 @@ void MergeTreeSink::consume(Chunk & chunk) std::vector all_partwriter_hashes; all_partwriter_hashes.reserve(part_blocks.size()); + if (deduplication_info && deduplicate && !deduplication_info->isDisabled()) + { + /// Preserve the pre-loop interrupt point that used to be the first checkTimeLimit() + /// inside the partition loop: a killed or timed-out insert should be noticed before + /// the full O(N) prewarm hash pass, not after it. + if (process_list_element) + process_list_element->checkTimeLimit(); + + /// Warm the data hashes once here: the per-partition infos produced by filterToPartition + /// below copy these tokens with their cached hash, so a token whose rows span several + /// partitions is hashed once instead of once per partition it landed in. + /// Time it under DuplicationElapsedMicroseconds like the per-partition dedup below, so + /// the profile event still reflects the total deduplication CPU. + ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); + deduplication_info->prewarmDataHashes(); + } + for (size_t part_index = 0; part_index < part_blocks.size(); ++part_index) { auto & current_block = part_blocks[part_index]; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 9ca653027115..5923efe68540 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -313,6 +313,20 @@ void ReplicatedMergeTreeSink::consume(Chunk & chunk) size_t total_streams = 0; bool support_parallel_write = false; + if (deduplication_info && deduplicate && !deduplication_info->isDisabled()) + { + /// A killed or timed-out insert should be noticed before the O(N) prewarm hash pass, + /// not only at the much later Keeper interaction; same interrupt point as in `MergeTreeSink`. + if (auto process_list_element = context->getProcessListElement()) + process_list_element->checkTimeLimit(); + + /// Warm the data hashes once here so the per-partition infos from filterToPartition below + /// reuse the cached token hash instead of rehashing a token that spans several partitions. + /// Time it under DuplicationElapsedMicroseconds like the per-partition dedup below. + ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); + deduplication_info->prewarmDataHashes(); + } + std::vector all_partitions_block_ids; for (size_t part_index = 0; part_index < part_blocks.size(); ++part_index) diff --git a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp index 90426d011ba2..b307e13d6dd5 100644 --- a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp +++ b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp @@ -136,4 +136,60 @@ TEST(AsyncInsertsTest, testSelfDeduplicateStrings) test_impl({"ab","c","a","bc"},{2,4},{"",""},{"ab","c","a","bc"}); } + +/// Verify that cloneSelf after prewarmDataHashes produces the same deduplication result as +/// operating on the original. This simulates the partition sink loop: the original +/// DeduplicationInfo is pre-warmed once, then cloned once per partition; each clone must +/// inherit the cached data_hash_batch and produce correct results without recomputing hashes. +std::vector testPrewarmDataHashes(std::vector data, std::vector offsets) +{ + MutableColumnPtr column = DataTypeString().createColumn(); + for (const auto & datum : data) + column->insert(datum); + Block block({ColumnWithTypeAndName(std::move(column), std::make_shared(), "a")}); + + auto deduplication_info = DeduplicationInfo::create(true); + deduplication_info->setRootViewID({}); + deduplication_info->disabled = false; + deduplication_info->updateOriginalBlock(Chunk(block.getColumns(), block.rows()), std::make_shared(block.cloneEmpty())); + + /// Empty user token → data-hash path, which is exactly what prewarmDataHashes covers. + deduplication_info->setUserToken("", offsets[0]); + for (size_t i = 1; i < offsets.size(); ++i) + deduplication_info->setUserToken("", offsets[i] - offsets[i - 1]); + + /// Pre-warm on the original, then clone — mirroring what MergeTreeSink / ReplicatedMergeTreeSink do. + deduplication_info->prewarmDataHashes(); + auto clone = deduplication_info->cloneSelf(); + + auto filtered = clone->filterImpl(clone->filterSelf("all")); + + if (filtered.removed_rows == 0 || !filtered.filtered_block) + return data; + + ColumnPtr col = filtered.filtered_block->getColumns()[0]; + std::vector result; + result.reserve(col->size()); + for (size_t i = 0; i < col->size(); i++) + result.push_back(String(col->getDataAt(i))); + return result; +} + +TEST(AsyncInsertsTest, testPrewarmDataHashes) +{ + auto test_impl = [](std::vector data, std::vector offsets, std::vector answer) + { + auto result = testPrewarmDataHashes(data, offsets); + ASSERT_EQ(answer, result); + }; + /// Two equal single-row blocks: prewarm + clone must still deduplicate correctly. + test_impl({"one line","one line"}, {1,2}, {"one line"}); + /// Equal multi-row blocks: first occurrence kept after clone. + test_impl({"a","bb","a","bb","ccc"}, {2,4,5}, {"a","bb","ccc"}); + /// Distinct blocks: no false deduplication through the pre-warmed clone. + test_impl({"ab","c","a","bc"}, {2,4}, {"ab","c","a","bc"}); + /// Three identical single-row blocks: only the first survives. + test_impl({"x","x","x"}, {1,2,3}, {"x"}); +} + } diff --git a/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.reference b/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.reference new file mode 100644 index 000000000000..2b4155104671 --- /dev/null +++ b/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.reference @@ -0,0 +1,11 @@ +4 +4 +8 +0 A +0 E +1 B +1 F +2 C +2 G +3 D +3 H diff --git a/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.sql b/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.sql new file mode 100644 index 000000000000..bd07c6fd0255 --- /dev/null +++ b/tests/queries/0_stateless/04603_async_insert_dedup_multi_partition.sql @@ -0,0 +1,34 @@ +-- Async insert deduplication must work correctly when an INSERT spans multiple partitions. +-- The deduplication hash for each token is computed once (via prewarmDataHashes) and then +-- inherited by per-partition clones, so this test verifies both correctness and that the +-- optimisation does not introduce any hash-collision false positives or false negatives. + +DROP TABLE IF EXISTS t; + +CREATE TABLE t (key Int64, value String) +ENGINE = MergeTree +PARTITION BY key % 4 +ORDER BY tuple() +SETTINGS non_replicated_deduplication_window = 100; + +-- First insert: 4 rows spread across 4 partitions. +INSERT INTO t SETTINGS async_insert = 1, wait_for_async_insert = 1, async_insert_deduplicate = 1 +VALUES (0,'A'),(1,'B'),(2,'C'),(3,'D'); + +SELECT count() FROM t; + +-- Identical insert: must be fully deduplicated, count stays at 4. +INSERT INTO t SETTINGS async_insert = 1, wait_for_async_insert = 1, async_insert_deduplicate = 1 +VALUES (0,'A'),(1,'B'),(2,'C'),(3,'D'); + +SELECT count() FROM t; + +-- Different data: must NOT be deduplicated, count reaches 8. +INSERT INTO t SETTINGS async_insert = 1, wait_for_async_insert = 1, async_insert_deduplicate = 1 +VALUES (0,'E'),(1,'F'),(2,'G'),(3,'H'); + +SELECT count() FROM t; + +SELECT * FROM t ORDER BY key, value; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.reference b/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.reference new file mode 100644 index 000000000000..0c47cd9b1d10 --- /dev/null +++ b/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.reference @@ -0,0 +1,3 @@ +0 0 1 +1 1 2 +total 2 misplaced 0 diff --git a/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.sql b/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.sql new file mode 100644 index 000000000000..3355ae4a64b5 --- /dev/null +++ b/tests/queries/0_stateless/04614_async_insert_dedup_identical_multi_partition.sql @@ -0,0 +1,32 @@ +-- Tags: no-fasttest +-- no-fasttest: needs the async insert queue to coalesce several entries into one flush. + +-- Two identical async-insert entries (no user token, data-hash path), each spanning two +-- partitions, coalesced into one flush. Self-deduplication must drop the second token, and +-- the per-partition block rewrite must keep only the current partition's rows: the +-- deduplication-retry chain ends with `SelectPartitionTransform`, so a row of another +-- partition must never leak into this partition's part. + +DROP TABLE IF EXISTS dedup_identical_multi_partition; + +CREATE TABLE dedup_identical_multi_partition (p UInt8, x UInt64) +ENGINE = MergeTree PARTITION BY p ORDER BY x +SETTINGS non_replicated_deduplication_window = 1000; + +-- First flush: two identical entries, self-deduplicated within the flush. +INSERT INTO dedup_identical_multi_partition SETTINGS async_insert = 1, wait_for_async_insert = 0, async_insert_busy_timeout_min_ms = 600000, async_insert_busy_timeout_max_ms = 600000, async_insert_use_adaptive_busy_timeout = 0, deduplicate_insert = 'enable' VALUES (0, 1), (1, 2); +INSERT INTO dedup_identical_multi_partition SETTINGS async_insert = 1, wait_for_async_insert = 0, async_insert_busy_timeout_min_ms = 600000, async_insert_busy_timeout_max_ms = 600000, async_insert_use_adaptive_busy_timeout = 0, deduplicate_insert = 'enable' VALUES (0, 1), (1, 2); + +SYSTEM FLUSH ASYNC INSERT QUEUE dedup_identical_multi_partition; + +-- Second flush with the same pair: must fully deduplicate against the deduplication log. +INSERT INTO dedup_identical_multi_partition SETTINGS async_insert = 1, wait_for_async_insert = 0, async_insert_busy_timeout_min_ms = 600000, async_insert_busy_timeout_max_ms = 600000, async_insert_use_adaptive_busy_timeout = 0, deduplicate_insert = 'enable' VALUES (0, 1), (1, 2); +INSERT INTO dedup_identical_multi_partition SETTINGS async_insert = 1, wait_for_async_insert = 0, async_insert_busy_timeout_min_ms = 600000, async_insert_busy_timeout_max_ms = 600000, async_insert_use_adaptive_busy_timeout = 0, deduplicate_insert = 'enable' VALUES (0, 1), (1, 2); + +SYSTEM FLUSH ASYNC INSERT QUEUE dedup_identical_multi_partition; + +-- Every row must sit in the part of its own partition. +SELECT _partition_id, p, x FROM dedup_identical_multi_partition ORDER BY ALL; +SELECT 'total', count(), 'misplaced', countIf(_partition_id != toString(p)) FROM dedup_identical_multi_partition; + +DROP TABLE dedup_identical_multi_partition; From 2a0ff76c556f0b2e0a131fd6481ef5f278c4e0b9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 20:48:31 +0000 Subject: [PATCH 23/86] Backport #111362 to 26.6: Guard StorageJoin in ANY-to-SEMI/ANTI join conversion --- .../convertAnyJoinToSemiOrAntiJoin.cpp | 11 ++++- ..._join_to_semi_storage_join_guard.reference | 3 ++ ...30_any_join_to_semi_storage_join_guard.sql | 49 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.reference create mode 100644 tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.sql diff --git a/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp b/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp index ebb30ea7e54c..95621be8f228 100644 --- a/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp @@ -130,7 +130,16 @@ size_t tryConvertAnyJoinToSemiOrAntiJoin(QueryPlan::Node * parent_node, QueryPla QueryPlan::Node * child_node = parent_node->children.front(); auto & child = child_node->step; auto * join = typeid_cast(child.get()); - if (!join) + if (!join || child_node->children.size() != 2) + return 0; + + /// The Join engine requires its declared join kind and strictness to remain unchanged. + auto isStorageJoin = [](auto & step) + { + auto * lookup_step = typeid_cast(step.get()); + return lookup_step && lookup_step->getPreparedJoinStorage().storage_join; + }; + if (isStorageJoin(child_node->children.back()->step)) return 0; auto & join_operator = join->getJoinOperator(); diff --git a/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.reference b/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.reference new file mode 100644 index 000000000000..16db301bb512 --- /dev/null +++ b/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.reference @@ -0,0 +1,3 @@ +1 +0 +1 diff --git a/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.sql b/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.sql new file mode 100644 index 000000000000..94d21133d7ec --- /dev/null +++ b/tests/queries/0_stateless/03630_any_join_to_semi_storage_join_guard.sql @@ -0,0 +1,49 @@ +-- Regression test for issue #103318: +-- ANY LEFT JOIN on a Join engine table with a WHERE filter on a right-side payload column +-- threw INCOMPATIBLE_TYPE_OF_JOIN, because the ANY -> SEMI/ANTI conversion mutated the +-- StorageJoin declared strictness, which StorageJoin::getJoinLocked rejects. + +SET enable_analyzer = 1; +SET query_plan_convert_any_join_to_semi_or_anti_join = 1; -- CI may inject False; pin so the conversion pass (whose StorageJoin guard is under test) always runs +SET join_use_nulls = 0; -- CI may inject True; the Join engine rejects a mismatched join_use_nulls, which is a different error unrelated to this test + +DROP TABLE IF EXISTS storage_join_103318; +CREATE TABLE storage_join_103318 (id UInt64, val String) ENGINE = Join(ANY, LEFT, id); +INSERT INTO storage_join_103318 VALUES (1, 'x'); + +-- The query used to throw INCOMPATIBLE_TYPE_OF_JOIN. It must now return a result. +SELECT count() +FROM (SELECT 1 :: UInt64 AS id) AS t +ANY LEFT JOIN storage_join_103318 AS j USING (id) +WHERE j.val != ''; + +-- The conversion must be declined for a StorageJoin: the join stays ANY (no SEMI). +SELECT count() +FROM +( + EXPLAIN actions = 1 + SELECT count() + FROM (SELECT 1 :: UInt64 AS id) AS t + ANY LEFT JOIN storage_join_103318 AS j USING (id) + WHERE j.val != '' +) +WHERE explain ILIKE '%Strictness: semi%'; + +-- The conversion must still fire for a non-StorageJoin right child (guard must not over-fire): +-- the same shape over a MergeTree table is rewritten to a SEMI join. +DROP TABLE IF EXISTS mt_103318; +CREATE TABLE mt_103318 (id UInt64, val String) ENGINE = MergeTree ORDER BY id; +INSERT INTO mt_103318 VALUES (1, 'x'); +SELECT count() +FROM +( + EXPLAIN actions = 1 + SELECT count() + FROM (SELECT 1 :: UInt64 AS id) AS t + ANY LEFT JOIN mt_103318 AS j USING (id) + WHERE j.val != '' +) +WHERE explain ILIKE '%Strictness: semi%'; + +DROP TABLE storage_join_103318; +DROP TABLE mt_103318; From d39a5069884f9b3ab84028367be2f9bd781fd760 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 22 Jul 2026 21:30:31 +0000 Subject: [PATCH 24/86] Backport #110006 to 26.6: Support index pruning for IS NOT DISTINCT FROM and IS TRUE --- src/Storages/MergeTree/KeyCondition.cpp | 354 ++++++++++++++++-- ...rimary_key_is_true_false_unknown.reference | 11 +- ...4304_primary_key_is_true_false_unknown.sql | 41 +- ...y_condition_is_not_distinct_from.reference | 153 ++++++++ ...401_key_condition_is_not_distinct_from.sql | 334 +++++++++++++++++ 5 files changed, 863 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.reference create mode 100644 tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.sql diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 6ada748fa728..d2c7e41ba87d 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -386,6 +386,17 @@ const KeyCondition::AtomMap KeyCondition::atom_map return true; } }, + { + /// For a non-NULL constant `c`, `key <=> c` matches the same rows as `key = c`. + /// The NULL-constant case (`<=>` meaning "is NULL") is rejected earlier, before the atom is built. + "isNotDistinctFrom", + [] (RPNElement & out, const Field & value) + { + out.function = RPNElement::FUNCTION_IN_RANGE; + out.range = Range(value); + return true; + } + }, { "less", [] (RPNElement & out, const Field & value) @@ -674,6 +685,24 @@ const KeyCondition::AtomMap KeyCondition::atom_map static const std::set always_relaxed_atom_elements = {KeyCondition::RPNElement::FUNCTION_UNKNOWN, KeyCondition::RPNElement::FUNCTION_ARGS_IN_HYPERRECTANGLE, KeyCondition::RPNElement::FUNCTION_POINT_IN_POLYGON}; +/// The `isNull`/`isNotNull` atoms (and the `key IS NOT DISTINCT FROM NULL` branch that reuses them) +/// deliberately ignore the key monotonic-functions chain ("nulls are kept"): they narrow a Nullable +/// index to the NULL granule as if the wrapper were absent. That is only sound for a BARE key. A +/// monotonic wrapper can change which rows are NULL or whether the predicate is even defined: +/// - `ifNull(k, 0)` / `coalesce(k, 0)` / `assumeNotNull(k)` map NULL to a non-NULL value, so +/// `isNull(wrapper(k))` is actually always false; +/// - `CAST(k, 'UInt32')` throws on a NULL row; +/// - `toDateTime(k)` on a `Date32` key or `intDiv(k, c)` throw on out-of-range / illegal non-NULL +/// values, so the predicate should raise on some granules rather than be pruned. +/// Reusing the bare `isNull` atom in any of these cases would match the NULL granule and mark it +/// exact-true, so exact-count / implicit-projection paths return wrong results. We therefore reuse the +/// null atom only when the key is not wrapped at all; a wrapped key falls back to a full scan, which is +/// always correct. +static bool monotonicChainSupportsNullAtom(const KeyCondition::MonotonicFunctionsChain & chain) +{ + return chain.empty(); +} + /// Functions with range inversion cannot be relaxed. It will become stricter instead. /// For example: /// create table test(d Date, k Int64, s String) Engine=MergeTree order by toYYYYMM(d); @@ -723,6 +752,7 @@ static std::string_view reverseComparisonOperator(std::string_view op) { if (op == "equals") return "equals"; if (op == "notEquals") return "notEquals"; + if (op == "isNotDistinctFrom") return "isNotDistinctFrom"; if (op == "less") return "greater"; if (op == "greater") return "less"; if (op == "lessOrEquals") return "greaterOrEquals"; @@ -875,6 +905,14 @@ static const ActionsDAG::Node * tryRewriteCoalesceComparison( const ActionsDAG::Node * const_node = node.children[c0 ? 0 : 1]; const std::string_view canonical_op = c0 ? mirrored : std::string_view{op_name}; + /// A NULL constant only reaches here through `isNotDistinctFrom` (`= NULL` is folded away). + /// `coalesce(k, 0) <=> NULL` is always false because `coalesce(k, 0)` is never NULL, but the + /// branch decomposition below would emit `(y_0 <=> NULL) OR ...` = `isNull(k) OR ...`, wrongly + /// narrowing a Nullable index to the NULL granule. Decline so `extractAtomFromTree` handles it + /// (its `isNotDistinctFrom` NULL branch declines NULL-erasing wrappers, giving a full scan). + if (const_node->column && const_node->column->isNullAt(0)) + return nullptr; + if (coalesce_node->type != ActionsDAG::ActionType::FUNCTION) return nullptr; @@ -975,6 +1013,36 @@ static const ActionsDAG::Node * tryRewriteCoalesceComparison( return &inverted_dag.addFunction(or_func, std::move(or_children), ""); } +/// True if `node` is a two-argument `ifNull(X, 0)` / `coalesce(X, 0)` with a falsy numeric-zero +/// constant fallback - the exact shape `tryRewriteCoalesceCondition` peels to its inner predicate `X` +/// in boolean context. Shared with `predicateIsBooleanResult` so the boolean-result gate and the +/// actual peel stay in sync. +static bool isFalsyZeroCoalesceCondition(const ActionsDAG::Node & node) +{ + if (node.type != ActionsDAG::ActionType::FUNCTION) + return false; + + const auto & name = node.function_base->getName(); + if (name != "coalesce" && name != "ifNull") + return false; + + if (node.children.size() != 2) + return false; + + const ActionsDAG::Node * fallback = node.children[1]; + if (fallback->type != ActionsDAG::ActionType::COLUMN || !fallback->column || !isColumnConst(*fallback->column)) + return false; + + const Field fallback_value = (*fallback->column)[0]; + switch (fallback_value.getType()) + { + case Field::Types::UInt64: return fallback_value.safeGet() == 0; + case Field::Types::Int64: return fallback_value.safeGet() == 0; + case Field::Types::Float64: return fallback_value.safeGet() == 0.0; + default: return false; + } +} + /// Rewrite an `ifNull(X, 0)` / `coalesce(X, 0)` used as a condition to `X` for key analysis, so the wrapped /// predicate becomes a prunable key atom. `ifNull(X, 0)` is truthy exactly when `X` is truthy, for any /// `X`, so no whitelist of inner functions is needed; but its value differs from `X` on NULL rows, so @@ -992,34 +1060,220 @@ static const ActionsDAG::Node * tryRewriteCoalesceCondition( if (name != "coalesce" && name != "ifNull") return nullptr; + if (!isFalsyZeroCoalesceCondition(node)) + return nullptr; + + /// The unwrapped predicate replaces the boolean wrapper, so it stays in boolean context. + return &cloneDAGWithInversionPushDown(*node.children[0], inverted_dag, inputs_mapping, context, false, /* boolean_context */ true); +} + +/// Boolean-valued functions (result in {0, 1, NULL}) that are NOT `atom_map` atoms: the logical +/// connectives (handled structurally by `cloneDAGWithInversionPushDown`) and boolean comparisons +/// `KeyCondition` does not prune (`isDistinctFrom`, `ilike`, `notILike`). The prunable boolean atoms +/// are taken directly from `atom_map` (see `predicateIsBooleanResult`), so this only holds the extras. +static const std::unordered_set extra_boolean_result_functions +{ + "not", "and", "or", "isDistinctFrom", "ilike", "notILike", +}; + +/// A positive boolean wrapper `wrapper(X, ...)` is truth-equivalent to bare `X` ONLY IF `X` is +/// boolean-valued (in {0, 1, NULL}). Otherwise, e.g. for `k` UInt32, `k <=> true` / `k != false` / +/// `k IN (true)` mean `k = 1` / `k != 0` / `k = 1`, NOT "k is truthy", so peeling the wrapper would +/// be wrong. This checks the boolean-result-ness of the predicate after peeling the non-semantic +/// wrappers that `cloneDAGWithInversionPushDown` strips transparently (alias, `materialize`, trivial +/// `CAST`). Peeling here keeps equivalent wrapped forms (`CAST(k = 42, 'UInt8') IS TRUE`, +/// `materialize(k = 42) IS TRUE`) from diverging: otherwise the wrapped predicate reaches the gate as +/// `CAST` / `materialize` (not boolean-valued), the rewrite declines, and the later clone strips the +/// wrapper anyway, leaving the un-prunable `isNotDistinctFrom(equals(k, 42), true)` in the DAG. The +/// caller still clones the ORIGINAL `predicate`, so the recursion strips the same wrappers under +/// boolean context. +/// +/// The allowlist is derived from `KeyCondition::atom_map` (the single source of truth for the atoms +/// `KeyCondition` can actually prune: comparisons, `in`/`notIn`, `has`, `empty`/`notEmpty`, `like`, +/// `startsWith`/`startsWithUTF8`, `match`, `isNull`/`isNotNull`, `pointInPolygon`), plus the boolean +/// connectives and boolean comparisons that are not atoms (`extra_boolean_result_functions`). This +/// way `startsWith(s, 'ab') IS TRUE` and `has([1, 10], id) IS TRUE` peel to the prunable atom instead +/// of being left behind. +/// +/// When `allow_coalesce_rewrite` is set, an inner falsy-zero `ifNull(Y, 0)` / `coalesce(Y, 0)` is +/// itself boolean-valued exactly when `Y` is (`ifNull(Y, 0)` is in {0, 1, NULL} iff `Y` is), so the +/// gate recurses into `Y`. This composes the outer positive-boolean-wrapper peel with the existing +/// `tryRewriteCoalesceCondition`: `ifNull(k = 42, 0) IS TRUE` passes the gate, the outer peel enters +/// boolean context, and the recursion in `cloneDAGWithInversionPushDown` then unwraps the `ifNull` +/// to the prunable `k = 42`. Without this the gate rejects `ifNull` (not an atom / extra), the outer +/// peel declines, and the fallback clone with `boolean_context = false` denies the coalesce rewrite +/// its chance too, leaving `isNotDistinctFrom(ifNull(equals(k, 42), 0), true)` at `Condition: true`. +static bool predicateIsBooleanResult(const ActionsDAG::Node * predicate, bool allow_coalesce_rewrite) +{ + const ActionsDAG::Node * unwrapped = predicate; + while (unwrapped->type == ActionsDAG::ActionType::ALIAS + || (unwrapped->type == ActionsDAG::ActionType::FUNCTION + && (unwrapped->function_base->getName() == "materialize" || isTrivialCast(*unwrapped)))) + { + if (unwrapped->children.empty()) + return false; + unwrapped = unwrapped->children.front(); + } + + if (unwrapped->type != ActionsDAG::ActionType::FUNCTION) + return false; + + /// `ifNull(Y, 0)` / `coalesce(Y, 0)` is boolean-valued iff `Y` is - recurse so the outer peel + /// composes with the coalesce rewrite (only when that rewrite is enabled). + if (allow_coalesce_rewrite && isFalsyZeroCoalesceCondition(*unwrapped)) + return predicateIsBooleanResult(unwrapped->children[0], allow_coalesce_rewrite); + + const auto & unwrapped_name = unwrapped->function_base->getName(); + return KeyCondition::atom_map.contains(unwrapped_name) + || extra_boolean_result_functions.contains(unwrapped_name); +} + +/// Rewrite a positive boolean wrapper around a predicate `X` to bare `X` for key analysis, so a +/// wrapped predicate like `(k = 42) IS TRUE` or `(k = 42) != false` becomes a prunable key atom on +/// `k`. Two truth-equivalent wrapper forms the analyzer produces are handled: +/// - `X IS TRUE`, lowered to `isNotDistinctFrom(X, true)` (const `true` == numeric 1) +/// - `X != false`, i.e. `notEquals(X, false)` (const `false` == numeric 0) +/// Both are truth-equivalent to `X` ONLY IF `X` is boolean-valued (see `predicateIsBooleanResult`): +/// for such `X`, `X <=> true` equals `X` on non-NULL values and is `false` (not NULL) on NULL, and +/// `X != false` equals `X` on all values including NULL; in a truth-tested position both `false` and +/// `NULL` reject the row, matching bare `X`. Like `tryRewriteCoalesceCondition`, it changes the value +/// on NULL rows (for the `IS TRUE` form), so the caller restricts it to non-inverted boolean position. +/// Returns nullptr if the pattern does not match. +static const ActionsDAG::Node * tryRewriteIsTrueCondition( + const ActionsDAG::Node & node, + const String & name, + ActionsDAG & inverted_dag, + std::unordered_map & inputs_mapping, + const ContextPtr & context) +{ + /// `X IS TRUE` -> `X <=> true` (const 1); `X != false` -> `X != 0` (const 0). + UInt64 expected_const = 0; + if (name == "isNotDistinctFrom") + expected_const = 1; + else if (name == "notEquals") + expected_const = 0; + else + return nullptr; + + if (node.children.size() != 2) + return nullptr; + + auto is_const = [](const ActionsDAG::Node & n) + { + return n.type == ActionsDAG::ActionType::COLUMN && n.column && isColumnConst(*n.column); + }; + + /// Find the `X const` shape. `isNotDistinctFrom` and `notEquals` are both symmetric, so the + /// constant may be on either side. + const bool c0 = is_const(*node.children[0]); + const bool c1 = is_const(*node.children[1]); + if (c0 == c1) + return nullptr; + + const ActionsDAG::Node * predicate = node.children[c0 ? 1 : 0]; + const ActionsDAG::Node * const_node = node.children[c0 ? 0 : 1]; + + /// The constant must be exactly `true` (numeric 1) for `<=>` or `false` (numeric 0) for `!=`. + /// `X IS FALSE` (`X <=> false`) and `X != true` (`X != 1`) are NOT truth-equivalent to `X`, so + /// they are not rewritten here. + const Field const_value = (*const_node->column)[0]; + if (const_value.getType() != Field::Types::UInt64 || const_value.safeGet() != expected_const) + return nullptr; + + if (!predicateIsBooleanResult(predicate, context->getSettingsRef()[Setting::allow_key_condition_coalesce_rewrite])) + return nullptr; + + /// The unwrapped predicate replaces the boolean wrapper, so it stays in boolean context. + return &cloneDAGWithInversionPushDown(*predicate, inverted_dag, inputs_mapping, context, false, /* boolean_context */ true); +} + +/// Rewrite `X IN ()`, e.g. `(k = 42) IN (true)`, to bare `X` for key analysis, +/// so the inner `k = 42` becomes a prunable key atom. For a boolean-valued `X`, `X IN (true)` matches +/// exactly the rows where `X` is true, i.e. it is truth-equivalent to `X`. This is only sound when +/// EVERY element of the set is exactly `true` (numeric 1) and non-NULL: +/// - `X IN (false)` matches `NOT X` (declined), `X IN (true, false)` matches "X is 0 or 1" +/// (always-true for non-NULL boolean, declined), and a NULL element changes NULL handling. +/// - the `X` boolean-result gate is the same as `tryRewriteIsTrueCondition`: `k IN (true)` for a +/// non-boolean `k` means `k = 1`, not "k truthy". +/// Only literal/constant sets whose elements are available at analysis time are handled; subquery +/// sets that are not yet built decline (return nullptr) and fall back to the existing behavior. +/// Returns nullptr if the pattern does not match. +static const ActionsDAG::Node * tryRewriteInTruthyCondition( + const ActionsDAG::Node & node, + const String & name, + ActionsDAG & inverted_dag, + std::unordered_map & inputs_mapping, + const ContextPtr & context) +{ + /// Only the plain `in`; `notIn` is negation and `globalIn`/`nullIn` have different NULL semantics. + if (name != "in") + return nullptr; + if (node.children.size() != 2) return nullptr; const ActionsDAG::Node * predicate = node.children[0]; - const ActionsDAG::Node * fallback = node.children[1]; + const ActionsDAG::Node * set_node = node.children[1]; - if (fallback->type != ActionsDAG::ActionType::COLUMN || !fallback->column || !isColumnConst(*fallback->column)) + /// The right argument must be a constant column wrapping a prepared set. + if (set_node->type != ActionsDAG::ActionType::COLUMN || !set_node->column) return nullptr; - const Field fallback_value = (*fallback->column)[0]; - switch (fallback_value.getType()) + const auto * column_set = checkAndGetColumn(&set_node->column->getDataColumn()); + if (!column_set) + return nullptr; + + auto future_set = column_set->getData(); + if (!future_set) + return nullptr; + + /// Only single-column sets: `X` is a scalar predicate, so a tuple/multi-column set is not this shape. + if (future_set->getTypes().size() != 1) + return nullptr; + + /// Gate on the (cheap) boolean-result check of the left-hand side BEFORE materializing the set. + /// The rewrite only applies when `X` is boolean-valued, which is a property of the predicate alone + /// and independent of the set. Checking it first avoids the `O(set size)` ordered-set + /// materialization (`buildOrderedSetInplace` + `getSetElements`) for common large non-boolean + /// filters like `user_id IN (1, 2, ... huge literal list)`, which would otherwise be built and + /// iterated only to be discarded here. This mirrors the discipline in `tryPrepareSetIndexForIn` + /// where ordered-set materialization happens only for `IN` predicates usable for key analysis. + if (!predicateIsBooleanResult(predicate, context->getSettingsRef()[Setting::allow_key_condition_coalesce_rewrite])) + return nullptr; + + /// Only inspect a set that is ALREADY built. `get()` returns a non-null set for literal-tuple + /// (`IN (true)`) and storage sets, which are available at planning time, and nullptr for a + /// subquery set that has not run yet. We must NOT force-build here: this rewrite runs during + /// key-condition DAG cloning for every query, so forcing the set would execute the `IN` subquery + /// purely for analysis, e.g. `X IN (SELECT throwIf(1))` would throw even when no index is used + /// (see 02707_skip_index_with_in). buildOrderedSetInplace on an already-built set is then cheap + /// (no subquery) and just materializes its ordered elements. + if (!future_set->get()) + return nullptr; + + auto prepared_set = future_set->buildOrderedSetInplace(context); + if (!prepared_set || !prepared_set->hasExplicitSetElements()) + return nullptr; + + const Columns set_elements = prepared_set->getSetElements(); + if (set_elements.size() != 1) + return nullptr; + + const IColumn & elements = *set_elements.front(); + const size_t num_elements = elements.size(); + /// Empty set: `X IN ()` is always false, not equivalent to `X`. Decline. + if (num_elements == 0) + return nullptr; + + /// Every element must be exactly `true` (non-NULL numeric 1). + for (size_t i = 0; i < num_elements; ++i) { - case Field::Types::UInt64: - if (fallback_value.safeGet() != 0) - return nullptr; - break; - case Field::Types::Int64: - if (fallback_value.safeGet() != 0) - return nullptr; - break; - case Field::Types::Float64: - if (fallback_value.safeGet() != 0.0) - return nullptr; - break; - default: return nullptr; + const Field element = elements[i]; + if (element.getType() != Field::Types::UInt64 || element.safeGet() != 1) + return nullptr; } - /// The unwrapped predicate replaces the boolean wrapper, so it stays in boolean context. + /// The unwrapped predicate replaces the `IN` wrapper, so it stays in boolean context. return &cloneDAGWithInversionPushDown(*predicate, inverted_dag, inputs_mapping, context, false, /* boolean_context */ true); } @@ -1141,6 +1395,13 @@ static const ActionsDAG::Node & cloneDAGWithInversionPushDown( res = &inverted_dag.addFunction(function_builder, children, ""); handled_inversion = true; } + else if (!need_inversion + && boolean_context + && ((res = tryRewriteIsTrueCondition(node, name, inverted_dag, inputs_mapping, context)) != nullptr + || (res = tryRewriteInTruthyCondition(node, name, inverted_dag, inputs_mapping, context)) != nullptr)) + { + handled_inversion = true; + } else if (!need_inversion && boolean_context && context->getSettingsRef()[Setting::allow_key_condition_coalesce_rewrite] @@ -1479,8 +1740,22 @@ static FieldRef applyFunction(const FunctionBasePtr & func, const DataTypePtr & { /// When cache is missed, we calculate the whole column where the field comes from. This will avoid repeated calculation. ColumnsWithTypeAndName args{(*columns)[field.column_idx]}; - field.columns->emplace_back(ColumnWithTypeAndName {nullptr, func->getResultType(), result_name}); - (*columns)[result_idx].column = func->execute(args, (*columns)[result_idx].type, columns->front().column->size(), /* dry_run = */ false); + /// Strip outer `LowCardinality` from the argument column and type before executing, keeping the + /// cached result full too. A monotonic-function chain is built against the outer-LowCardinality + /// stripped key type (`applyFunctionChainToColumn` strips it the same way), so a specialized + /// wrapper such as the UInt8->Bool `CAST` does `checkAndGetColumn` on the raw + /// column and aborts with a bad cast on a `ColumnLowCardinality` (e.g. a `LowCardinality(Bool)` + /// key compared with a `LowCardinality` constant). `removeLowCardinality` / + /// `convertToFullColumnIfLowCardinality` are no-ops for non-LC inputs. + if (args[0].column && args[0].column->lowCardinality()) + { + args[0].column = args[0].column->convertToFullColumnIfLowCardinality(); + args[0].type = removeLowCardinality(args[0].type); + } + field.columns->emplace_back(ColumnWithTypeAndName {nullptr, removeLowCardinality(func->getResultType()), result_name}); + (*columns)[result_idx].column + = func->execute(args, (*columns)[result_idx].type, args.front().column->size(), /* dry_run = */ false) + ->convertToFullColumnIfLowCardinality(); } return {field.columns, field.row_idx, result_idx}; @@ -3511,6 +3786,13 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu /// empty/notEmpty produce a meaningful range only for String key columns. if ((func_name == "empty" || func_name == "notEmpty") && !isString(*key_expr_type)) return false; + + /// The `isNull`/`isNotNull` atoms ignore the monotonic-functions chain (nulls are kept), so + /// they are sound only for a bare key. A wrapped key (`isNull(ifNull(k, 0))`, + /// `isNull(toDateTime(date32_k))`, ...) would otherwise be analyzed like `isNull(k)` and + /// wrongly prune a granule the predicate does not cover; decline and fall back to a scan. + if ((func_name == "isNull" || func_name == "isNotNull") && !monotonicChainSupportsNullAtom(chain)) + return false; } else if (num_args == 2) { @@ -3564,6 +3846,36 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu /// If the const operand is null, the atom will be always false if (const_value.isNull()) { + /// `key <=> NULL` means "key IS NULL", not "key = NULL". Reuse the existing `isNull` + /// atom (same handling as bare `key IS NULL`) so a Nullable PK / minmax index prunes + /// to the NULL granule exactly, instead of declining and scanning every granule. + if (func_name == "isNotDistinctFrom") + { + size_t key_arg_pos = 1 - const_arg_pos; + auto key_arg = func.getArgumentAt(key_arg_pos); + if (!isKeyPossiblyWrappedByMonotonicFunctions( + key_arg, info, key_column_num, argument_num_of_space_filling_curve, key_expr_type, chain)) + return false; + + if (key_column_num == static_cast(-1)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "`key_column_num` wasn't initialized. It is a bug."); + + /// The `isNull` atom ignores the monotonic-functions chain (nulls are kept), so it is + /// sound only for a bare key. A wrapped key (`ifNull(k, 0) IS NOT DISTINCT FROM NULL` + /// is always false; `toDateTime(date32_k) IS NOT DISTINCT FROM NULL` may raise) would + /// otherwise be analyzed like `isNull(k)` and prune a granule the predicate does not + /// cover (wrong results); decline and fall back to a scan. + if (!monotonicChainSupportsNullAtom(chain)) + return false; + + out.key_columns.push_back(key_column_num); + out.monotonic_functions_chain = std::move(chain); + out.argument_num_of_space_filling_curve = argument_num_of_space_filling_curve; + + const auto atom_it = atom_map.find("isNull"); + return atom_it->second(out, const_value); + } + out.function = RPNElement::ALWAYS_FALSE; return true; } @@ -3632,7 +3944,7 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu { condition_is_relaxed = true; } - else if (func_name == "equals" || func_name == "notEquals") + else if (func_name == "equals" || func_name == "notEquals" || func_name == "isNotDistinctFrom") { bool is_injective = false; if (!canConstantBeWrappedByDeterministicFunctions( diff --git a/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.reference b/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.reference index 15471361be53..1c78a381f9f4 100644 --- a/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.reference +++ b/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.reference @@ -5,9 +5,16 @@ IS UNKNOWN 8 IS NOT TRUE 16 IS NOT FALSE 16 IS NOT UNKNOWN 16 -IS FALSE 3 3 +IS FALSE 1 3 IS NOT FALSE 3 3 IS NOT TRUE 3 3 IS NOT UNKNOWN 2 3 -IS TRUE 3 3 +IS TRUE 2 3 IS UNKNOWN 2 3 +k = 42 1 +(k = 42) IS TRUE 1 +CAST(k = 42, UInt8) IS TRUE 1 +materialize(k = 42) IS TRUE 1 +1 +1 +1 diff --git a/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.sql b/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.sql index 923b293327a0..9631b9341350 100644 --- a/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.sql +++ b/tests/queries/0_stateless/04304_primary_key_is_true_false_unknown.sql @@ -12,7 +12,10 @@ CREATE TABLE bool_pk ) ENGINE = MergeTree ORDER BY (b, id) -SETTINGS index_granularity = 8, allow_nullable_key = 1; +-- Pin the layout so the granule-count assertions below are stable: randomized +-- adaptive granularity (index_granularity_bytes) could split the table into a +-- different number of marks and break the hard-coded SelectedMarks counts. +SETTINGS index_granularity = 8, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, allow_nullable_key = 1; -- One part with three granules of 8 rows each, sorted by `b`: -- granule 0 = only `false`, granule 1 = only `true`, granule 2 = only NULL. @@ -37,12 +40,13 @@ SYSTEM FLUSH LOGS query_log; -- `SelectedMarks` counts the granules read after primary-key pruning; -- `SelectedMarksTotal` counts the granules considered before pruning. --- `KeyCondition` recognises `isNull` and `isNotNull` (the lowered forms of --- `IS UNKNOWN` and `IS NOT UNKNOWN`) and prunes one of the three granules --- for both forms. The other four predicates lower to `isNotDistinctFrom` / --- `isDistinctFrom`, which `KeyCondition` currently treats as the trivial --- `true` condition, so no granule is dropped and `SelectedMarks` equals --- `SelectedMarksTotal`. +-- `IS TRUE` / `IS FALSE` lower to `b <=> true` / `b <=> false` +-- (`isNotDistinctFrom` against a non-NULL constant), which `KeyCondition` maps +-- like `b = true` / `b = false` and prunes to the single matching granule. +-- `IS UNKNOWN` / `IS NOT UNKNOWN` lower to `isNull` / `isNotNull` and prune the +-- NULL granule. The `IS NOT TRUE` / `IS NOT FALSE` forms lower to `isDistinctFrom`, +-- which `KeyCondition` still treats as the trivial `true` condition, so no +-- granule is dropped and `SelectedMarks` equals `SelectedMarksTotal`. SELECT splitByString('04304 ', log_comment)[2] AS predicate, ProfileEvents['SelectedMarks'] AS granules_read, @@ -54,3 +58,26 @@ WHERE current_database = currentDatabase() ORDER BY predicate; DROP TABLE bool_pk; + +-- Wrapped `IS TRUE` forms must prune the same as the bare predicate. The analyzer lowers +-- `X IS TRUE` to `isNotDistinctFrom(X, true)`; when `X` is a non-semantic wrapper over a +-- boolean predicate (`materialize(k = 42)`, trivial `CAST(k = 42, 'UInt8')`), `KeyCondition` +-- peels the wrapper before the boolean-result gate so the inner `k = 42` becomes a key atom. +-- Assert the granule pruning is preserved (1 of 10 granules) via `EXPLAIN indexes = 1`. +DROP TABLE IF EXISTS int_pk; +CREATE TABLE int_pk (k UInt32) ENGINE = MergeTree ORDER BY k +SETTINGS index_granularity = 8, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, add_minmax_index_for_numeric_columns = 0; +INSERT INTO int_pk SELECT number FROM numbers(80) SETTINGS max_insert_threads = 1; +OPTIMIZE TABLE int_pk FINAL; + +SELECT 'k = 42', countIf(explain LIKE '%Granules: 1/10%') FROM (EXPLAIN indexes = 1 SELECT count() FROM int_pk WHERE k = 42); +SELECT '(k = 42) IS TRUE', countIf(explain LIKE '%Granules: 1/10%') FROM (EXPLAIN indexes = 1 SELECT count() FROM int_pk WHERE (k = 42) IS TRUE); +SELECT 'CAST(k = 42, UInt8) IS TRUE', countIf(explain LIKE '%Granules: 1/10%') FROM (EXPLAIN indexes = 1 SELECT count() FROM int_pk WHERE CAST(k = 42, 'UInt8') IS TRUE); +SELECT 'materialize(k = 42) IS TRUE', countIf(explain LIKE '%Granules: 1/10%') FROM (EXPLAIN indexes = 1 SELECT count() FROM int_pk WHERE materialize(k = 42) IS TRUE); + +-- Results must be unchanged by the pruning (one matching row each). +SELECT count() FROM int_pk WHERE (k = 42) IS TRUE SETTINGS optimize_trivial_count_query = 0; +SELECT count() FROM int_pk WHERE CAST(k = 42, 'UInt8') IS TRUE SETTINGS optimize_trivial_count_query = 0; +SELECT count() FROM int_pk WHERE materialize(k = 42) IS TRUE SETTINGS optimize_trivial_count_query = 0; + +DROP TABLE int_pk; diff --git a/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.reference b/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.reference new file mode 100644 index 000000000000..ad5d50c88d78 --- /dev/null +++ b/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.reference @@ -0,0 +1,153 @@ +--- IS NOT DISTINCT FROM prunes via primary key (non-Nullable key) --- +1 +--- (k = c) IS TRUE prunes via primary key (non-Nullable key) --- +1 +--- IS NOT DISTINCT FROM prunes via primary key (Nullable key) --- +1 +--- (k = c) IS TRUE prunes via primary key (Nullable key) --- +1 +--- const on the left side prunes too --- +1 +--- (k < c) IS TRUE prunes an ordered condition --- +1 +--- IS NOT DISTINCT FROM prunes via minmax skip index --- +1 +--- (v = c) IS TRUE prunes via minmax skip index --- +1 +--- b = true prunes partitions (baseline) --- +1 +--- b IS TRUE prunes partitions the same as b = true --- +1 +--- b IS NOT DISTINCT FROM true prunes partitions the same as b = true --- +1 +--- (k = c) != false prunes via primary key --- +1 +--- (v = c) != false prunes via minmax skip index --- +1 +--- b != false prunes partitions the same as b = true --- +1 +--- (k = c) != true does NOT get the IS TRUE pruning --- +0 +--- (k = c) IN (true) prunes via primary key --- +1 +--- (k = c) IN (true, true) prunes via primary key --- +1 +--- (v = c) IN (true) prunes via minmax skip index --- +1 +--- b IN (true) prunes partitions the same as b = true --- +1 +--- (k = c) IN (false) does NOT get the pruning --- +0 +--- (k = c) IN (true, false) does NOT get the pruning --- +0 +--- (k = c) NOT IN (true) does NOT get the pruning --- +0 +--- IS NOT DISTINCT FROM NULL prunes to the NULL granule like IS NULL --- +1 +--- NULL IS NOT DISTINCT FROM key prunes too (const on the left) --- +1 +--- bare IS NULL prunes to the NULL granule (reference for the above) --- +1 +--- ifNull(k, 0) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- ifNull(k, 0) IS NOT DISTINCT FROM NULL does NOT prune with coalesce rewrite off (0/) --- +0 +--- coalesce(k, 0) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- assumeNotNull(k) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- bare isNull(ifNull(k, 0)) does NOT prune to the NULL granule (0/) --- +0 +--- CAST(k, non-Nullable) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- bare isNull(CAST(k, non-Nullable)) does NOT prune to the NULL granule (0/) --- +0 +--- toUInt32(k) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- toUInt32(ifNull(k, 0)) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) --- +0 +--- ifNull(k, Nullable non-NULL fallback) IS NOT DISTINCT FROM NULL does NOT prune (0/) --- +0 +--- coalesce(k, Nullable non-NULL fallback) IS NOT DISTINCT FROM NULL does NOT prune (0/) --- +0 +--- isNull(toDateTime(d)) does NOT prune, keeps all granules --- +0 +--- toDateTime(d) IS NOT DISTINCT FROM NULL does NOT prune --- +0 +--- isNull(intDiv(k, 0)) does NOT prune, keeps all granules --- +0 +--- intDiv(k, 0) IS NOT DISTINCT FROM NULL does NOT prune --- +0 +--- isNull(intDiv(k, 2)) (also wrapped) does NOT prune --- +0 +--- startsWith(s, p) IS TRUE prunes via primary key like the bare atom --- +1 +--- bare startsWith(s, p) prunes via primary key (reference) --- +1 +--- startsWith(s, p) != false prunes via primary key like the bare atom --- +1 +--- startsWith(s, p) IN (true) prunes via primary key like the bare atom --- +1 +--- ifNull(k = 42, 0) IS TRUE prunes via primary key like the bare wrapped atom --- +1 +--- bare ifNull(k = 42, 0) prunes via primary key (reference) --- +1 +--- coalesce(k = 42, 0) IS TRUE prunes via primary key like the bare wrapped atom --- +1 +--- ifNull(k = 42, 0) != false prunes via primary key like the bare wrapped atom --- +1 +--- ifNull(k = 42, 0) IN (true) prunes via primary key like the bare wrapped atom --- +1 +--- ifNull(k = 42, 0) IS TRUE does NOT prune when allow_key_condition_coalesce_rewrite = 0 (0/) --- +0 +--- correctness is preserved across all forms --- +ndf_nonnull 1 +eq_nonnull 1 +istrue_eq 1 +istrue_lt 42 +ndf_nullable 1 +ndf_null_is_null 5000 +is_null 5000 +k_istrue_means_eq_1 1 +kplus1_istrue_means_eq_1 1 +isfalse 99999 +ne_false_eq 1 +ne_true_eq 99999 +in_true_eq 1 +in_truetrue_eq 1 +in_false_eq 99999 +in_truefalse_eq 100000 +notin_true_eq 99999 +k_in_true_means_eq_1 1 +ndf_null_left 5000 +ifnull_ndf_null 0 +coalesce_ndf_null 0 +assumenotnull_ndf_null 0 +ifnull_isnull 0 +ifnull_ndf_null_corw_off 0 +touint32_ndf_null 5000 +touint32_ifnull_ndf_null 0 +ifnull_nfb_ndf_null 0 +coalesce_nfb_ndf_null 0 +ifnull_nfb_ndf_null_iproj 0 +todatetime_ndf_null_ignore 100 +intdiv2_ndf_null 1 +ifnull_bare 1 +ifnull_istrue 1 +coalesce_istrue 1 +ifnull_ne_false 1 +ifnull_in_true 1 +startswith_bare 111 +startswith_istrue 111 +startswith_ne_false 111 +startswith_in_true 111 +part_eq 2 +part_istrue 2 +part_ndf 2 +part_ne_false 2 +part_in_true 2 +explain_in_subquery_not_built 1 +lc_less 8 +lc_eq 16 +lc_istrue 8 +lc_ndf 16 diff --git a/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.sql b/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.sql new file mode 100644 index 000000000000..4ba4aabac704 --- /dev/null +++ b/tests/queries/0_stateless/04401_key_condition_is_not_distinct_from.sql @@ -0,0 +1,334 @@ +-- Tags: no-replicated-database, no-parallel-replicas, no-random-merge-tree-settings +-- no-replicated-database: EXPLAIN output differs for replicated database. +-- no-parallel-replicas: EXPLAIN output differs for parallel replicas. +-- no-random-merge-tree-settings: the test asserts exact `Granules: N/M` counts, which depend on the +-- data layout; randomized MergeTree settings (index_granularity, use_const_adaptive_granularity, ...) +-- change the granule boundaries and flip the counts. +-- See `src/Storages/MergeTree/KeyCondition.cpp` (atom_map "isNotDistinctFrom", +-- reverseComparisonOperator, tryRewriteIsTrueCondition, tryRewriteInTruthyCondition). + +SET use_query_condition_cache = 0; +SET use_skip_indexes_on_data_read = 0; +-- The implicit `_exact_count_projection` hides the ReadFromMergeTree step (and its Granules line) +-- from `EXPLAIN indexes = 1 SELECT count() ...`; disable it so the granule counts are visible. +SET optimize_use_implicit_projections = 0; + +DROP TABLE IF EXISTS pk; +DROP TABLE IF EXISTS pk_null; +DROP TABLE IF EXISTS mm; +DROP TABLE IF EXISTS part; +DROP TABLE IF EXISTS spk; + +-- Pin index_granularity_bytes = 0 (non-adaptive) on every table whose exact `Granules: N/M` count is +-- asserted below, so granule boundaries depend only on the row-count `index_granularity` and not on the +-- CI-randomized byte cap (a small cap makes tiny granules, splitting the matched key range across more +-- than one granule and flipping the count). min_bytes_for_wide_part = 0 forces Wide so the non-adaptive +-- granularity does not log the "can't create parts with adaptive granularity" warning (Fast test fails +-- on any stderr). +CREATE TABLE pk (k UInt32) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, add_minmax_index_for_numeric_columns = 0; +INSERT INTO pk SELECT number FROM numbers(100000); + +CREATE TABLE pk_null (k Nullable(UInt32)) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, allow_nullable_key = 1, add_minmax_index_for_numeric_columns = 0; +INSERT INTO pk_null SELECT number FROM numbers(100000); +INSERT INTO pk_null SELECT NULL FROM numbers(5000); +-- The NULL rows arrive in a second part that sorts last; whether a background merge fires mid-test is +-- nondeterministic and flips whether the NULLs share a granule boundary with the non-NULL tail (1 vs 2 +-- granules for `k IS NULL` / `<=> NULL`). Merge to a single part up front so the count is stable. +OPTIMIZE TABLE pk_null FINAL; + +CREATE TABLE mm (id UInt32, v UInt32, INDEX v_idx v TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY id + SETTINGS index_granularity = 1024, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, add_minmax_index_for_numeric_columns = 0; +INSERT INTO mm SELECT number, number FROM numbers(100000); + +-- Partition pruning uses a separate KeyCondition path (PartitionPruner over the +-- partition value / minmax), not the primary-key granule path. The same rewrites +-- must apply there too. +CREATE TABLE part (b Bool, id UInt8) ENGINE = MergeTree PARTITION BY b ORDER BY id; +INSERT INTO part VALUES (false, 0), (false, 1), (true, 2), (true, 3); + +-- String primary key: the boolean-wrapper peel must reach the `startsWith` atom (a prunable boolean +-- atom that is NOT a comparison), not just `equals`/`less`. +CREATE TABLE spk (s String) ENGINE = MergeTree ORDER BY s + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, add_minmax_index_for_numeric_columns = 0; +INSERT INTO spk SELECT toString(number) FROM numbers(100000); + +SELECT '--- IS NOT DISTINCT FROM prunes via primary key (non-Nullable key) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE k IS NOT DISTINCT FROM 42) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k = c) IS TRUE prunes via primary key (non-Nullable key) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- IS NOT DISTINCT FROM prunes via primary key (Nullable key) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE k IS NOT DISTINCT FROM 42) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k = c) IS TRUE prunes via primary key (Nullable key) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE (k = 42) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- const on the left side prunes too ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE 42 IS NOT DISTINCT FROM k) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k < c) IS TRUE prunes an ordered condition ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k < 42) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- IS NOT DISTINCT FROM prunes via minmax skip index ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM mm WHERE v IS NOT DISTINCT FROM 42) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (v = c) IS TRUE prunes via minmax skip index ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM mm WHERE (v = 42) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- b = true prunes partitions (baseline) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT * FROM part WHERE b = true) WHERE explain ILIKE '%Parts: 1/2%'; + +SELECT '--- b IS TRUE prunes partitions the same as b = true ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT * FROM part WHERE b IS TRUE) WHERE explain ILIKE '%Parts: 1/2%'; + +SELECT '--- b IS NOT DISTINCT FROM true prunes partitions the same as b = true ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT * FROM part WHERE b IS NOT DISTINCT FROM true) WHERE explain ILIKE '%Parts: 1/2%'; + +-- `X != false` (`notEquals(X, false)`) is truth-equivalent to `X` for a boolean-valued `X`, so it +-- must prune the same as the bare atom across all index families. +SELECT '--- (k = c) != false prunes via primary key ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) != false) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (v = c) != false prunes via minmax skip index ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM mm WHERE (v = 42) != false) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- b != false prunes partitions the same as b = true ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT * FROM part WHERE b != false) WHERE explain ILIKE '%Parts: 1/2%'; + +-- `X != true` (`notEquals(X, true)`) is NOT truth-equivalent to `X`, so it must NOT be rewritten. +SELECT '--- (k = c) != true does NOT get the IS TRUE pruning ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) != true) WHERE explain ILIKE '%Granules: 1/%'; + +-- `X IN ()` is truth-equivalent to `X` for a boolean-valued `X`. +SELECT '--- (k = c) IN (true) prunes via primary key ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) IN (true)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k = c) IN (true, true) prunes via primary key ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) IN (true, true)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (v = c) IN (true) prunes via minmax skip index ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM mm WHERE (v = 42) IN (true)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- b IN (true) prunes partitions the same as b = true ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT * FROM part WHERE b IN (true)) WHERE explain ILIKE '%Parts: 1/2%'; + +-- `X IN (false)`, `X IN (true, false)`, `X IN (2)` and `X NOT IN (true)` are NOT truth-equivalent to +-- `X`, so they must NOT get the IS TRUE pruning. +SELECT '--- (k = c) IN (false) does NOT get the pruning ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) IN (false)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k = c) IN (true, false) does NOT get the pruning ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) IN (true, false)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- (k = c) NOT IN (true) does NOT get the pruning ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE (k = 42) NOT IN (true)) WHERE explain ILIKE '%Granules: 1/%'; + +-- `key <=> NULL` means "key IS NULL", so it reuses the existing `isNull` atom and prunes the Nullable +-- index to the NULL granule exactly (NOT the "=" range). It must prune the same as bare `key IS NULL`. +SELECT '--- IS NOT DISTINCT FROM NULL prunes to the NULL granule like IS NULL ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE k IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- NULL IS NOT DISTINCT FROM key prunes too (const on the left) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE NULL IS NOT DISTINCT FROM k) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- bare IS NULL prunes to the NULL granule (reference for the above) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE k IS NULL) WHERE explain ILIKE '%Granules: 1/%'; + +-- The `isNull` atom ignores the monotonic-functions chain (nulls are kept), so it is reused only for a +-- bare Nullable key. Any monotonic wrapper around the key declines the atom and falls back to a full +-- scan, which is always correct. A wrapper can be unsound for the atom in several ways: it can erase +-- NULL (`ifNull` / `coalesce` / `assumeNotNull` / `CAST` to a non-Nullable type make `wrapper(k) <=> +-- NULL` always false), or be partial (`toDateTime(Date32)` / `intDiv(k, 0)` throw on some non-NULL +-- rows). Reusing `isNull(k)` for any of these would narrow the index to the NULL granule and mark it +-- exact-true, so exact-count / implicit-projection paths return wrong results. All wrapped forms below +-- must NOT prune (no `Granules: 1/`). +SELECT '--- ifNull(k, 0) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE ifNull(k, 0) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- ifNull(k, 0) IS NOT DISTINCT FROM NULL does NOT prune with coalesce rewrite off (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE ifNull(k, 0) IS NOT DISTINCT FROM NULL SETTINGS allow_key_condition_coalesce_rewrite = 0) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- coalesce(k, 0) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE coalesce(k, 0) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- assumeNotNull(k) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE assumeNotNull(k) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- bare isNull(ifNull(k, 0)) does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE isNull(ifNull(k, 0))) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- CAST(k, non-Nullable) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE CAST(k, 'UInt32') IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- bare isNull(CAST(k, non-Nullable)) does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE isNull(CAST(k, 'UInt32'))) WHERE explain ILIKE '%Granules: 1/%'; +-- A NULL-preserving conversion (`toUInt32(Nullable(UInt32))` stays Nullable) is still a wrapper, so it +-- also declines and does NOT prune (the count is unchanged: a full scan returns the same NULL rows). +SELECT '--- toUInt32(k) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE toUInt32(k) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- toUInt32(ifNull(k, 0)) IS NOT DISTINCT FROM NULL does NOT prune to the NULL granule (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE toUInt32(ifNull(k, 0)) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- ifNull(k, Nullable non-NULL fallback) IS NOT DISTINCT FROM NULL does NOT prune (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE ifNull(k, CAST(0, 'Nullable(UInt32)')) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; +SELECT '--- coalesce(k, Nullable non-NULL fallback) IS NOT DISTINCT FROM NULL does NOT prune (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk_null WHERE coalesce(k, CAST(0, 'Nullable(UInt32)')) IS NOT DISTINCT FROM NULL) WHERE explain ILIKE '%Granules: 1/%'; + +-- A partial wrapper is scanned (and raises) instead of being pruned as always-false: `toDateTime(d)` on +-- a `Date32` value outside the DateTime range raises under `date_time_overflow_behavior = 'throw'`. The +-- `d32_null` key crosses the range: the `1900-01-01` granule overflows, the mid-range granule does not. +SET session_timezone = 'UTC'; +DROP TABLE IF EXISTS d32_null; +CREATE TABLE d32_null (d Nullable(Date32)) ENGINE = MergeTree ORDER BY d + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0, allow_nullable_key = 1, add_minmax_index_for_numeric_columns = 0; +INSERT INTO d32_null SELECT toDate32('1900-01-01') FROM numbers(8192); +INSERT INTO d32_null SELECT toDate32('2000-01-01') + number FROM numbers(8192); +INSERT INTO d32_null SELECT NULL FROM numbers(100); +OPTIMIZE TABLE d32_null FINAL; + +SELECT '--- isNull(toDateTime(d)) does NOT prune, keeps all granules ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM d32_null WHERE isNull(toDateTime(d)) SETTINGS date_time_overflow_behavior = 'throw') WHERE explain ILIKE '%Granules: %/%' AND explain NOT ILIKE '%Granules: 3/3%'; +SELECT '--- toDateTime(d) IS NOT DISTINCT FROM NULL does NOT prune ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM d32_null WHERE toDateTime(d) IS NOT DISTINCT FROM NULL SETTINGS date_time_overflow_behavior = 'throw') WHERE explain ILIKE '%Granules: %/%' AND explain NOT ILIKE '%Granules: 3/3%'; + +DROP TABLE IF EXISTS i64_null; +CREATE TABLE i64_null (k Nullable(Int64)) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 1, allow_nullable_key = 1, add_minmax_index_for_numeric_columns = 0; +INSERT INTO i64_null VALUES (5); +INSERT INTO i64_null VALUES (NULL); + +SELECT '--- isNull(intDiv(k, 0)) does NOT prune, keeps all granules ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM i64_null WHERE isNull(intDiv(k, 0)) SETTINGS optimize_use_implicit_projections = 0) WHERE explain ILIKE '%Granules: %/%' AND explain NOT ILIKE '%Granules: 2/2%'; +SELECT '--- intDiv(k, 0) IS NOT DISTINCT FROM NULL does NOT prune ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM i64_null WHERE intDiv(k, 0) IS NOT DISTINCT FROM NULL SETTINGS optimize_use_implicit_projections = 0) WHERE explain ILIKE '%Granules: %/%' AND explain NOT ILIKE '%Granules: 2/2%'; +SELECT '--- isNull(intDiv(k, 2)) (also wrapped) does NOT prune ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM i64_null WHERE isNull(intDiv(k, 2)) SETTINGS optimize_use_implicit_projections = 0) WHERE explain ILIKE '%Granules: 1/%'; + +-- The boolean-wrapper peel must reach ANY prunable boolean atom, not just comparisons: +-- `startsWith(s, p) IS TRUE` / `!= false` / `IN (true)` must prune the String key the same as bare +-- `startsWith(s, p)` (the gate is derived from `atom_map` in `predicateIsBooleanResult`). +SELECT '--- startsWith(s, p) IS TRUE prunes via primary key like the bare atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM spk WHERE startsWith(s, '999') IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- bare startsWith(s, p) prunes via primary key (reference) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM spk WHERE startsWith(s, '999')) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- startsWith(s, p) != false prunes via primary key like the bare atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM spk WHERE startsWith(s, '999') != false) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- startsWith(s, p) IN (true) prunes via primary key like the bare atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM spk WHERE startsWith(s, '999') IN (true)) WHERE explain ILIKE '%Granules: 1/%'; + +-- The positive-boolean-wrapper peel composes with the existing `ifNull(X, 0)` / `coalesce(X, 0)` +-- boolean rewrite (gated by `allow_key_condition_coalesce_rewrite`): `ifNull(k = 42, 0) IS TRUE` +-- (and the `!= false` / `IN (true)` forms) must prune the same as bare `ifNull(k = 42, 0)`. +SELECT '--- ifNull(k = 42, 0) IS TRUE prunes via primary key like the bare wrapped atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE ifNull(k = 42, 0) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- bare ifNull(k = 42, 0) prunes via primary key (reference) ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE ifNull(k = 42, 0)) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- coalesce(k = 42, 0) IS TRUE prunes via primary key like the bare wrapped atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE coalesce(k = 42, 0) IS TRUE) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- ifNull(k = 42, 0) != false prunes via primary key like the bare wrapped atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE ifNull(k = 42, 0) != false) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- ifNull(k = 42, 0) IN (true) prunes via primary key like the bare wrapped atom ---'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE ifNull(k = 42, 0) IN (true)) WHERE explain ILIKE '%Granules: 1/%'; + +-- With the coalesce rewrite disabled the composed peel must NOT prune (behavior preserved when off). +SELECT '--- ifNull(k = 42, 0) IS TRUE does NOT prune when allow_key_condition_coalesce_rewrite = 0 (0/) ---'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM pk WHERE ifNull(k = 42, 0) IS TRUE SETTINGS allow_key_condition_coalesce_rewrite = 0) WHERE explain ILIKE '%Granules: 1/%'; + +SELECT '--- correctness is preserved across all forms ---'; +SELECT 'ndf_nonnull', count() FROM pk WHERE k IS NOT DISTINCT FROM 42; +SELECT 'eq_nonnull', count() FROM pk WHERE k = 42; +SELECT 'istrue_eq', count() FROM pk WHERE (k = 42) IS TRUE; +SELECT 'istrue_lt', count() FROM pk WHERE (k < 42) IS TRUE; +SELECT 'ndf_nullable', count() FROM pk_null WHERE k IS NOT DISTINCT FROM 42; +SELECT 'ndf_null_is_null', count() FROM pk_null WHERE k IS NOT DISTINCT FROM NULL; +SELECT 'is_null', count() FROM pk_null WHERE k IS NULL; +-- non-boolean X: (k) IS TRUE means k = 1, NOT "k truthy"; must stay correct. +SELECT 'k_istrue_means_eq_1', count() FROM pk WHERE k IS TRUE; +SELECT 'kplus1_istrue_means_eq_1', count() FROM pk WHERE (k + 1) IS TRUE; +-- IS FALSE must not be rewritten to the IS TRUE path. +SELECT 'isfalse', count() FROM pk WHERE (k = 42) IS FALSE; +-- `!= false` / `IN (truthy)` forms match the bare atom; the non-equivalent forms match their own semantics. +SELECT 'ne_false_eq', count() FROM pk WHERE (k = 42) != false; +SELECT 'ne_true_eq', count() FROM pk WHERE (k = 42) != true; +SELECT 'in_true_eq', count() FROM pk WHERE (k = 42) IN (true); +SELECT 'in_truetrue_eq', count() FROM pk WHERE (k = 42) IN (true, true); +SELECT 'in_false_eq', count() FROM pk WHERE (k = 42) IN (false); +SELECT 'in_truefalse_eq', count() FROM pk WHERE (k = 42) IN (true, false); +SELECT 'notin_true_eq', count() FROM pk WHERE (k = 42) NOT IN (true); +-- non-boolean X: `k IN (true)` means `k = 1`, not "k truthy"; must stay correct. +SELECT 'k_in_true_means_eq_1', count() FROM pk WHERE k IN (true); +-- `key <=> NULL` returns the NULL rows (like IS NULL); the const-on-left form matches too. +SELECT 'ndf_null_left', count() FROM pk_null WHERE NULL IS NOT DISTINCT FROM k; +-- NULL-erasing wrappers over the key are ALWAYS FALSE against NULL (never match), unlike bare `k <=> NULL`. +SELECT 'ifnull_ndf_null', count() FROM pk_null WHERE ifNull(k, 0) IS NOT DISTINCT FROM NULL; +SELECT 'coalesce_ndf_null', count() FROM pk_null WHERE coalesce(k, 0) IS NOT DISTINCT FROM NULL; +SELECT 'assumenotnull_ndf_null', count() FROM pk_null WHERE assumeNotNull(k) IS NOT DISTINCT FROM NULL; +SELECT 'ifnull_isnull', count() FROM pk_null WHERE isNull(ifNull(k, 0)); +SELECT 'ifnull_ndf_null_corw_off', count() FROM pk_null WHERE ifNull(k, 0) IS NOT DISTINCT FROM NULL SETTINGS allow_key_condition_coalesce_rewrite = 0; +-- A wrapper over the key declines the atom and full-scans, so the count still matches the true result: +-- `toUInt32(k) <=> NULL` matches the NULL rows; the NULL-erasing / non-NULL-fallback forms are all false. +SELECT 'touint32_ndf_null', count() FROM pk_null WHERE toUInt32(k) IS NOT DISTINCT FROM NULL; +SELECT 'touint32_ifnull_ndf_null', count() FROM pk_null WHERE toUInt32(ifNull(k, 0)) IS NOT DISTINCT FROM NULL; +SELECT 'ifnull_nfb_ndf_null', count() FROM pk_null WHERE ifNull(k, CAST(0, 'Nullable(UInt32)')) IS NOT DISTINCT FROM NULL; +SELECT 'coalesce_nfb_ndf_null', count() FROM pk_null WHERE coalesce(k, CAST(0, 'Nullable(UInt32)')) IS NOT DISTINCT FROM NULL; +SELECT 'ifnull_nfb_ndf_null_iproj', count() FROM pk_null WHERE ifNull(k, CAST(0, 'Nullable(UInt32)')) IS NOT DISTINCT FROM NULL SETTINGS optimize_use_implicit_projections = 1; +-- Partial wrappers are scanned, not pruned to always-false, so the out-of-range non-NULL granule raises +-- exactly like a full scan (`toDateTime(Date32)` under 'throw', `intDiv(k, 0)`). +SELECT 'todatetime_isnull_throws' FROM d32_null WHERE isNull(toDateTime(d)) SETTINGS date_time_overflow_behavior = 'throw'; -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT 'todatetime_ndf_null_throws' FROM d32_null WHERE toDateTime(d) IS NOT DISTINCT FROM NULL SETTINGS date_time_overflow_behavior = 'throw'; -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +-- Under 'ignore', `toDateTime` saturates instead of throwing, so the scan succeeds and returns 0. +SELECT 'todatetime_ndf_null_ignore', count() FROM d32_null WHERE toDateTime(d) IS NOT DISTINCT FROM NULL SETTINGS date_time_overflow_behavior = 'ignore'; +SELECT 'intdiv0_isnull_throws' FROM i64_null WHERE isNull(intDiv(k, 0)); -- { serverError ILLEGAL_DIVISION } +SELECT 'intdiv0_ndf_null_throws' FROM i64_null WHERE intDiv(k, 0) IS NOT DISTINCT FROM NULL; -- { serverError ILLEGAL_DIVISION } +-- `intDiv(k, 2)` never raises, so the full scan returns the NULL-row count. +SELECT 'intdiv2_ndf_null', count() FROM i64_null WHERE intDiv(k, 2) IS NOT DISTINCT FROM NULL; +-- ifNull / coalesce composed wrapper forms match the bare wrapped atom. +SELECT 'ifnull_bare', count() FROM pk WHERE ifNull(k = 42, 0); +SELECT 'ifnull_istrue', count() FROM pk WHERE ifNull(k = 42, 0) IS TRUE; +SELECT 'coalesce_istrue', count() FROM pk WHERE coalesce(k = 42, 0) IS TRUE; +SELECT 'ifnull_ne_false', count() FROM pk WHERE ifNull(k = 42, 0) != false; +SELECT 'ifnull_in_true', count() FROM pk WHERE ifNull(k = 42, 0) IN (true); +-- startsWith wrapper forms match the bare atom. +SELECT 'startswith_bare', count() FROM spk WHERE startsWith(s, '999'); +SELECT 'startswith_istrue', count() FROM spk WHERE startsWith(s, '999') IS TRUE; +SELECT 'startswith_ne_false', count() FROM spk WHERE startsWith(s, '999') != false; +SELECT 'startswith_in_true', count() FROM spk WHERE startsWith(s, '999') IN (true); +-- Partition pruning correctness: IS TRUE / IS NOT DISTINCT FROM / != false / IN (true) match b = true. +SELECT 'part_eq', count() FROM part WHERE b = true; +SELECT 'part_istrue', count() FROM part WHERE b IS TRUE; +SELECT 'part_ndf', count() FROM part WHERE b IS NOT DISTINCT FROM true; +SELECT 'part_ne_false', count() FROM part WHERE b != false; +SELECT 'part_in_true', count() FROM part WHERE b IN (true); + +-- The `IN (truthy)` key-condition rewrite must NOT force-build the set during analysis: +-- for a subquery set (`(k = 42) IN (SELECT ...)`) the set is left for execution, so EXPLAIN +-- does not run the subquery. Regression for a server abort where the subquery was executed +-- during key analysis (see 02707_skip_index_with_in). The plan keeps a deferred `CreatingSet`. +SELECT 'explain_in_subquery_not_built', count() > 0 +FROM (EXPLAIN SELECT count() FROM pk WHERE (k = 42) IN (SELECT throwIf(1)) SETTINGS use_skip_indexes = 0) +WHERE explain ILIKE '%CreatingSet%'; + +-- A `LowCardinality` key compared with a `LowCardinality` constant drives a monotonic-function chain +-- (UInt8->Bool `CAST`) in `applyFunction`. Regression for a server abort where the raw +-- `ColumnLowCardinality` reached a wrapper doing `checkAndGetColumn` (bad cast). The +-- constant comes from a subquery so it is not folded away before key analysis. +SET allow_suspicious_low_cardinality_types = 1; +DROP TABLE IF EXISTS lc_bool; +CREATE TABLE lc_bool (b LowCardinality(Bool)) ENGINE = MergeTree ORDER BY tuple(b) SETTINGS index_granularity = 8, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; +INSERT INTO lc_bool SELECT multiIf(number < 8, false, number < 16, true, NULL) FROM numbers(24); +SELECT 'lc_less', count() FROM lc_bool WHERE toLowCardinality((SELECT false)) < b; +SELECT 'lc_eq', count() FROM lc_bool WHERE toLowCardinality((SELECT false)) = b; +SELECT 'lc_istrue', count() FROM lc_bool WHERE b IS TRUE; +SELECT 'lc_ndf', count() FROM lc_bool WHERE b IS NOT DISTINCT FROM false; +DROP TABLE lc_bool; + +DROP TABLE pk; +DROP TABLE pk_null; +DROP TABLE mm; +DROP TABLE part; +DROP TABLE spk; +DROP TABLE IF EXISTS d32_null; +DROP TABLE IF EXISTS i64_null; From caa805b79b623eea5aedbc4be0995d08f5b1362f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 08:00:11 +0000 Subject: [PATCH 25/86] Backport #110246 to 26.6: Do not use text index for has/mapContainsKey/mapContainsValue with an empty needle --- .../MergeTree/MergeTreeIndexConditionText.cpp | 11 ++++ .../02346_text_index_bug110092.reference | 14 ++++++ .../02346_text_index_bug110092.sql | 50 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 tests/queries/0_stateless/02346_text_index_bug110092.reference create mode 100644 tests/queries/0_stateless/02346_text_index_bug110092.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index b4e6f021425e..9b8698dd6fd5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -861,6 +861,11 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( auto make_map_function = [&](VectorWithMemoryTracking tokens) { + /// Empty needles produce no tokens that can be searched for --> fall back to brute force scan. + /// See function "equals" for a longer explanation. + if (tokens.empty()) + return false; + out.function = RPNElement::FUNCTION_EQUALS; out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; @@ -1276,6 +1281,12 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (function_name == "has") { auto tokens = stringToTokens(value_field); + + /// Empty needles produce no tokens that can be searched for, fall back to brute force scan. + /// See function "equals" for a longer explanation. + if (tokens.empty()) + return false; + out.function = RPNElement::FUNCTION_EQUALS; out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; diff --git a/tests/queries/0_stateless/02346_text_index_bug110092.reference b/tests/queries/0_stateless/02346_text_index_bug110092.reference new file mode 100644 index 000000000000..4f36c598c919 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug110092.reference @@ -0,0 +1,14 @@ +-- empty needle, index lookup +0 +0 +0 +0 +0 +0 +-- empty needle, no index +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02346_text_index_bug110092.sql b/tests/queries/0_stateless/02346_text_index_bug110092.sql new file mode 100644 index 000000000000..928d1ba5103d --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug110092.sql @@ -0,0 +1,50 @@ +-- Bug 110092: has() and mapContainsKey/Value() with empty needles + +DROP TABLE IF EXISTS tab; +CREATE TABLE tab +( + id UInt64, + arr Array(String), + mp Map(String, String), + INDEX a_text arr TYPE text(tokenizer = 'array'), + INDEX mk_text mapKeys(mp) TYPE text(tokenizer = 'array'), + INDEX mv_text mapValues(mp) TYPE text(tokenizer = 'array') +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 64; + +-- No empty strings anywhere in the data; id is always < 8192. +INSERT INTO tab +SELECT + number, + [concat('tok', toString(number))], + map(concat('k', toString(number)), concat('v', toString(number))) +FROM numbers(8192); + + +SELECT '-- empty needle, index lookup'; +SET use_skip_indexes = 1; +-- The wrong-results path is only reached when the direct read from the text index is enabled, which +-- otherwise gets randomized by clickhouse-test. Force it on so the regression is deterministic. +SET query_plan_direct_read_from_text_index = 1; +SELECT count() FROM tab WHERE has(arr, ''); +SELECT count() FROM tab WHERE has(mp, ''); +SELECT count() FROM tab WHERE mapContainsKey(mp, ''); +SELECT count() FROM tab WHERE mapContainsValue(mp, ''); +SELECT count() FROM tab WHERE mapContainsKeyLike(mp, ''); +SELECT count() FROM tab WHERE mapContainsValueLike(mp, ''); +SET query_plan_direct_read_from_text_index = default; +SET use_skip_indexes = default; + +SELECT '-- empty needle, no index'; +SET use_skip_indexes = 0; +SELECT count() FROM tab WHERE has(arr, ''); +SELECT count() FROM tab WHERE has(mp, ''); +SELECT count() FROM tab WHERE mapContainsKey(mp, ''); +SELECT count() FROM tab WHERE mapContainsValue(mp, ''); +SELECT count() FROM tab WHERE mapContainsKeyLike(mp, ''); +SELECT count() FROM tab WHERE mapContainsValueLike(mp, ''); +SET use_skip_indexes = default; + +DROP TABLE tab; From 9bb72ddd5e009450a6bbc9a3ad46dc3c38ab93cd Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 23 Jul 2026 10:59:44 +0200 Subject: [PATCH 26/86] Fix backport build: declare process_list_element inline and use 2-arg DeduplicationInfo::create The backport of #111150 added a pre-loop `checkTimeLimit` guard to `MergeTreeSink::consume`, but this branch has neither the hoisted `process_list_element` local nor the `ProcessList.h` include that exist on master, breaking every build variant with `use of undeclared identifier 'process_list_element'`. Declare the variable inline, matching the same guard in `ReplicatedMergeTreeSink` on this branch, and add the include. Also pass the branch's second `DeduplicationInfo::create` argument in the new `testPrewarmDataHashes` gtest. Behavior is unchanged. --- src/Storages/MergeTree/MergeTreeSink.cpp | 3 ++- src/Storages/MergeTree/tests/gtest_async_inserts.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 5dfacc1b1a3c..0de2c34055c8 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -121,7 +122,7 @@ void MergeTreeSink::consume(Chunk & chunk) /// Preserve the pre-loop interrupt point that used to be the first checkTimeLimit() /// inside the partition loop: a killed or timed-out insert should be noticed before /// the full O(N) prewarm hash pass, not after it. - if (process_list_element) + if (auto process_list_element = context->getProcessListElement()) process_list_element->checkTimeLimit(); /// Warm the data hashes once here: the per-partition infos produced by filterToPartition diff --git a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp index b307e13d6dd5..bb9f0b7be75c 100644 --- a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp +++ b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp @@ -148,7 +148,7 @@ std::vector testPrewarmDataHashes(std::vector data, std::vector< column->insert(datum); Block block({ColumnWithTypeAndName(std::move(column), std::make_shared(), "a")}); - auto deduplication_info = DeduplicationInfo::create(true); + auto deduplication_info = DeduplicationInfo::create(true, InsertDeduplicationVersions::NEW_UNIFIED_HASHES); deduplication_info->setRootViewID({}); deduplication_info->disabled = false; deduplication_info->updateOriginalBlock(Chunk(block.getColumns(), block.rows()), std::make_shared(block.cloneEmpty())); From 62f6e1e3ed06312a08ab7992ebd70f62aa127f7f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 09:02:51 +0000 Subject: [PATCH 27/86] Backport #109676 to 26.6: Compare Iceberg primitive type strings ignoring whitespace --- .../DataLakes/Iceberg/SchemaProcessor.cpp | 184 +++++++++++-- .../tests/gtest_iceberg_schema_processor.cpp | 256 ++++++++++++++++++ 2 files changed, 410 insertions(+), 30 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 8c6fd9ee22a2..d65c798de358 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -118,6 +120,128 @@ bool equals(const T & first, const T & second) bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second, const std::unordered_map & type_mapping); +/// Canonicalize spacing in an Iceberg primitive type string by removing ASCII whitespace that is +/// only optional formatting around the delimiters '(', ')', '[', ']', ',' (and at the string edges). +/// Whitespace embedded inside a token (e.g. between the digits of "decimal(2 0,0)") is preserved so +/// that malformed spellings remain malformed and are still rejected by the parser. +String canonicalizeTypeSpacing(const String & s) +{ + auto is_delimiter = [](char c) { return c == '(' || c == ')' || c == '[' || c == ']' || c == ','; }; + String result; + result.reserve(s.size()); + for (size_t i = 0; i < s.size(); ++i) + { + if (!isWhitespaceASCII(s[i])) + { + result.push_back(s[i]); + continue; + } + const char prev = result.empty() ? '\0' : result.back(); + size_t j = i + 1; + while (j < s.size() && isWhitespaceASCII(s[j])) + ++j; + const char next = j < s.size() ? s[j] : '\0'; + /// Drop whitespace only when it is next to a delimiter or at the start/end of the string; + /// keep it when it sits between two token characters. + const bool drop = prev == '\0' || next == '\0' || is_delimiter(prev) || is_delimiter(next); + if (!drop) + result.push_back(s[i]); + } + return result; +} + +/// Compare two Iceberg type descriptors for the same field. A type is either a primitive +/// string ("long", "decimal(20, 0)", ...) or a nested object ("struct" with a fields array, +/// or "list" / "map" wrappers whose element/key/value members are themselves types). +/// Primitive type strings are whitespace-insensitive per the Iceberg spec, so recurse into +/// list/map members instead of comparing the wrapper object textually. +bool typesAreStructurallyIdentical( + const Poco::Dynamic::Var & first_in, const Poco::Dynamic::Var & second_in, const std::unordered_map & type_mapping) +{ + Poco::Dynamic::Var first = first_in; + Poco::Dynamic::Var second = second_in; + + /// Apply configured type aliases (e.g. geography -> binary) to string types. + /// Canonicalize spacing before the alias prefix match so leading/trailing whitespace does not + /// defeat it: " geography(C,A)" must map to the same alias as "geography(C, A)". Otherwise the + /// same schema-id serialized with one spelling that skips aliasing and another that maps to + /// "binary" would compare unequal and be wrongly rejected. + if (first.isString()) + { + const String canon = canonicalizeTypeSpacing(first.toString()); + first = canon; + for (const auto & [prefix, mapped] : type_mapping) + if (canon.starts_with(prefix)) + { + first = mapped; + break; + } + } + if (second.isString()) + { + const String canon = canonicalizeTypeSpacing(second.toString()); + second = canon; + for (const auto & [prefix, mapped] : type_mapping) + if (canon.starts_with(prefix)) + { + second = mapped; + break; + } + } + + /// Primitive type strings: e.g. both "decimal(20,0)" and "decimal(20, 0)" denote the same + /// type. Different writers emit different spacing, so ignore ASCII whitespace. + if (first.isString() && second.isString()) + return canonicalizeTypeSpacing(first.toString()) == canonicalizeTypeSpacing(second.toString()); + + const bool both_objects + = first.type() == typeid(Poco::JSON::Object::Ptr) && second.type() == typeid(Poco::JSON::Object::Ptr); + if (both_objects) + { + const auto & first_obj = first.extract(); + const auto & second_obj = second.extract(); + + /// struct: compare nested field list recursively. + if (first_obj->isArray(f_fields) || second_obj->isArray(f_fields)) + return schemasAreIdentical(*first_obj, *second_obj, type_mapping); + + /// list / map wrappers: same member set, with the nested type members (element / key / + /// value) compared recursively so their primitive strings are whitespace-insensitive too, + /// and the remaining scalar members (ids, required flags) compared textually. + auto names_first = first_obj->getNames(); + auto names_second = second_obj->getNames(); + std::sort(names_first.begin(), names_first.end()); + std::sort(names_second.begin(), names_second.end()); + if (names_first != names_second) + return false; + for (const auto & name : names_first) + { + if (name == f_element || name == f_key || name == f_value) + { + if (!typesAreStructurallyIdentical(first_obj->get(name), second_obj->get(name), type_mapping)) + return false; + } + else + { + Poco::JSON::Object wrapper_first; + wrapper_first.set(name, first_obj->get(name)); + Poco::JSON::Object wrapper_second; + wrapper_second.set(name, second_obj->get(name)); + if (!equals(wrapper_first, wrapper_second)) + return false; + } + } + return true; + } + + /// Mismatched shapes (string vs object) or scalar values: compare textually. + Poco::JSON::Object wrapper_first; + wrapper_first.set(f_type, first); + Poco::JSON::Object wrapper_second; + wrapper_second.set(f_type, second); + return equals(wrapper_first, wrapper_second); +} + bool schemaFieldsAreStructurallyIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second, const std::unordered_map & type_mapping) { static constexpr const char * structural_keys[] = {f_id, f_name, f_required, f_type}; @@ -130,37 +254,17 @@ bool schemaFieldsAreStructurallyIdentical(const Poco::JSON::Object & first, cons if (!first_has) continue; - if (key == f_type && first.isObject(key) && second.isObject(key)) + if (key == f_type) { - const auto first_type = first.getObject(key); - const auto second_type = second.getObject(key); - if (first_type->isArray(f_fields) || second_type->isArray(f_fields)) - { - if (!schemasAreIdentical(*first_type, *second_type, type_mapping)) - return false; - continue; - } - } - - auto key_first = first.get(key); - auto key_second = second.get(key); - if (key == f_type && key_first.isString()) - { - for (const auto & [prefix, mapped] : type_mapping) - if (key_first.toString().starts_with(prefix)) - key_first = mapped; - } - if (key == f_type && key_second.isString()) - { - for (const auto & [prefix, mapped] : type_mapping) - if (key_second.toString().starts_with(prefix)) - key_second = mapped; + if (!typesAreStructurallyIdentical(first.get(key), second.get(key), type_mapping)) + return false; + continue; } Poco::JSON::Object wrapper_first; - wrapper_first.set(key, key_first); + wrapper_first.set(key, first.get(key)); Poco::JSON::Object wrapper_second; - wrapper_second.set(key, key_second); + wrapper_second.set(key, second.get(key)); if (!equals(wrapper_first, wrapper_second)) return false; } @@ -194,7 +298,11 @@ std::pair parseDecimal(const String & type_name) skipWhitespaceIfAny(buf); assertChar(',', buf); skipWhitespaceIfAny(buf); - tryReadIntText(scale, buf); + /// readIntText (not tryReadIntText) so a missing scale ("decimal(20,)") is rejected instead of + /// silently read as 0. assertEOF then rejects trailing garbage (e.g. "decimal(20,0 0)", whose + /// inner whitespace survives canonicalization), mirroring the fixed[N] handling. + readIntText(scale, buf); + assertEOF(buf); return {precision, scale}; } @@ -293,8 +401,13 @@ NamesAndTypesList IcebergSchemaProcessor::tryGetFieldsCharacteristics(Int32 sche return fields; } -DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name, bool allow_geo_parser) +DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name_arg, bool allow_geo_parser) { + /// Parameterized primitive type strings (decimal(P, S), fixed[N], geography(...)) can be + /// serialized with different inner whitespace across metadata files. Canonicalize by removing + /// ASCII whitespace so parsing accepts every spelling the whitespace-insensitive comparison does. + const String type_name = canonicalizeTypeSpacing(type_name_arg); + if (type_name == f_boolean) return DataTypeFactory::instance().get("Bool"); if (type_name == f_int) @@ -336,6 +449,9 @@ DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name, bool ReadBufferFromString buf(std::string_view(type_name.begin() + 6, type_name.end() - 1)); size_t n = 0; readIntText(n, buf); + /// Reject trailing garbage such as embedded whitespace ("fixed[1 6]"): the canonicalized + /// form of a valid spelling has no characters left after the size. + assertEOF(buf); return std::make_shared(n); } @@ -431,8 +547,13 @@ DataTypePtr IcebergSchemaProcessor::getFieldType( * decimal(P, S) -> decimal(P', S) where P' > P * This function checks if `old_type` and `new_type` satisfy to one of these conditions. **/ -bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_type, const String & new_type) +bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_type_arg, const String & new_type_arg) { + /// Match the whitespace-insensitive rules of the comparison and the parser: a whitespace-only + /// difference in a parameterized type string denotes the identical type. + const String old_type = canonicalizeTypeSpacing(old_type_arg); + const String new_type = canonicalizeTypeSpacing(new_type_arg); + bool allowed_type_conversion = (old_type == new_type); allowed_type_conversion |= (old_type == f_int) && (new_type == f_long); allowed_type_conversion |= (old_type == f_float) && (new_type == f_double); @@ -499,7 +620,10 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( String new_type = field->getValue(f_type); const ActionsDAG::Node * node = old_node; - if (old_type == new_type) + /// Parameterized primitive types (decimal, geography, ...) can be serialized with + /// different spacing across metadata files, so compare ignoring ASCII whitespace: + /// a whitespace-only difference is the same type and needs only a rename, not a cast. + if (canonicalizeTypeSpacing(old_type) == canonicalizeTypeSpacing(new_type)) { if (old_json->getValue(f_name) != name) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index 8719762b0ba2..504f0f8fc7e9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -4,8 +4,20 @@ #include #include +#include +#include + using namespace DB::Iceberg; +namespace +{ +Poco::JSON::Object::Ptr parseSchema(const std::string & json) +{ + Poco::JSON::Parser parser; + return parser.parse(json).extract(); +} +} + TEST(IcebergSchemaProcessor, GetSimpleTypeBoolean) { auto type = IcebergSchemaProcessor::getSimpleType("boolean"); @@ -112,3 +124,247 @@ TEST(IcebergSchemaProcessor, GetSimpleTypeUnknownThrows) { EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("unknown_type"), DB::Exception); } + +/// The Iceberg primitive type grammar is a closed set: scalars, decimal(P, S) and fixed[N] whose +/// only parameters are integers, geography/geometry whose parameters are bare identifiers, and the +/// list/map/struct wrappers. None of them carries a quoted string literal. A spelling that embeds +/// one (e.g. "MyType('Hello ( world )')") matches no branch of getSimpleType and is rejected before +/// any comparison runs, so canonicalizeTypeSpacing never sees whitespace inside a quoted literal. +TEST(IcebergSchemaProcessor, GetSimpleTypeWithStringLiteralArgumentThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("MyType('Hello ( world )')"), DB::Exception); +} + +/// The same string-literal-bearing spelling must be rejected as an initial schema type, i.e. the +/// parser guards the entry point so a quoted literal never reaches the whitespace canonicalization. +TEST(IcebergSchemaProcessor, InitialSchemaTypeWithStringLiteralArgumentThrows) +{ + auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"MyType('Hello ( world )')"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_THROW(processor.addIcebergTableSchema(schema), DB::Exception); +} + +/// The primitive parser must accept the same inner-whitespace spellings that the +/// whitespace-insensitive comparison treats as equivalent. Without canonicalizing the type string +/// before parsing, readIntText does not skip the leading space, so "decimal( 20, 0 )" and +/// "fixed[ 16 ]" fail to parse even though they denote decimal(20, 0) / fixed[16]. +TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalInnerWhitespace) +{ + auto type = IcebergSchemaProcessor::getSimpleType("decimal( 20, 0 )"); + EXPECT_EQ(type->getName(), "Decimal(20, 0)"); +} + +TEST(IcebergSchemaProcessor, GetSimpleTypeFixedInnerWhitespace) +{ + auto type = IcebergSchemaProcessor::getSimpleType("fixed[ 16 ]"); + EXPECT_EQ(type->getName(), "FixedString(16)"); +} + +/// Regression test for https://github.com/ClickHouse/ClickHouse/issues/109642 +/// The same schema-id can be serialized by different Iceberg writers with different +/// whitespace in parameterized primitive type strings, e.g. the table metadata JSON +/// emits "decimal(20,0)" while the manifest Avro metadata emits "decimal(20, 0)". +/// Both denote the identical type per the Iceberg spec, so re-adding the schema-id +/// must NOT be rejected as a rebinding to a different schema. +TEST(IcebergSchemaProcessor, DecimalTypeWhitespaceIsInsensitive) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0)"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20, 0)"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second)); +} + +/// A genuinely different type bound to the same schema-id must still be rejected. +TEST(IcebergSchemaProcessor, RebindingSchemaIdToDifferentTypeStillRejected) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0)"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,2)"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_THROW(processor.addIcebergTableSchema(second), DB::Exception); +} + +/// A renamed field bound to the same schema-id must still be rejected (issue #107316). +TEST(IcebergSchemaProcessor, RebindingSchemaIdToRenamedFieldStillRejected) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"long"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c9","required":false,"type":"long"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_THROW(processor.addIcebergTableSchema(second), DB::Exception); +} + +/// The whitespace-insensitive comparison must reach into list/map wrappers: the nested +/// element/key/value primitive types (here list) can also be serialized with +/// different spacing across metadata files. +TEST(IcebergSchemaProcessor, ListElementDecimalWhitespaceIsInsensitive) +{ + auto first = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20,0)"}}]})json"); + auto second = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20, 0)"}}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second)); +} + +/// Same for map key/value primitive types (here map). +TEST(IcebergSchemaProcessor, MapKeyValueDecimalWhitespaceIsInsensitive) +{ + auto first = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"map","key-id":2,"key":"decimal(20,0)","value-id":3,"value-required":false,"value":"decimal(10,2)"}}]})json"); + auto second = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"map","key-id":2,"key":"decimal(20, 0)","value-id":3,"value-required":false,"value":"decimal(10, 2)"}}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second)); +} + +/// The Iceberg geography/geometry primitives carry parameters too, e.g. +/// "geography(crs, algorithm)", so their serialization can also differ by whitespace +/// across metadata files. With the geo parser enabled, re-adding the same schema-id with +/// different spacing must not be rejected. +TEST(IcebergSchemaProcessor, GeographyTypeWhitespaceIsInsensitive) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C,A)"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C, A)"}]})json"); + IcebergSchemaProcessor processor(/*allow_geo_parser_=*/true); + processor.addIcebergTableSchema(first); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second)); +} + +/// A geo type string carrying leading/trailing whitespace must map to its alias just like the +/// space-free spelling. The alias prefix match (geography -> binary) runs on the canonicalized +/// spelling, so " geography(C,A)" and "geography(C, A)" under the same schema-id compare equal +/// instead of one skipping aliasing (staying "geography") and the other becoming "binary". +TEST(IcebergSchemaProcessor, GeographyTypeEdgeWhitespaceIsInsensitive) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":" geography(C,A) "}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"geography(C, A)"}]})json"); + IcebergSchemaProcessor processor(/*allow_geo_parser_=*/true); + processor.addIcebergTableSchema(first); + EXPECT_NO_THROW(processor.addIcebergTableSchema(second)); +} + +/// Schema-evolution path: renaming a geo field across two schema-ids while only changing the +/// whitespace of its parameterized type string must resolve to a rename, so the transform DAG +/// exposes the NEW column name. Without whitespace-insensitive comparison the old node is kept +/// unchanged and the DAG would still expose the old name. +TEST(IcebergSchemaProcessor, RenameGeoFieldAcrossSchemaIdsWithWhitespaceIsRename) +{ + auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"geography(C,A)"}]})json"); + auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"b","required":false,"type":"geography(C, A)"}]})json"); + IcebergSchemaProcessor processor(/*allow_geo_parser_=*/true); + processor.addIcebergTableSchema(old_schema); + processor.addIcebergTableSchema(new_schema); + + auto dag = processor.getSchemaTransformationDagByIds(0, 1); + ASSERT_TRUE(dag); + const auto & outputs = dag->getOutputs(); + ASSERT_EQ(outputs.size(), 1u); + EXPECT_EQ(outputs[0]->result_name, "b"); +} + +/// A whitespace-heavy type string must be accepted in the INITIAL/current schema (not just the +/// repeated-same-schema-id path): the parser runs before any comparison, so it has to tolerate the +/// same spellings on its own. +TEST(IcebergSchemaProcessor, InitialSchemaDecimalInnerWhitespaceAccepted) +{ + auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal( 20, 0 )"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_NO_THROW(processor.addIcebergTableSchema(schema)); +} + +/// Schema-evolution across two schema-ids where a decimal widens (allowed conversion) while its +/// type string also carries inner whitespace. allowPrimitiveTypeConversion must canonicalize the +/// spacing so the widening is still recognized and the DAG casts to the new type under the new name. +TEST(IcebergSchemaProcessor, WidenDecimalAcrossSchemaIdsWithInnerWhitespace) +{ + auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(10,2)"}]})json"); + auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal( 20, 2 )"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(old_schema); + processor.addIcebergTableSchema(new_schema); + + auto dag = processor.getSchemaTransformationDagByIds(0, 1); + ASSERT_TRUE(dag); + const auto & outputs = dag->getOutputs(); + ASSERT_EQ(outputs.size(), 1u); + EXPECT_EQ(outputs[0]->result_type->getName(), "Nullable(Decimal(20, 2))"); +} + +/// A genuinely different nested type inside a list wrapper must still be rejected. +TEST(IcebergSchemaProcessor, RebindingListElementToDifferentTypeStillRejected) +{ + auto first = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20,0)"}}]})json"); + auto second = parseSchema( + R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":{"type":"list","element-id":2,"element-required":false,"element":"decimal(20,2)"}}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_THROW(processor.addIcebergTableSchema(second), DB::Exception); +} + +/// Spacing normalization only removes whitespace adjacent to the delimiters '(', ')', '[', ']', ','. +/// Whitespace embedded inside a numeric token is not formatting, so malformed spellings such as +/// "decimal(2 0,0)" or "fixed[1 6]" must NOT canonicalize to a valid type and must still be rejected. +TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalMalformedInnerTokenWhitespaceThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("decimal(2 0,0)"), DB::Exception); +} + +TEST(IcebergSchemaProcessor, GetSimpleTypeFixedMalformedInnerTokenWhitespaceThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("fixed[1 6]"), DB::Exception); +} + +/// The same malformed spelling must be rejected when it appears as an initial schema type, i.e. the +/// broadened normalization must not let invalid metadata pass through addIcebergTableSchema. +TEST(IcebergSchemaProcessor, InitialSchemaDecimalMalformedInnerTokenWhitespaceThrows) +{ + auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(2 0,0)"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_THROW(processor.addIcebergTableSchema(schema), DB::Exception); +} + +/// Trailing garbage after the scale token must be rejected. Canonicalizing spacing does not remove +/// whitespace between two digits, so "decimal(20,0 0)" keeps the embedded space; the parser must not +/// stop after reading the scale and silently ignore the rest. This mirrors the fixed[N] handling. +TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalTrailingGarbageInScaleThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("decimal(20,0 0)"), DB::Exception); +} + +/// The same malformed scale spelling must be rejected as an initial schema type. +TEST(IcebergSchemaProcessor, InitialSchemaDecimalTrailingGarbageInScaleThrows) +{ + auto schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,0 0)"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_THROW(processor.addIcebergTableSchema(schema), DB::Exception); +} + +/// A new schema-id introduced during evolution is parsed at add time (getSimpleType runs on every +/// field), so a malformed scale in the new schema is rejected when the new schema is added and never +/// reaches the evolution DAG. The old, valid schema-id remains added. +TEST(IcebergSchemaProcessor, SchemaEvolutionDecimalTrailingGarbageInScaleThrows) +{ + auto old_schema = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(10,2)"}]})json"); + auto new_schema = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"c0","required":false,"type":"decimal(20,2 2)"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(old_schema); + EXPECT_THROW(processor.addIcebergTableSchema(new_schema), DB::Exception); +} + +/// A missing scale ("decimal(20,)") or a sign-only scale ("decimal(20,+)") is malformed metadata and +/// must be rejected, not silently read as scale 0. The scale is parsed with readIntText, which throws +/// at end of buffer or on a non-digit, matching how the precision is parsed. +TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalEmptyScaleThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("decimal(20,)"), DB::Exception); +} + +TEST(IcebergSchemaProcessor, GetSimpleTypeDecimalSignOnlyScaleThrows) +{ + EXPECT_THROW(IcebergSchemaProcessor::getSimpleType("decimal(20,+)"), DB::Exception); +} From 06c827a2a6dcd5658927a4ff6ac1aae8305d52ac Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 10:53:14 +0000 Subject: [PATCH 28/86] Backport #111489 to 26.6: Fix typo in complex schema evolution --- .../ComplexTypeSchemaProcessorFunctions.cpp | 2 +- .../test_array_map_evolved_with_struct.py | 147 ++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_storage_iceberg_schema_evolution/test_array_map_evolved_with_struct.py diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ComplexTypeSchemaProcessorFunctions.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ComplexTypeSchemaProcessorFunctions.cpp index 44534ede1490..9c6ad6a2439a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ComplexTypeSchemaProcessorFunctions.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ComplexTypeSchemaProcessorFunctions.cpp @@ -257,7 +257,7 @@ void IIcebergSchemaTransform::transform(ComplexNode & initial_node) } else if (current_tuple[subfield_index].tryGet(tmp_node_map)) { - current_node = std::move(tmp_node_array); + current_node = std::move(tmp_node_map); } else diff --git a/tests/integration/test_storage_iceberg_schema_evolution/test_array_map_evolved_with_struct.py b/tests/integration/test_storage_iceberg_schema_evolution/test_array_map_evolved_with_struct.py new file mode 100644 index 000000000000..7702b5eaffb2 --- /dev/null +++ b/tests/integration/test_storage_iceberg_schema_evolution/test_array_map_evolved_with_struct.py @@ -0,0 +1,147 @@ +import pytest + +from helpers.iceberg_utils import ( + get_uuid_str, + check_schema_and_data, + default_upload_directory, + get_creation_expression +) + + +@pytest.mark.parametrize("format_version", ["1", "2"]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_array_map_evolved_with_struct( + started_cluster_iceberg_schema_evolution, format_version, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + spark = started_cluster_iceberg_schema_evolution.spark_session + TABLE_NAME = ( + "test_array_map_evolved_with_struct_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster_iceberg_schema_evolution, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME};") + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + x ARRAY>> + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (ARRAY(MAP(1, named_struct('a', 10, 'b', 'hello')), MAP(2, named_struct('a', 20, 'b', 'world')))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + # Before evolution: the [ARRAY, MAP, STRUCT] traversal must read cleanly. + check_schema_and_data( + instance, + table_function, + [ + ['x', 'Array(Map(Int32, Tuple(\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["[{1:(10,'hello')},{2:(20,'world')}]"], + ], + ) + + # ADD a column to the innermost struct nested under Array(Map(...)). + # This is what threw std::bad_variant_access before the fix. + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ADD COLUMN x.element.value.c INT; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ['x', 'Array(Map(Int32, Tuple(\\n a Nullable(Int32),\\n b Nullable(String),\\n c Nullable(Int32))))'], + ], + [ + ["[{1:(10,'hello',NULL)},{2:(20,'world',NULL)}]"], + ], + ) + + # REORDER a struct field. + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN x.element.value.c FIRST; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ['x', 'Array(Map(Int32, Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["[{1:(NULL,10,'hello')},{2:(NULL,20,'world')}]"], + ], + ) + + # RENAME a struct field. + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} RENAME COLUMN x.element.value.a TO renamed_a; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ['x', 'Array(Map(Int32, Tuple(\\n c Nullable(Int32),\\n renamed_a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["[{1:(NULL,10,'hello')},{2:(NULL,20,'world')}]"], + ], + ) + + # DROP a struct field. + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} DROP COLUMN x.element.value.b; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ['x', 'Array(Map(Int32, Tuple(\\n c Nullable(Int32),\\n renamed_a Nullable(Int32))))'], + ], + [ + ["[{1:(NULL,10)},{2:(NULL,20)}]"], + ], + ) + return From 006a2fdb1b7d6ee7a6b2c3fa9bf83efd62cc788b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 12:26:49 +0000 Subject: [PATCH 29/86] =?UTF-8?q?Backport=20#108287=20to=2026.6:=20Fix=20c?= =?UTF-8?q?olumn=20not=20found=20error=20for=20lazy=20materialization=20an?= =?UTF-8?q?d=20part=5Foffset=20i=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../QueryPlan/ReadFromMergeTree.cpp | 70 +++++++-- ...rialization_part_offset_prewhere.reference | 145 ++++++++++++++++++ ...y_materialization_part_offset_prewhere.sql | 58 +++++++ 3 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.reference create mode 100644 tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.sql diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 385d91586587..8f2653729ab6 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -3414,28 +3414,64 @@ std::unique_ptr ReadFromMergeTree::keepOnlyRequiredColu void ReadFromMergeTree::addStartingPartOffsetAndPartOffset(bool & added_part_starting_offset, bool & added_part_offset) { - added_part_starting_offset = true; - added_part_offset = true; + /// A read column consumed by a filter is exposed again by adding it back to the filter outputs, + /// the same way `PREWHERE` keeps pass-through columns. When the column is the filter column itself + /// (e.g. `PREWHERE _part_offset`), it is already among the outputs, so the remove-filter flag is + /// cleared instead: after filtering it keeps its original values for the surviving rows. + /// Both are required for the data-read pipeline to actually emit it, not only for the plan header. + auto reexpose_in_filter = [](ActionsDAG & filter_actions, const String & filter_column_name, bool & remove_filter_column, const String & column_name) -> bool + { + auto & dag_outputs = filter_actions.getOutputs(); + for (const auto * input : filter_actions.getInputs()) + { + if (input->result_name != column_name) + continue; + + if (std::ranges::find(dag_outputs, input) == dag_outputs.end()) + { + dag_outputs.push_back(input); + return true; + } + + if (filter_column_name == column_name && remove_filter_column) + { + remove_filter_column = false; + return true; + } + } + return false; + }; - for (const auto & col_name : all_column_names) + auto expose_in_output = [&](const String & column_name) -> bool { - if (col_name == "_part_starting_offset") - added_part_starting_offset = false; - if (col_name == "_part_offset") - added_part_offset = false; - } + if (output_header && output_header->has(column_name)) + return false; - if (!added_part_starting_offset && !added_part_offset) - return; + bool reexposed = false; + if (query_info.row_level_filter) + reexposed |= reexpose_in_filter( + query_info.row_level_filter->actions, + query_info.row_level_filter->column_name, + query_info.row_level_filter->do_remove_column, + column_name); + if (query_info.prewhere_info) + reexposed |= reexpose_in_filter( + query_info.prewhere_info->prewhere_actions, + query_info.prewhere_info->prewhere_column_name, + query_info.prewhere_info->remove_prewhere_column, + column_name); - Names new_column_names; - if (added_part_starting_offset) - new_column_names.push_back("_part_starting_offset"); - if (added_part_offset) - new_column_names.push_back("_part_offset"); + if (!reexposed && std::ranges::find(all_column_names, column_name) == all_column_names.end()) + all_column_names.insert(all_column_names.begin(), column_name); - new_column_names.insert(new_column_names.end(), all_column_names.begin(), all_column_names.end()); - all_column_names = std::move(new_column_names); + return true; + }; + + added_part_starting_offset = expose_in_output("_part_starting_offset"); + added_part_offset = expose_in_output("_part_offset"); + + if (!added_part_starting_offset && !added_part_offset) + return; output_header = std::make_shared(MergeTreeSelectProcessor::transformHeader( storage_snapshot->getSampleBlockForColumns(all_column_names), diff --git a/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.reference b/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.reference new file mode 100644 index 000000000000..054586059e41 --- /dev/null +++ b/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.reference @@ -0,0 +1,145 @@ +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 +49 +48 +47 +46 +45 +44 +43 +42 +41 +40 +98 +96 +94 +92 +90 +88 +86 +84 +82 +80 +98 +96 +94 +92 +90 +88 +86 +84 +82 +80 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +999 +998 +997 +996 +995 +994 +993 +992 +991 +990 +1 1 +2 2 +3 3 +4 4 +5 5 +999 +998 +997 +996 +995 +994 +993 +992 +991 +990 +999 +998 +997 +996 +995 +994 +993 +992 +991 +990 +99 +98 +97 +96 +95 +94 +93 +92 +91 +90 diff --git a/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.sql b/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.sql new file mode 100644 index 000000000000..c2889865542f --- /dev/null +++ b/tests/queries/0_stateless/04408_lazy_materialization_part_offset_prewhere.sql @@ -0,0 +1,58 @@ +DROP ROW POLICY IF EXISTS repro_pol ON repro; +DROP TABLE IF EXISTS repro; + +SET query_plan_optimize_lazy_materialization = 1; +SET query_plan_max_limit_for_lazy_materialization = 100; + +CREATE TABLE repro (Id UInt64, EventId UInt64, flag UInt8) ENGINE = MergeTree ORDER BY Id; +INSERT INTO repro SELECT number, number, number % 2 FROM numbers(1000); + +SELECT EventId FROM repro WHERE _part_starting_offset + _part_offset < 100 ORDER BY Id DESC LIMIT 10; +SELECT EventId FROM repro PREWHERE _part_starting_offset + _part_offset < 100 ORDER BY Id DESC LIMIT 10; + +CREATE ROW POLICY repro_pol ON repro USING _part_offset < 100 TO ALL; + +-- The `_part_offset` is consumed by the row-level filter, no PREWHERE. +SELECT EventId FROM repro ORDER BY Id DESC LIMIT 10; +-- Both filters consume the offset columns. +SELECT EventId FROM repro PREWHERE _part_starting_offset + _part_offset < 100 ORDER BY Id DESC LIMIT 10; + +DROP ROW POLICY repro_pol ON repro; + +CREATE ROW POLICY repro_pol ON repro USING _part_starting_offset + _part_offset < 100 TO ALL; + +-- Both offset virtual columns are consumed by the row-level filter, no PREWHERE. +SELECT EventId FROM repro ORDER BY Id DESC LIMIT 10; + +-- The offset filter additionally appears in PREWHERE: both filters consume the offset columns. +SELECT EventId FROM repro WHERE _part_starting_offset + _part_offset < 50 ORDER BY Id DESC LIMIT 10; + +-- Virtual columns are consumed by the row-level filter, PREWHERE refers unrelated columns. +SELECT EventId FROM repro WHERE flag = 0 ORDER BY Id DESC LIMIT 10; +SELECT EventId FROM repro PREWHERE flag = 0 ORDER BY Id DESC LIMIT 10; + +DROP ROW POLICY repro_pol ON repro; +CREATE ROW POLICY repro_pol ON repro USING flag = 0 TO ALL; + +-- PREWHERE consumes the virtual columns, but the row-level refers unrelated columns. +SELECT EventId FROM repro WHERE _part_starting_offset + _part_offset < 50 ORDER BY Id DESC LIMIT 10; +SELECT EventId FROM repro PREWHERE _part_starting_offset + _part_offset < 50 ORDER BY Id DESC LIMIT 10; + +DROP ROW POLICY repro_pol ON repro; + +-- The bare virtual column is the PREWHERE filter column itself. +SELECT EventId FROM repro PREWHERE _part_offset ORDER BY Id DESC LIMIT 10; +-- The bare virtual column is both the PREWHERE filter column and a projected column. +SELECT EventId, _part_offset FROM repro PREWHERE _part_offset ORDER BY Id LIMIT 5; + +CREATE ROW POLICY repro_pol ON repro USING _part_offset TO ALL; + +-- The bare virtual column is the row-level filter column itself. +SELECT EventId FROM repro ORDER BY Id DESC LIMIT 10; +-- The bare virtual column is both the row-level and the PREWHERE filter column. +SELECT EventId FROM repro PREWHERE _part_offset ORDER BY Id DESC LIMIT 10; +-- The bare virtual column is the row-level filter column and a PREWHERE expression input. +SELECT EventId FROM repro PREWHERE _part_offset < 100 ORDER BY Id DESC LIMIT 10; + +DROP ROW POLICY repro_pol ON repro; +DROP TABLE repro; From f56b6c22dfce4a5e1888c210fd10bbcb5388dad3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 13:02:43 +0000 Subject: [PATCH 30/86] Backport #110299 to 26.6: Set `Content-Length` explicitly for Azure requests in the Poco HTTP transport --- src/IO/AzureBlobStorage/PocoHTTPClient.cpp | 12 ++ .../tests/gtest_poco_azure_http_client.cpp | 159 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 src/IO/AzureBlobStorage/tests/gtest_poco_azure_http_client.cpp diff --git a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp index 105358a576c7..9074fcc96b0f 100644 --- a/src/IO/AzureBlobStorage/PocoHTTPClient.cpp +++ b/src/IO/AzureBlobStorage/PocoHTTPClient.cpp @@ -348,6 +348,18 @@ std::unique_ptr PocoAzureHTTPClient::makeRequest poco_request.set(header.name, header.value); } + /// Some SDK clients (e.g. Key Vault) do not set the `Content-Length` header themselves + /// and rely on the transport to compute it from the body stream (the removed curl-based + /// transport did that). Without it the request body is sent with no framing at all, + /// and Azure responds with `411 Length Required`. Body-less requests get a `NullBodyStream`, + /// so mirror the curl transport and skip only the methods that never carry a body. + if (method != "GET" && method != "HEAD" + && !poco_request.has(Poco::Net::HTTPRequest::CONTENT_LENGTH)) + { + if (const auto * request_body_stream = request.GetBodyStream()) + poco_request.setContentLength(request_body_stream->Length()); + } + if (method == "GET" || method == "HEAD") request_throttler.throttleHTTPGet(); else if (method == "PUT" || method == "POST" || method == "PATCH") diff --git a/src/IO/AzureBlobStorage/tests/gtest_poco_azure_http_client.cpp b/src/IO/AzureBlobStorage/tests/gtest_poco_azure_http_client.cpp new file mode 100644 index 000000000000..4a433e55eea4 --- /dev/null +++ b/src/IO/AzureBlobStorage/tests/gtest_poco_azure_http_client.cpp @@ -0,0 +1,159 @@ +#include + +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace +{ + +struct CapturedRequest +{ + bool has_content_length = false; + Int64 content_length = -1; + std::string body; +}; + +class CapturingRequestHandler : public Poco::Net::HTTPRequestHandler +{ + CapturedRequest & captured; + +public: + explicit CapturingRequestHandler(CapturedRequest & captured_) + : captured(captured_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + captured.has_content_length = request.hasContentLength(); + captured.content_length = request.hasContentLength() ? request.getContentLength64() : -1; + captured.body.clear(); + Poco::StreamCopier::copyToString(request.stream(), captured.body); + + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); + response.setContentType("application/json"); + response.setContentLength(2); + response.send() << "{}"; + } +}; + +class CapturingRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ + CapturedRequest & captured; + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new CapturingRequestHandler(captured); + } + +public: + explicit CapturingRequestHandlerFactory(CapturedRequest & captured_) + : captured(captured_) + { + } +}; + +} + +/// The Key Vault SDK clients do not set `Content-Length` themselves and rely on the transport +/// to compute it from the body stream (the removed curl-based transport did that). A request +/// sent without it has no body framing at all and Azure rejects it with `411 Length Required`. +TEST(PocoAzureHTTPClient, SetsContentLengthFromBodyStream) +{ + CapturedRequest captured; + Poco::Net::ServerSocket server_socket(Poco::Net::SocketAddress("127.0.0.1", 0)); + Poco::Net::HTTPServer server( + new CapturingRequestHandlerFactory(captured), server_socket, new Poco::Net::HTTPServerParams); + server.start(); + + DB::RemoteHostFilter remote_host_filter; + DB::PocoAzureHTTPClient client(DB::PocoAzureHTTPClientConfiguration{ + .remote_host_filter = remote_host_filter, + .max_redirects = 3, + .for_disk_azure = false, + .request_throttler = {}, + .extra_headers = {}, + }); + + const std::string body = R"({"alg":"RSA-OAEP-256","value":"dGVzdA"})"; + const auto url = fmt::format("http://{}/keys/test-key/decrypt", server_socket.address().toString()); + + /// No `Content-Length` in the SDK request: the transport must synthesize it. + { + Azure::Core::IO::MemoryBodyStream body_stream(reinterpret_cast(body.data()), body.size()); + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, Azure::Core::Url(url), &body_stream); + request.SetHeader("Content-Type", "application/json"); + + auto response = client.Send(request, Azure::Core::Context()); + + EXPECT_EQ(static_cast(response->GetStatusCode()), 200); + EXPECT_TRUE(captured.has_content_length); + EXPECT_EQ(captured.content_length, static_cast(body.size())); + EXPECT_EQ(captured.body, body); + } + + /// An explicitly set `Content-Length` (e.g. by the blob storage or identity clients) + /// is passed through unchanged. + { + captured = {}; + Azure::Core::IO::MemoryBodyStream body_stream(reinterpret_cast(body.data()), body.size()); + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, Azure::Core::Url(url), &body_stream); + request.SetHeader("Content-Type", "application/json"); + request.SetHeader("Content-Length", std::to_string(body.size())); + + auto response = client.Send(request, Azure::Core::Context()); + + EXPECT_EQ(static_cast(response->GetStatusCode()), 200); + EXPECT_TRUE(captured.has_content_length); + EXPECT_EQ(captured.content_length, static_cast(body.size())); + EXPECT_EQ(captured.body, body); + } + + /// A body-carrying method with an empty body still gets `Content-Length: 0`, + /// like the curl-based transport did. + { + captured = {}; + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, Azure::Core::Url(url)); + + auto response = client.Send(request, Azure::Core::Context()); + + EXPECT_EQ(static_cast(response->GetStatusCode()), 200); + EXPECT_TRUE(captured.has_content_length); + EXPECT_EQ(captured.content_length, 0); + EXPECT_TRUE(captured.body.empty()); + } + + /// `GET` never carries a body and must not get a synthesized header. + { + captured = {}; + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Get, Azure::Core::Url(url)); + + auto response = client.Send(request, Azure::Core::Context()); + + EXPECT_EQ(static_cast(response->GetStatusCode()), 200); + EXPECT_FALSE(captured.has_content_length); + EXPECT_TRUE(captured.body.empty()); + } + + server.stop(); +} + +#endif From f78c271aa9b6470ea01f93e4f601f01488dcf346 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 15:32:41 +0000 Subject: [PATCH 31/86] Backport #110722 to 26.6: Apply lazy materialization to FINAL with a filter and LIMIT without ORDER BY --- .../QueryPlan/JoinLazyColumnsStep.cpp | 7 ++ .../optimizeLazyMaterialization.cpp | 69 ++++++++++---- .../lazy_materialization_final_limit.xml | 24 +++++ ...rialization_final_limit_no_order.reference | 14 +++ ...y_materialization_final_limit_no_order.sql | 93 +++++++++++++++++++ 5 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 tests/performance/lazy_materialization_final_limit.xml create mode 100644 tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.reference create mode 100644 tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.sql diff --git a/src/Processors/QueryPlan/JoinLazyColumnsStep.cpp b/src/Processors/QueryPlan/JoinLazyColumnsStep.cpp index f9e3141b8e47..928915e59bd2 100644 --- a/src/Processors/QueryPlan/JoinLazyColumnsStep.cpp +++ b/src/Processors/QueryPlan/JoinLazyColumnsStep.cpp @@ -24,6 +24,13 @@ QueryPipelineBuilderPtr JoinLazyColumnsStep::updatePipeline(QueryPipelineBuilder if (pipelines.size() != 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "JoinLazyColumnsStep must have two pipelines"); + /// The transform joins exactly one main and one lazy stream. The main branch is not + /// necessarily single-stream (e.g. a limit without a sorting step above the reading). + if (pipelines[0]->getNumStreams() > 1) + pipelines[0]->resize(1); + if (pipelines[1]->getNumStreams() > 1) + pipelines[1]->resize(1); + auto transform = std::make_shared(input_headers.front(), input_headers.back(), lazy_materializing_rows, dataflow_cache_updater); transform->setPassThrough(pass_through); return QueryPipelineBuilder::mergePipelines(std::move(pipelines[0]), std::move(pipelines[1]), transform, &processors); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp b/src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp index 6130ac782d5a..1c8da68a8cb9 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp @@ -348,6 +348,22 @@ static SplitFilterResult splitFilterStep(const FilterStep & filter_step, const s break; } } + + /// The filter column is computed and consumed by the main half; in the lazy half it can + /// only be a dangling input pass-through, and the column does not exist downstream of the + /// main filter. Remove it here: `removeDanglingNodes` is not applied to the last lazy DAG + /// (its pass-through outputs form the final header), so a filter at the bottom of the + /// chain would otherwise keep an input that no block provides. + auto & lazy_outputs = split_result.second.getOutputs(); + for (size_t i = 0; i < lazy_outputs.size(); ++i) + { + if (lazy_outputs[i]->result_name == name) + { + lazy_outputs.erase(lazy_outputs.begin() + i); + split_result.second.removeUnusedActions(); + break; + } + } } FilterDAGInfo filter_dag_info; @@ -439,13 +455,24 @@ bool optimizeLazyMaterialization2(QueryPlan::Node & root, QueryPlan & query_plan return false; auto * sorting_step = typeid_cast(root.children.front()->step.get()); - if (!sorting_step) - return false; + bool reading_in_order = false; - if (sorting_step->getType() != SortingStep::Type::Full && sorting_step->getType() != SortingStep::Type::FinishSorting) - return false; + /// The chain of Expression/Filter steps down to the reading step starts right below + /// the sorting step, or right below the limit when there is no sorting. The latter is + /// allowed only for FINAL with a filter (checked below): the filter cannot run before + /// the FINAL merge, so all columns are read for every scanned row until the limit is + /// reached, and deferring the unneeded ones pays off the same way as with a sort. + QueryPlan::Node * chain_top_node = root.children.front(); - bool reading_in_order = sorting_step->getType() == SortingStep::Type::FinishSorting; + if (sorting_step) + { + if (sorting_step->getType() != SortingStep::Type::Full && sorting_step->getType() != SortingStep::Type::FinishSorting) + return false; + + reading_in_order = sorting_step->getType() == SortingStep::Type::FinishSorting; + + chain_top_node = root.children.front()->children.front(); + } const auto limit = limit_step->getLimit(); if (limit == 0 || (max_limit_for_lazy_materialization != 0 && limit > max_limit_for_lazy_materialization)) @@ -453,29 +480,29 @@ bool optimizeLazyMaterialization2(QueryPlan::Node & root, QueryPlan & query_plan StepStack steps_to_update; - auto * sorting_node = root.children.front(); - auto * reading_step = findReadingStep(*sorting_node->children.front(), steps_to_update); + auto * reading_step = findReadingStep(*chain_top_node, steps_to_update); if (!reading_step) return false; if (!canUseLazyMaterializationForReadingStep(reading_step)) return false; - if (!allExpressionsSuitableForLazyMaterialization(sorting_node->children.front())) + if (!allExpressionsSuitableForLazyMaterialization(chain_top_node)) return false; - const auto & sorting_header = *sorting_step->getOutputHeader(); + const auto & chain_top_header = *chain_top_node->step->getOutputHeader(); /// At this moment, required_columns are corresponding to output header columns of every step. - std::vector required_columns(sorting_header.columns(), false); + std::vector required_columns(chain_top_header.columns(), false); - for (const auto & descr : sorting_step->getSortDescription()) - required_columns[sorting_header.getPositionByName(descr.column_name)] = true; + if (sorting_step) + for (const auto & descr : sorting_step->getSortDescription()) + required_columns[chain_top_header.getPositionByName(descr.column_name)] = true; bool has_filter = false; std::vector steps_to_split; - auto * node = sorting_node->children.front(); + auto * node = chain_top_node; while (!node->children.empty()) { IQueryPlanStep * step = node->step.get(); @@ -537,6 +564,13 @@ bool optimizeLazyMaterialization2(QueryPlan::Node & root, QueryPlan & query_plan if (reading_in_order && !has_filter) return false; + /// Without a sorting step, defer columns only for FINAL with a filter: the filter cannot + /// be moved to PREWHERE (it would run before the FINAL merge and change its result), so + /// this is the only way to avoid reading all columns for every scanned row. For non-FINAL + /// reads, PREWHERE already covers this shape. + if (!sorting_step && (!read_from_merge_tree->isQueryWithFinal() || !has_filter)) + return false; + std::unique_ptr lazy_reading; { auto initial_header = read_from_merge_tree->getOutputHeader(); @@ -636,9 +670,12 @@ bool optimizeLazyMaterialization2(QueryPlan::Node & root, QueryPlan & query_plan } } - auto new_sorting_step = std::move(root.children.front()->step); // = std::make_unique(main_plan.getCurrentHeader(), sorting_step->getSortDescription(), sorting_step->getLimit(), sorting_step->getSettings()); - new_sorting_step->updateInputHeader(main_plan.getCurrentHeader()); - main_plan.addStep(std::move(new_sorting_step)); + if (sorting_step) + { + auto new_sorting_step = std::move(root.children.front()->step); + new_sorting_step->updateInputHeader(main_plan.getCurrentHeader()); + main_plan.addStep(std::move(new_sorting_step)); + } limit_step->updateInputHeader(main_plan.getCurrentHeader()); main_plan.addStep(std::move(root.step)); diff --git a/tests/performance/lazy_materialization_final_limit.xml b/tests/performance/lazy_materialization_final_limit.xml new file mode 100644 index 000000000000..d9f609152fd5 --- /dev/null +++ b/tests/performance/lazy_materialization_final_limit.xml @@ -0,0 +1,24 @@ + + + + CREATE TABLE lazy_mat_final_limit (k UInt64, v UInt64, payload String) + ENGINE = ReplacingMergeTree + ORDER BY k + + + SYSTEM STOP MERGES lazy_mat_final_limit + INSERT INTO lazy_mat_final_limit SELECT number, cityHash64(number, 1) % 10000, randomPrintableASCII(2000) FROM numbers_mt(500000) + INSERT INTO lazy_mat_final_limit SELECT number, cityHash64(number, 2) % 10000, randomPrintableASCII(2000) FROM numbers_mt(500000) + + SELECT k, payload FROM lazy_mat_final_limit FINAL WHERE v = 7 LIMIT 10 FORMAT Null + SELECT k, payload FROM lazy_mat_final_limit FINAL WHERE v = 7 LIMIT 100 FORMAT Null + SELECT k, payload FROM lazy_mat_final_limit FINAL WHERE v = 7 ORDER BY k LIMIT 10 FORMAT Null + + DROP TABLE IF EXISTS lazy_mat_final_limit + diff --git a/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.reference b/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.reference new file mode 100644 index 000000000000..eb1dc50e5dbc --- /dev/null +++ b/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.reference @@ -0,0 +1,14 @@ +FINAL filter limit: 1 +no FINAL: 0 +FINAL no filter: 0 +FINAL no limit: 0 +FINAL big limit: 0 +FINAL prewhere limit: 1 +winners: 10 10 10 +prewhere winners: 10 10 10 +full set equal: 1 +prewhere full set equal: 1 +is_deleted plan: 1 +is_deleted rows: 875 0 +row policy plan: 1 +row policy rows: 10 10 1 diff --git a/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.sql b/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.sql new file mode 100644 index 000000000000..0b350204d73b --- /dev/null +++ b/tests/queries/0_stateless/04540_lazy_materialization_final_limit_no_order.sql @@ -0,0 +1,93 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: the test checks the shape of the local query plan. + +DROP TABLE IF EXISTS t_lazy_final_limit; +DROP TABLE IF EXISTS t_lazy_final_limit_ver; + +CREATE TABLE t_lazy_final_limit (k UInt64, v UInt64, payload String) ENGINE = ReplacingMergeTree ORDER BY k; + +SYSTEM STOP MERGES t_lazy_final_limit; + +INSERT INTO t_lazy_final_limit SELECT number, if(number < 1000, 7, 999), 'v1_' || toString(number) FROM numbers(100000); +INSERT INTO t_lazy_final_limit SELECT number, if(number < 1000, 7, 999), 'v2_' || toString(number) FROM numbers(100000); + +-- The test relies on settings that are randomized by the test runner: pin them. +SET enable_analyzer = 1, max_threads = 4; +SET query_plan_optimize_lazy_materialization = 1, query_plan_max_limit_for_lazy_materialization = 10000; +SET optimize_move_to_prewhere = 1; + +-- FINAL with a filter and a small LIMIT without ORDER BY: lazy materialization applies +-- (the filter cannot be moved to PREWHERE, so this is the only way to avoid reading +-- all columns for every scanned row). +SELECT 'FINAL filter limit:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL WHERE v = 7 LIMIT 10); + +-- Without FINAL the filter is served by PREWHERE: not applied. +SELECT 'no FINAL:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit WHERE v = 7 LIMIT 10); + +-- Without a filter: not applied. +SELECT 'FINAL no filter:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL LIMIT 10); + +-- Without a limit: not applied. +SELECT 'FINAL no limit:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL WHERE v = 7); + +-- A limit above query_plan_max_limit_for_lazy_materialization: not applied. +SELECT 'FINAL big limit:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL WHERE v = 7 LIMIT 100000); + +-- An explicit PREWHERE is carried by the reading step itself, with no FilterStep above it: +-- lazy materialization applies. +SELECT 'FINAL prewhere limit:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL PREWHERE v = 7 LIMIT 10); + +-- Every returned row must be the FINAL winner (the second insert), keys distinct. +SELECT 'winners:', count(), countIf(payload LIKE 'v2\_%'), uniqExact(k) +FROM (SELECT k, payload FROM t_lazy_final_limit FINAL WHERE v = 7 LIMIT 10); + +SELECT 'prewhere winners:', count(), countIf(payload LIKE 'v2\_%'), uniqExact(k) +FROM (SELECT k, payload FROM t_lazy_final_limit FINAL PREWHERE v = 7 LIMIT 10); + +-- The full matching set must be the same with and without the optimization. +SELECT 'full set equal:', + (SELECT (count(), sum(cityHash64(payload)), sum(k)) FROM (SELECT k, payload FROM t_lazy_final_limit FINAL WHERE v = 7 LIMIT 10000)) + = + (SELECT (count(), sum(cityHash64(payload)), sum(k)) FROM (SELECT k, payload FROM t_lazy_final_limit FINAL WHERE v = 7 LIMIT 10000 SETTINGS query_plan_optimize_lazy_materialization = 0)); + +SELECT 'prewhere full set equal:', + (SELECT (count(), sum(cityHash64(payload)), sum(k)) FROM (SELECT k, payload FROM t_lazy_final_limit FINAL PREWHERE v = 7 LIMIT 10000)) + = + (SELECT (count(), sum(cityHash64(payload)), sum(k)) FROM (SELECT k, payload FROM t_lazy_final_limit FINAL PREWHERE v = 7 LIMIT 10000 SETTINGS query_plan_optimize_lazy_materialization = 0)); + +-- Version and is_deleted columns are handled by the FINAL merge on the main branch. +CREATE TABLE t_lazy_final_limit_ver (k UInt64, ver UInt64, is_del UInt8, v UInt64, payload String) +ENGINE = ReplacingMergeTree(ver, is_del) ORDER BY k; + +SYSTEM STOP MERGES t_lazy_final_limit_ver; + +INSERT INTO t_lazy_final_limit_ver SELECT number, 1, 0, if(number < 1000, 7, 999), 'v1_' || toString(number) FROM numbers(100000); +INSERT INTO t_lazy_final_limit_ver SELECT number, 2, number % 4 = 0, if(number < 1000, 7, 999), 'v2_' || toString(number) FROM numbers(500); + +SELECT 'is_deleted plan:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit_ver FINAL WHERE v = 7 LIMIT 2000); + +-- 1000 matching keys minus 125 deleted by the second insert. +SELECT 'is_deleted rows:', count(), countIf(k < 500 AND payload NOT LIKE 'v2\_%') +FROM (SELECT k, payload FROM t_lazy_final_limit_ver FINAL WHERE v = 7 LIMIT 2000); + +-- A row policy is a reader-embedded filter as well: lazy materialization applies even +-- without WHERE, and the returned rows respect the policy. +CREATE ROW POLICY policy_04540 ON t_lazy_final_limit FOR SELECT USING k < 50000 TO ALL; + +SELECT 'row policy plan:', countIf(explain LIKE '%LazilyReadFromMergeTree%') > 0 +FROM (EXPLAIN SELECT payload FROM t_lazy_final_limit FINAL LIMIT 10); + +SELECT 'row policy rows:', count(), countIf(payload LIKE 'v2\_%'), max(k) < 50000 +FROM (SELECT k, payload FROM t_lazy_final_limit FINAL LIMIT 10); + +DROP ROW POLICY policy_04540 ON t_lazy_final_limit; + +DROP TABLE t_lazy_final_limit; +DROP TABLE t_lazy_final_limit_ver; From 8ef03524f24895568f903db074a525ed1b44431c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Jul 2026 15:34:59 +0000 Subject: [PATCH 32/86] Backport #109374 to 26.6: Fix UNKNOWN_IDENTIFIER on ALTER TABLE ... DROP COLUMN with aliased default expression --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 20 +++++++-- ...r_drop_column_default_with_alias.reference | 2 + ...2_alter_drop_column_default_with_alias.sql | 43 +++++++++++++++++++ ..._analyze_indexes_alias_predicate.reference | 3 ++ ...getree_analyze_indexes_alias_predicate.sql | 21 +++++++++ 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.reference create mode 100644 tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.sql create mode 100644 tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.reference create mode 100644 tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.sql diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 62a3dedc7c8a..fbd3d1d40d0d 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -286,12 +286,16 @@ void QueryAnalyzer::resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & ta scope.table_expressions_in_resolve_process.erase(table_expression.get()); } + /// Collect aliases defined inside the expression (e.g. `f(...) AS a, ..., a`) into the scope + /// before resolution, so that later references to them can be resolved. This must be done for + /// a single expression node too, not only for a list: otherwise an alias defined and later + /// referenced within a standalone expression (such as a column DEFAULT expression checked + /// during `ALTER TABLE ... DROP COLUMN`) is not found and resolution fails with UNKNOWN_IDENTIFIER. + QueryExpressionsAliasVisitor visitor(scope.aliases); + visitor.visit(node); + if (node_type == QueryTreeNodeType::LIST) - { - QueryExpressionsAliasVisitor visitor(scope.aliases); - visitor.visit(node); resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - } else resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -346,6 +350,14 @@ void QueryAnalyzer::resolveConstantExpression(QueryTreeNodePtr & node, const Que scope.table_expressions_in_resolve_process.erase(table_expression.get()); } + /// Collect aliases defined inside the expression (e.g. `f(...) AS a, ..., a`) into the scope + /// before resolution, so that later references to them can be resolved. This mirrors `resolve` + /// above and is needed for a single expression node too, not only for a list: otherwise an alias + /// defined and later referenced within a standalone constant expression (such as a user predicate + /// passed to `mergeTreeAnalyzeIndexes`) is not found and resolution fails with UNKNOWN_IDENTIFIER. + QueryExpressionsAliasVisitor visitor(scope.aliases); + visitor.visit(node); + if (node_type == QueryTreeNodeType::LIST) resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); else diff --git a/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.reference b/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.reference new file mode 100644 index 000000000000..98b1e7301230 --- /dev/null +++ b/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.reference @@ -0,0 +1,2 @@ +1 hello HEL-HELLO +1 prefix/1a/1a7 diff --git a/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.sql b/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.sql new file mode 100644 index 000000000000..b7b84a297ae5 --- /dev/null +++ b/tests/queries/0_stateless/04502_alter_drop_column_default_with_alias.sql @@ -0,0 +1,43 @@ +-- Tests that ALTER TABLE ... DROP COLUMN works when another column has a DEFAULT/MATERIALIZED +-- expression that defines and later references an inline alias (`expr AS a ... a`). +-- Previously the dependency check for the dropped column resolved each remaining default expression +-- as a standalone node, which did not collect its internal aliases, so it failed with UNKNOWN_IDENTIFIER. + +DROP TABLE IF EXISTS t_04502; + +CREATE TABLE t_04502 +( + id UInt64, + src String, + to_drop String DEFAULT '', + aliased String DEFAULT concat(substring(upper(src) AS u, 1, 3), '-', u) +) +ENGINE = MergeTree ORDER BY id; + +ALTER TABLE t_04502 DROP COLUMN to_drop; + +INSERT INTO t_04502 (id, src) VALUES (1, 'hello'); +SELECT id, src, aliased FROM t_04502 ORDER BY id; + +DROP TABLE t_04502; + +-- A more elaborate case matching the original report: the alias is defined outside a lambda +-- while another alias lives inside the lambda body. + +DROP TABLE IF EXISTS t_04502_nested; + +CREATE TABLE t_04502_nested +( + id UInt64, + url String, + md5 String MATERIALIZED lower(hex(MD5(url))), + s3_url String DEFAULT concat('prefix/', substring(arrayStringConcat(arrayMap(i -> substring(lower(hex(MD5(url))) AS hx, i, 1), range(1, 4))) AS h, 1, 2), '/', h) +) +ENGINE = MergeTree ORDER BY id; + +ALTER TABLE t_04502_nested DROP COLUMN md5; + +INSERT INTO t_04502_nested (id, url) VALUES (1, 'example'); +SELECT id, s3_url FROM t_04502_nested ORDER BY id; + +DROP TABLE t_04502_nested; diff --git a/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.reference b/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.reference new file mode 100644 index 000000000000..e8183f05f5db --- /dev/null +++ b/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.sql b/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.sql new file mode 100644 index 000000000000..0c85b835c1c9 --- /dev/null +++ b/tests/queries/0_stateless/04514_mergetree_analyze_indexes_alias_predicate.sql @@ -0,0 +1,21 @@ +-- Tests that a predicate passed to `mergeTreeAnalyzeIndexes` works when it defines and later +-- references an inline alias (`expr AS a ... a`). The predicate is resolved as a standalone +-- constant expression via `QueryAnalyzer::resolveConstantExpression`, which previously did not +-- collect the expression's internal aliases, so such a predicate failed with UNKNOWN_IDENTIFIER +-- even though the same shape is accepted everywhere else (SELECT, DEFAULT expressions, etc.). + +DROP TABLE IF EXISTS t_04514; + +CREATE TABLE t_04514 (key Int, value Int) ENGINE = MergeTree ORDER BY key; +INSERT INTO t_04514 SELECT number, number + 1000000 FROM numbers(100000); + +-- Sanity check: a plain predicate resolves. +SELECT count() > 0 FROM mergeTreeAnalyzeIndexes(currentDatabase(), t_04514, key = 8193); + +-- Inline alias defined and later referenced inside a nested function. +SELECT count() > 0 FROM mergeTreeAnalyzeIndexes(currentDatabase(), t_04514, concat(substring(toString(key) AS h, 1, 1), h) = '11'); + +-- Alias defined and referenced at the top level of the predicate. +SELECT count() > 0 FROM mergeTreeAnalyzeIndexes(currentDatabase(), t_04514, (key AS a) = a); + +DROP TABLE t_04514; From 65cd808dab941003a6a93d1d34e6dbd5ac80dabb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Jul 2026 00:29:17 +0000 Subject: [PATCH 33/86] Backport #110460 to 26.6: Fix `refresh_parts_interval` and `table_disk` on read-only object-storage disks --- src/Disks/ReadOnlyDiskWrapper.h | 4 + src/Storages/MergeTree/MergeTreeSettings.cpp | 23 ++++-- ...545_read_only_disk_refresh_parts.reference | 2 + .../04545_read_only_disk_refresh_parts.sh | 75 +++++++++++++++++++ 4 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/04545_read_only_disk_refresh_parts.reference create mode 100755 tests/queries/0_stateless/04545_read_only_disk_refresh_parts.sh diff --git a/src/Disks/ReadOnlyDiskWrapper.h b/src/Disks/ReadOnlyDiskWrapper.h index e75dd8623d24..784d84b0655c 100644 --- a/src/Disks/ReadOnlyDiskWrapper.h +++ b/src/Disks/ReadOnlyDiskWrapper.h @@ -37,6 +37,10 @@ class ReadOnlyDiskWrapper : public IDisk DirectoryIteratorPtr iterateDirectory(const String & path) const override { return delegate->iterateDirectory(path); } + /// Forward refresh() so a read-only replica re-reads object-storage metadata (e.g. `plain_rewritable`) and sees + /// files written by another server. + void refresh(UInt64 not_sooner_than_milliseconds) override { delegate->refresh(not_sooner_than_milliseconds); } + void copyDirectoryContent( const String & from_dir, const std::shared_ptr & to_disk, diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 81a1428a0e3e..a2abf17c6b36 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -1774,8 +1774,11 @@ namespace ErrorCodes Name of storage disk. Can be specified instead of storage policy. )", 0) \ DECLARE(Bool, table_disk, false, R"( - This is table disk, the path/endpoint should point to the table data, not to - the database data. Can be set only for s3_plain/s3_plain_rewritable/web. + This is table disk: the path/endpoint points to the table data, not the database data. + Supported for object-storage disks whose metadata lives on the object storage itself + (s3_plain, s3_plain_rewritable, web, web_index) and their cached variants. Encrypted + variants are supported only over the writable s3_plain / s3_plain_rewritable disks, not + over the read-only web / web_index disks. )", 0) \ DECLARE(Bool, allow_nullable_key, false, R"( Allow Nullable types as primary keys. @@ -2341,11 +2344,19 @@ static void validateTableDisk(const DiskPtr & disk) { if (!disk) throw Exception(ErrorCodes::BAD_ARGUMENTS, "MergeTree settings `table_disk` requires `disk` setting."); - const auto * disk_object_storage = dynamic_cast(disk.get()); - if (!disk_object_storage) + + const auto description = disk->getDataSourceDescription(); + if (description.type != DataSourceType::ObjectStorage) throw Exception(ErrorCodes::BAD_ARGUMENTS, "MergeTree settings `table_disk` is not supported for non-ObjectStorage disks"); - if (!(disk_object_storage->isReadOnly() || disk_object_storage->isPlain())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "MergeTree settings `table_disk` is not supported for {}", disk_object_storage->getStructure()); + + /// table_disk loads the table straight from the disk root (no database/UUID path), so its metadata must be + /// reconstructable from the object storage alone; random blob keys (Local, Keeper) keep the map elsewhere. + const auto metadata_storage = disk->getMetadataStorage(); + if (metadata_storage->areBlobPathsRandom()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "MergeTree settings `table_disk` is not supported for {}: it requires metadata stored on the object storage.", + description.toString()); } IMPLEMENT_SETTINGS_TRAITS_CUSTOM_IMPL(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS, MergeTreeSettings, MergeTreeSetting) diff --git a/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.reference b/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.reference new file mode 100644 index 000000000000..a90d005a270e --- /dev/null +++ b/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.reference @@ -0,0 +1,2 @@ +Hello +local metadata read-only disk rejected for table_disk diff --git a/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.sh b/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.sh new file mode 100755 index 000000000000..6d8a269bc8a0 --- /dev/null +++ b/tests/queries/0_stateless/04545_read_only_disk_refresh_parts.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-object-storage, no-replicated-database, no-shared-merge-tree +# Tag no-random-settings: enable after root causing flakiness +# Tag no-replicated-database: plain rewritable should not be shared between replicas + +# A read-only object-storage disk created via `read_only = true` is wrapped in ReadOnlyDiskWrapper. That wrapper must: +# (1) forward refresh() so that `refresh_parts_interval` picks up parts written by another server; +# (2) not be mistaken for a non-object-storage disk, so `table_disk = true` is accepted. +# Before the fix, creating the reader table failed with "table_disk is not supported for non-ObjectStorage disks", +# and even when created it never observed new parts (logs kept saying "added 0 items"). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS writer SYNC" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS reader SYNC" + +disk_path="disks/04545/${CLICKHOUSE_DATABASE}/" + +# Writer: a read-write plain_rewritable object-storage disk. +${CLICKHOUSE_CLIENT} --query " +CREATE TABLE writer (s String) ORDER BY () +SETTINGS table_disk = true, + disk = disk( + name = 04545_writer_${CLICKHOUSE_DATABASE}, + type = object_storage, + object_storage_type = local, + metadata_type = plain_rewritable, + path = '${disk_path}') +" + +# Reader: the SAME storage, but read-only (wrapped in ReadOnlyDiskWrapper via `read_only = true`). +# This CREATE fails before the fix (issue 2: table_disk validation does not look through the wrapper). +${CLICKHOUSE_CLIENT} --query " +CREATE TABLE reader (s String) ORDER BY () +SETTINGS table_disk = true, refresh_parts_interval = 1, + disk = disk( + read_only = true, + name = 04545_reader_${CLICKHOUSE_DATABASE}, + type = object_storage, + object_storage_type = local, + metadata_type = plain_rewritable, + path = '${disk_path}') +" + +${CLICKHOUSE_CLIENT} --query "INSERT INTO writer VALUES ('Hello')" + +# The read-only reader must observe the newly written part via refresh_parts_interval, without a restart. +# Before the fix (issue 1: refresh() is a no-op on the wrapper) 'Hello' would never appear here. +for _ in {1..300}; do + [ "$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM reader")" = "Hello" ] && break + sleep 0.1 +done + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM reader" + +# Negative: a disk with a LOCAL metadata layer must stay rejected for `table_disk` (its metadata is not +# self-contained on the object storage). Assert the metadata-specific message so the check actually covers the +# metadata-kind predicate: the generic "is not supported for" also matches the older cast-based gate. +${CLICKHOUSE_CLIENT} --query " +CREATE TABLE reader_local (s String) ORDER BY () +SETTINGS table_disk = true, + disk = disk( + read_only = true, + name = 04545_local_${CLICKHOUSE_DATABASE}, + type = object_storage, + object_storage_type = local, + metadata_type = local, + path = 'disks/04545_local/${CLICKHOUSE_DATABASE}/') +" 2>&1 | grep -qF "requires metadata stored on the object storage" && echo "local metadata read-only disk rejected for table_disk" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS reader_local SYNC" +${CLICKHOUSE_CLIENT} --query "DROP TABLE reader SYNC" +${CLICKHOUSE_CLIENT} --query "DROP TABLE writer SYNC" From 498f9dbdba0dc4236db9ebba4e981fc80182cdd0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Jul 2026 08:57:49 +0000 Subject: [PATCH 34/86] Backport #109490 to 26.6: Configurable per-CPU jemalloc arena pool for MergeTree metadata --- contrib/jemalloc | 2 +- programs/local/LocalServer.cpp | 6 + programs/server/Server.cpp | 18 ++ src/Common/AsynchronousMetrics.cpp | 53 +++-- src/Common/CurrentMetrics.cpp | 2 +- src/Common/Jemalloc.h | 18 ++ src/Common/JemallocMergeTreeArena.cpp | 175 +++++++++++++--- src/Common/JemallocMergeTreeArena.h | 57 ++++-- src/Common/PerCPU.cpp | 34 ++++ src/Common/PerCPU.h | 53 +++++ src/Core/ServerSettings.cpp | 1 + src/Interpreters/Context.h | 1 + .../MergeTree/DataPartStorageOnDiskBase.cpp | 20 +- .../MergeTree/DataPartStorageOnDiskFull.cpp | 3 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 190 +++++++++++++----- src/Storages/MergeTree/IMergeTreeDataPart.h | 7 + .../MergeTree/IMergeTreeDataPartWriter.cpp | 7 + src/Storages/MergeTree/MergeTask.cpp | 60 ++++-- src/Storages/MergeTree/MergeTreeData.cpp | 61 ++++-- .../MergeTree/MergeTreeDataPartBuilder.cpp | 5 + .../MergeTreeDataPartWriterOnDisk.cpp | 7 + .../MergeTree/MergeTreeDataWriter.cpp | 15 +- .../MergeTree/MergeTreeIndexGranularity.h | 5 + .../MergeTreeIndexGranularityAdaptive.h | 1 + .../MergeTreeIndexGranularityConstant.h | 1 + .../MergeTree/MergedBlockOutputStream.cpp | 26 ++- src/Storages/MergeTree/MutateTask.cpp | 36 +++- .../ReplicatedMergeTreeRestartingThread.cpp | 6 + src/Storages/StorageMergeTree.cpp | 12 +- src/Storages/StorageReplicatedMergeTree.cpp | 39 +++- .../__init__.py | 0 .../configs/capped.xml | 3 + .../configs/disabled.xml | 3 + .../configs/pool.xml | 7 + .../configs/single.xml | 3 + .../test_jemalloc_merge_tree_arenas/test.py | 181 +++++++++++++++++ ...8_system_parts_index_granularity.reference | 4 +- .../03268_system_parts_index_granularity.sql | 5 +- 38 files changed, 952 insertions(+), 175 deletions(-) create mode 100644 src/Common/PerCPU.cpp create mode 100644 src/Common/PerCPU.h create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/__init__.py create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/configs/capped.xml create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/configs/disabled.xml create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/configs/pool.xml create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/configs/single.xml create mode 100644 tests/integration/test_jemalloc_merge_tree_arenas/test.py diff --git a/contrib/jemalloc b/contrib/jemalloc index b0f2213ab254..e6cea775d316 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit b0f2213ab254e6e5c03717726997e27575c7361d +Subproject commit e6cea775d316273b05581f6a7c3609be649eb754 diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b5b2c9ecb065..efaa6c71a527 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -130,6 +131,7 @@ namespace ServerSetting extern const ServerSettingsBool jemalloc_enable_background_threads; extern const ServerSettingsBool jemalloc_enable_global_profiler; extern const ServerSettingsUInt64 jemalloc_max_background_threads_num; + extern const ServerSettingsUInt64 jemalloc_merge_tree_arenas; extern const ServerSettingsUInt64 jemalloc_profiler_sampling_rate; extern const ServerSettingsUInt64 compiled_expression_cache_elements_size; extern const ServerSettingsUInt64 compiled_expression_cache_size; @@ -359,6 +361,10 @@ void LocalServer::initialize(Poco::Util::Application & self) server_settings[ServerSetting::jemalloc_profiler_sampling_rate]); #endif + /// Create the dedicated MergeTree metadata arena pool before any parts are loaded, same as the + /// server. Without this `clickhouse-local` would ignore `jemalloc_merge_tree_arenas`. + JemallocMergeTreeArena::initialize(server_settings[ServerSetting::jemalloc_merge_tree_arenas]); + GlobalThreadPool::initialize( server_settings[ServerSetting::max_thread_pool_size], server_settings[ServerSetting::max_thread_pool_free_size], diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 35e8cc701b8f..d1420a9b0429 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -140,6 +140,7 @@ #include #include +#include #include "config.h" #include @@ -407,6 +408,7 @@ namespace ServerSetting extern const ServerSettingsBool skip_binary_checksum_checks; extern const ServerSettingsBool abort_on_logical_error; extern const ServerSettingsUInt64 jemalloc_flush_profile_interval_bytes; + extern const ServerSettingsUInt64 jemalloc_merge_tree_arenas; extern const ServerSettingsBool jemalloc_flush_profile_on_memory_exceeded; extern const ServerSettingsUInt64 jemalloc_flush_profile_on_memory_exceeded_interval; extern const ServerSettingsString allowed_disks_for_table_engines; @@ -1761,6 +1763,22 @@ try server_settings.loadSettingsFromConfig(config()); global_context->configureServerWideThrottling(); + /// Create the dedicated MergeTree metadata arena pool. Placed after the ZooKeeper-include reload + /// above so a `from_zk` value of the setting is honored, and still well before any parts are loaded. + JemallocMergeTreeArena::initialize(server_settings[ServerSetting::jemalloc_merge_tree_arenas]); + const size_t created_arenas = JemallocMergeTreeArena::getArenaIndices().size(); + const size_t intended_arenas = JemallocMergeTreeArena::getIntendedArenaCount(); + if (created_arenas < intended_arenas) + { + global_context->addOrUpdateWarningMessage( + Context::WarningType::MERGE_TREE_JEMALLOC_ARENA_POOL_DEGRADED, + PreformattedMessage::create( + "Could only create {} of the {} requested dedicated jemalloc arena(s) for MergeTree metadata; {}.", + created_arenas, intended_arenas, + created_arenas > 0 ? "the pool runs with the created arenas" + : "MergeTree metadata falls back to the default arenas")); + } + #if defined(OS_LINUX) if (server_settings[ServerSetting::skip_binary_checksum_checks]) { diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index a507a4fbfedd..a2e131313893 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -1232,21 +1232,44 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) /// `loadChecksums` and from `MergeTreeDataPartBuilder::build`), /// - per-table metadata (allocations from `MergeTreeData::setProperties`, /// `resetSerializationHints`, `updateSerializationHints`). + new_values["jemalloc.mergetree_arena.count"] = { JemallocMergeTreeArena::getArenaIndices().size(), + "Number of dedicated jemalloc arenas for long-lived MergeTree metadata, controlled by the " + "`jemalloc_merge_tree_arenas` server setting. 0 means the dedicated arena pool is disabled and " + "metadata is allocated in the default per-CPU arenas; 1 is a single shared arena; N > 1 is a " + "pool sharded by CPU. See `jemalloc.mergetree_arena.active_bytes`." }; + if (JemallocMergeTreeArena::isEnabled()) { - unsigned mergetree_arena = JemallocMergeTreeArena::getArenaIndex(); - auto mt_pactive = saveJemallocMetricImpl(new_values, - fmt::format("stats.arenas.{}.pactive", mergetree_arena), - "jemalloc.mergetree_arena.pactive"); - auto mt_pdirty = saveJemallocMetricImpl(new_values, - fmt::format("stats.arenas.{}.pdirty", mergetree_arena), - "jemalloc.mergetree_arena.pdirty"); + /// Sum across every arena in the pool. + size_t mt_pactive = 0; + size_t mt_pdirty = 0; + bool read_ok = true; + for (unsigned arena : JemallocMergeTreeArena::getArenaIndices()) + { + size_t pactive = 0; + size_t pdirty = 0; + if (!Jemalloc::tryGetValue(fmt::format("stats.arenas.{}.pactive", arena).c_str(), pactive) + || !Jemalloc::tryGetValue(fmt::format("stats.arenas.{}.pdirty", arena).c_str(), pdirty)) + { + read_ok = false; + break; + } + mt_pactive += pactive; + mt_pdirty += pdirty; + } - if (mt_pactive && mt_pdirty) + /// Publish only complete sums: a partial prefix would look like a plausible pool total. + if (read_ok) { + new_values["jemalloc.mergetree_arena.pactive"] = { mt_pactive, + "Active pages summed across the dedicated jemalloc MergeTree arena pool." }; + new_values["jemalloc.mergetree_arena.pdirty"] = { mt_pdirty, + "Dirty pages summed across the dedicated jemalloc MergeTree arena pool." }; + const size_t page_size = jemalloc_page_size_mib.getValue(); - new_values["jemalloc.mergetree_arena.active_bytes"] = { *mt_pactive * page_size, - "Active bytes in the dedicated jemalloc MergeTree arena. Holds long-lived MergeTree heap " + new_values["jemalloc.mergetree_arena.active_bytes"] = { mt_pactive * page_size, + "Active bytes summed across the dedicated jemalloc MergeTree arena pool " + "(`jemalloc.mergetree_arena.count` arenas). Holds long-lived MergeTree heap " "state: per-part metadata (`NamesAndTypesList`, `SerializationInfoByName`, the " "`serializations` map, `column_name_to_position`, `MergeTreeDataPartChecksums` tree, the " "`Poco::LRUCache` delegates inside each `IMergeTreeDataPart`, the " @@ -1258,10 +1281,12 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) "contribute. Disjoint from the cache arena and JIT arena. The per-part columns " "`system.parts.primary_key_bytes_in_memory[_allocated]` and " "`system.parts.index_granularity_bytes_in_memory[_allocated]` are subsets of this metric " - "(when their values are non-zero — they can also live in `PrimaryIndexCacheBytes` instead, " - "which is in the cache arena and not counted here)."}; - new_values["jemalloc.mergetree_arena.dirty_bytes"] = { *mt_pdirty * page_size, - "Dirty bytes in the MergeTree arena that are eligible for purging back to the OS."}; + "(when their values are non-zero). The primary index is allocated here even when it is " + "owned by `PrimaryIndexCache` (deliberate: re-homing it at the cache boundary could fail " + "after a part is already committed), so `PrimaryIndexCacheBytes` overlaps with this metric."}; + new_values["jemalloc.mergetree_arena.dirty_bytes"] = { mt_pdirty * page_size, + "Dirty bytes summed across the dedicated jemalloc MergeTree arena pool that are eligible " + "for purging back to the OS."}; } } #endif diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index cdb399e249ce..c89c1bd95b78 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -364,7 +364,7 @@ M(DeleteBitmapCacheBytes, "Total size of the UNIQUE KEY delete-bitmap cache in bytes") \ M(DeleteBitmapCacheEntries, "Total number of UNIQUE KEY delete bitmaps cached") \ M(NamedCollection, "Number of named collections") \ - M(PrimaryIndexCacheBytes, "Total size of primary index cache in bytes. Holds primary-key indices loaded on demand when `primary_key_lazy_load=1` and `use_primary_key_cache=1`. Allocations live in the dedicated cache jemalloc arena (`jemalloc.cache_arena.*`). NEVER overlaps with `system.parts.primary_key_bytes_in_memory[_allocated]` — a part's index lives either in this cache (counted here) or in the part itself (counted there); never both. To get total primary-index memory across all parts, sum the two.") \ + M(PrimaryIndexCacheBytes, "Total size of primary index cache in bytes. Holds primary-key indices loaded on demand when `primary_key_lazy_load=1` and `use_primary_key_cache=1`. When the dedicated MergeTree metadata arena pool is enabled (`jemalloc_merge_tree_arenas` > 0 with jemalloc), the index columns are allocated there (`jemalloc.mergetree_arena.*`) even when owned by this cache. NEVER overlaps with `system.parts.primary_key_bytes_in_memory[_allocated]` — a part's index lives either in this cache (counted here) or in the part itself (counted there); never both. To get total primary-index memory across all parts, sum the two.") \ M(PrimaryIndexCacheFiles, "Total number of index files cached in the primary index cache") \ M(PageCacheBytes, "Total size of userspace page cache in bytes") \ M(PageCacheCells, "Total number of entries in the userspace page cache") \ diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 17214687f969..c85c2931a45b 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -3,6 +3,7 @@ #include "config.h" #include +#include namespace DB::Jemalloc { @@ -203,3 +204,20 @@ class ScopedJemallocThreadArena } #endif + +namespace DB +{ + +/// Replace `value` with a fresh copy of itself and drop the original. Because the copy is a new +/// allocation, its storage is served by whatever arena the calling thread is currently bound to. +/// This does not select an arena itself: enter a `ScopedJemallocThreadArena` scope first, then call +/// this to re-home an already-built object into that arena. Pure copy + move, so it is harmless and +/// works whether or not jemalloc is compiled in. +template +void reallocateByCopy(T & value) +{ + auto copy = value; + value = std::move(copy); +} + +} diff --git a/src/Common/JemallocMergeTreeArena.cpp b/src/Common/JemallocMergeTreeArena.cpp index ca121a187cd9..926c196ee1fb 100644 --- a/src/Common/JemallocMergeTreeArena.cpp +++ b/src/Common/JemallocMergeTreeArena.cpp @@ -5,13 +5,23 @@ #if USE_JEMALLOC #include +#include #include #include #include +#include +#include +#include +#include + #include #include -#include + +#if defined(OS_LINUX) +#include +#include +#endif namespace ProfileEvents { @@ -25,51 +35,161 @@ namespace DB::JemallocMergeTreeArena namespace { -struct ArenaState -{ - unsigned index; - bool enabled; -}; - -ArenaState createArena() +/// Written once by `initialize` before `initialized` is published; read-only afterwards. +std::vector arena_indices; +/// Maps an absolute CPU id (from `getCurrentCPU`) to a slot in `arena_indices`. Sized to the +/// highest allowed CPU id. Built so every created arena is reachable regardless of the mask. +std::vector slot_by_cpu; +/// Number of arenas `initialize` tried to create (configured count capped at the allowed CPUs); +/// `arena_indices.size()` may be smaller if `arenas.create` failed. +size_t intended_arena_count = 0; +std::atomic initialized = false; + +std::optional createArena() { unsigned arena_index = 0; size_t arena_index_size = sizeof(arena_index); int err = je_mallctl("arenas.create", &arena_index, &arena_index_size, nullptr, 0); if (err) { - /// Don't throw: this arena is a fragmentation optimization, not a correctness prerequisite. - /// `getArenaIndex` is on every part-loading and table-creating hot path; throwing here - /// would brick startup and every CREATE / INSERT / merge. Fall back to arena 0 — passing 0 - /// to `ScopedJemallocThreadArena` is already a documented no-op that allocates from the - /// default arena. The cost is degraded fragmentation, not correctness. LOG_ERROR( &Poco::Logger::get("JemallocMergeTreeArena"), "Failed to create dedicated jemalloc MergeTree arena (mallctl error: {}). " - "MergeTree allocations will use the default arena and the fragmentation reduction " - "documented for `jemalloc.mergetree_arena.*` is disabled.", + "The pool continues with the arenas created so far (the default arenas if none).", err); - return {0, false}; + return {}; } - return {arena_index, true}; + return arena_index; } -const ArenaState & state() +/// CPUs this process may run on, in ascending order. Honors the affinity mask on Linux, so a +/// cpuset-limited process only routes to the CPUs it actually uses; falls back to +/// [0, getNumCPUs()) elsewhere. +std::vector getAllowedCPUs() { - static const ArenaState s = createArena(); - return s; + std::vector cpus; +#if defined(OS_LINUX) + /// The fixed `cpu_set_t` holds only `__CPU_SETSIZE` (1024) CPUs, so `sched_getaffinity` fails + /// with EINVAL once the mask covers a CPU id >= 1024. Grow a dynamically-allocated mask until it + /// fits, so pools on machines with many CPUs (or cpusets pinned to high ids) still map every + /// allowed CPU to a reachable arena. Runs once, at startup. + /// + /// The mask is a plain 64-bit-word array (`cpu_set_t` is an array of `unsigned long`, which is + /// 64-bit on the supported Linux platforms) rather than `CPU_ALLOC`, because `CPU_ALLOC` / + /// `CPU_FREE` resolve to `__sched_cpualloc` / `__sched_cpufree` which are only available since + /// GLIBC_2.7 and would break the release-binary compatibility check (max allowed GLIBC 2.4). + /// `CPU_ALLOC_SIZE` and `CPU_ISSET_S` are macros with no such dependency. `UInt64` gives the + /// alignment `cpu_set_t` needs. + for (size_t num_cpus = 1024; num_cpus <= (size_t{1} << 20); num_cpus *= 2) + { + const size_t set_size = CPU_ALLOC_SIZE(num_cpus); + std::vector mask((set_size + sizeof(UInt64) - 1) / sizeof(UInt64)); + auto * set = reinterpret_cast(mask.data()); + if (sched_getaffinity(0, set_size, set) == 0) + { + for (UInt32 cpu = 0; cpu < num_cpus; ++cpu) + { + if (CPU_ISSET_S(cpu, set_size, set)) + cpus.push_back(cpu); + } + break; + } + if (errno != EINVAL) /// EINVAL means the set was too small; anything else is a real error. + break; + } + /// On a real affinity-read failure, return empty rather than fabricating [0, getNumCPUs()): + /// a guessed mask could map allowed CPUs onto only a few slots while unreachable arenas still + /// count in the pool. The caller fails closed to a single arena, reachable from any CPU. +#else + for (UInt32 cpu = 0; cpu < PerCPU::getNumCPUs(); ++cpu) + cpus.push_back(cpu); +#endif + return cpus; +} + } +void initialize(size_t num_arenas) +{ + /// Startup-only; ignore repeated calls (e.g. a config reload). + if (initialized.load(std::memory_order_acquire)) + return; + + if (num_arenas > 0) + { + /// Size the pool to the CPUs we can actually route to and build a dense CPU->slot map, so + /// every created arena receives allocations even under a restrictive or sparse affinity + /// mask (`cpu_id % N` alone would leave arenas unreachable and overstate the count). + std::vector allowed_cpus = getAllowedCPUs(); + if (allowed_cpus.empty()) + { + /// Affinity discovery failed; fail closed to one shared arena (reachable from any CPU, + /// no CPU map needed) instead of guessing a map that could leave arenas unreachable. + LOG_WARNING( + &Poco::Logger::get("JemallocMergeTreeArena"), + "Cannot determine the CPUs this process may run on; " + "using a single dedicated MergeTree arena instead of {}.", + num_arenas); + allowed_cpus.push_back(0); + } + num_arenas = std::min(num_arenas, allowed_cpus.size()); + intended_arena_count = num_arenas; + + std::vector indices; + indices.reserve(num_arenas); + for (size_t i = 0; i < num_arenas; ++i) + { + auto index = createArena(); + if (!index) + break; /// Keep whatever we managed to create; the rest falls back to the default arena. + indices.push_back(*index); + } + + if (!indices.empty()) + { + /// Size the map to the highest allowed CPU id (ids are ascending), not a fixed cap, so + /// hosts with many CPUs / high-id cpusets still map every allowed CPU to a real slot. + slot_by_cpu.assign(static_cast(allowed_cpus.back()) + 1, 0); + for (size_t dense = 0; dense < allowed_cpus.size(); ++dense) + slot_by_cpu[allowed_cpus[dense]] = static_cast(dense % indices.size()); + } + + arena_indices = std::move(indices); + } + + initialized.store(true, std::memory_order_release); } unsigned getArenaIndex() { - return state().index; + if (!initialized.load(std::memory_order_acquire)) + return 0; + + const size_t n = arena_indices.size(); + if (n == 0) + return 0; + if (n == 1) + return arena_indices[0]; + + const Int32 cpu = PerCPU::getCurrentCPU(); + if (cpu < 0 || static_cast(cpu) >= slot_by_cpu.size()) + return arena_indices[0]; + return arena_indices[slot_by_cpu[cpu]]; +} + +const std::vector & getArenaIndices() +{ + return arena_indices; +} + +size_t getIntendedArenaCount() +{ + return intended_arena_count; } bool isEnabled() { - return state().enabled; + return initialized.load(std::memory_order_acquire) && !arena_indices.empty(); } void purge() @@ -77,10 +197,12 @@ void purge() if (!isEnabled()) return; - static Jemalloc::MibCache purge_mib(fmt::format("arena.{}.purge", getArenaIndex()).c_str()); - Stopwatch watch; - purge_mib.run(); + for (unsigned index : arena_indices) + { + Jemalloc::MibCache purge_mib(fmt::format("arena.{}.purge", index).c_str()); + purge_mib.run(); + } ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); } @@ -92,7 +214,10 @@ void purge() namespace DB::JemallocMergeTreeArena { +void initialize(size_t) {} unsigned getArenaIndex() { return 0; } +const std::vector & getArenaIndices() { static const std::vector empty; return empty; } +size_t getIntendedArenaCount() { return 0; } bool isEnabled() { return false; } void purge() {} diff --git a/src/Common/JemallocMergeTreeArena.h b/src/Common/JemallocMergeTreeArena.h index d506e305c17e..b354a9e5c1dd 100644 --- a/src/Common/JemallocMergeTreeArena.h +++ b/src/Common/JemallocMergeTreeArena.h @@ -1,38 +1,51 @@ #pragma once +#include +#include namespace DB::JemallocMergeTreeArena { -/// Returns the jemalloc arena index dedicated to long-lived MergeTree heap state. -/// Holds: +/// Dedicated jemalloc arena(s) for long-lived MergeTree heap state: /// - per-part metadata: `NamesAndTypesList`, `SerializationInfoByName`, the `serializations` /// map, `column_name_to_position`, `MergeTreeDataPartChecksums` tree, `ColumnsSubstreams`, -/// the per-part `Poco::LRUCache(1024)` and its delegates, the -/// `ColumnSize`/`IndexSize` maps, `MinMaxIndex`, `VersionMetadataOnDisk`, -/// `index_granularity_info`, and the primary index / index-granularity arrays themselves. -/// - per-table metadata: the `MergeTreeData` object's mutable schema state — `ColumnsDescription`, -/// `VirtualColumnsDescription`, `StorageInMemoryMetadata` clones, the `serialization_hints` -/// aggregation across active parts, and the `columns_descriptions_cache` populated from -/// `setColumns`. +/// the per-part `ColumnSize`/`IndexSize` maps, `MinMaxIndex`, `index_granularity`, and the +/// primary index arrays. +/// - per-table metadata: `ColumnsDescription`, `VirtualColumnsDescription`, +/// `StorageInMemoryMetadata` clones, the `serialization_hints` aggregation, and the +/// `columns_descriptions_cache`. +/// Isolating these off the default arenas reduces fragmentation of query-lifetime allocations. /// -/// Creates the arena on first call (thread-safe via Meyers singleton). -/// Returns 0 (meaning "use default arena selection") if jemalloc is not available, or if -/// `mallctl("arenas.create", ...)` failed at first call — in which case an error is logged -/// and `isEnabled` returns false. Passing 0 to `ScopedJemallocThreadArena` is a documented -/// no-op, so callers do not need to branch on availability. -/// -/// Callers route allocations into this arena for a tightly-bounded scope by using -/// `ScopedJemallocThreadArena` from `Common/Jemalloc.h`. Frees auto-route via jemalloc's -/// per-extent metadata, so only allocation paths need scoping. +/// Callers route allocations here for a bounded scope via `ScopedJemallocThreadArena` from +/// `Common/Jemalloc.h`. Frees auto-route via jemalloc's per-extent metadata, so only allocation +/// paths need scoping. + +/// Configure the arena pool. Call once at startup, before parts are loaded. Startup-only: +/// subsequent calls are ignored. +/// num_arenas == 0 -> disabled: `getArenaIndex` returns 0 (default arena selection), a no-op. +/// num_arenas == 1 -> one shared arena. +/// num_arenas > 1 -> a per-CPU pool. Capped at the number of CPUs the process may run on (its +/// affinity mask); a large value (or the core count) yields one arena per +/// allowed CPU. A dense CPU->slot map keeps every created arena reachable even +/// under a restrictive or sparse affinity mask. +void initialize(size_t num_arenas); + +/// Arena index for the calling thread's current CPU, or 0 (default arena) when disabled or not +/// yet initialized. Resolved per call from the current CPU (via the CPU->slot map) rather than +/// cached per thread, so a thread that migrates CPUs follows its CPU's arena. unsigned getArenaIndex(); -/// Whether the dedicated MergeTree arena is available (jemalloc compiled in and -/// `arenas.create` succeeded on first call). +/// All arena indices in the pool (empty when disabled). For metrics aggregation and purge. +const std::vector & getArenaIndices(); + +/// Number of arenas the pool intended to create: the configured count capped at the number of +/// allowed CPUs. Compare with `getArenaIndices().size()` to detect arena-creation failures. +size_t getIntendedArenaCount(); + +/// Whether the pool is enabled (at least one arena created). bool isEnabled(); -/// Purge dirty pages only in the MergeTree arena, returning memory to the OS. -/// No-op if the arena is not available (`isEnabled()` returns false). +/// Purge dirty pages in every pool arena, returning memory to the OS. No-op when disabled. void purge(); } diff --git a/src/Common/PerCPU.cpp b/src/Common/PerCPU.cpp new file mode 100644 index 000000000000..6184a68fd905 --- /dev/null +++ b/src/Common/PerCPU.cpp @@ -0,0 +1,34 @@ +#include + +#if defined(OS_LINUX) +#include +#elif defined(OS_DARWIN) +#include +#endif + +#include + +namespace PerCPU +{ + +UInt32 getNumCPUs() noexcept +{ + static const UInt32 cached = [] + { +#if defined(OS_LINUX) + const Int64 n = get_nprocs_conf(); +#elif defined(OS_DARWIN) + const Int64 n = ::sysconf(_SC_NPROCESSORS_ONLN); +#else + /// `getCurrentCPU` is not implemented here, so per-CPU routing is impossible; report one + /// CPU so callers size a single shard instead of creating unreachable ones (e.g. FreeBSD). + const Int64 n = 1; +#endif + if (n <= 0) + return UInt32{1}; + return std::min(static_cast(n), MAX_CPUS); + }(); + return cached; +} + +} diff --git a/src/Common/PerCPU.h b/src/Common/PerCPU.h new file mode 100644 index 000000000000..6d1ba0799692 --- /dev/null +++ b/src/Common/PerCPU.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#if defined(OS_LINUX) +#include +#endif + +namespace PerCPU +{ + +/// Hard upper bound on the kernel cpu_id we'll route to. The BSS-backed per-CPU storage in +/// callers is sized with this constant, so it must be a compile-time value — but only the +/// first `getNumCPUs()` shards are used at runtime (and only those get faulted in). +constexpr UInt32 MAX_CPUS = 1024; + +/// Number of CPUs `getCurrentCPU` can route to, capped at `MAX_CPUS`. Cached on first call. +/// `get_nprocs_conf()` on Linux, `sysconf(_SC_NPROCESSORS_ONLN)` on Darwin; 1 on platforms where +/// `getCurrentCPU` is unimplemented (routing collapses to one shard there) or if unavailable. +UInt32 getNumCPUs() noexcept; + +/// Current CPU id, or -1 if unavailable (callers must treat a negative value as "unknown" and +/// fall back to a fixed shard). The id is not guaranteed to be dense in [0, getNumCPUs()); callers +/// bound it (`cpu % N` or `cpu < N ? cpu : 0`). Cheap on every supported platform (no syscall). +ALWAYS_INLINE inline Int32 getCurrentCPU() +{ +#if defined(OS_LINUX) + /// TLS read via glibc rseq on modern kernels (see glibc-compatibility/musl/sched_getcpu.c). + return sched_getcpu(); +#elif defined(OS_DARWIN) && defined(__aarch64__) + /// macOS has no `sched_getcpu`. XNU exposes the current CPU number to userspace in the low 12 + /// bits of a per-CPU register, extracted exactly as libsyscall's `_os_cpu_number` (up to 4096 + /// CPUs): https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/libsyscall/os/tsd.h + /// The layout is Apple-internal and documented there as "subject to change"; e.g. macOS 11 + /// kept the CPU number in `TPIDRRO_EL0` instead, so there this reads unrelated TLS bits. Callers + /// bound the value, so on such systems the worst case is degraded sharding, not incorrectness. + UInt64 tpidr; + __asm__ volatile("mrs %0, TPIDR_EL0" : "=r"(tpidr)); + return static_cast(tpidr & 0xfff); +#elif defined(OS_DARWIN) && defined(__x86_64__) + /// Same source as above (`_os_cpu_number`): XNU encodes the CPU number in the low 12 bits of the + /// per-CPU IDTR *limit* (the first word `sidt` stores: 16-bit limit + low 48 bits of the base), + /// so masking the first word matches Apple's implementation exactly. + struct { UInt64 limit_and_base_low; UInt64 base_high; } idtr; + __asm__ volatile("sidt %0" : "=m"(idtr)); + return static_cast(idtr.limit_and_base_low & 0xfff); +#else + return -1; +#endif +} + +} diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 1f6eb7e54653..ceb9d58d28e5 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1337,6 +1337,7 @@ The policy on how to perform a scheduling of CPU slots specified by `concurrent_ DECLARE(UInt64, handshake_timeout_milliseconds, 30000, R"(Wall-clock timeout in milliseconds for the entire TCP handshake phase (Hello + Addendum). Limits how long an unauthenticated connection can hold a thread. Set to 0 to disable.)", 0) \ DECLARE(Bool, skip_binary_checksum_checks, false, R"(Skips ClickHouse binary checksum integrity checks)", 0) \ DECLARE(Bool, abort_on_logical_error, false, R"(Crash the server on LOGICAL_ERROR exceptions. Only for experts.)", 0) \ + DECLARE(UInt64, jemalloc_merge_tree_arenas, 1, R"(Number of dedicated jemalloc arenas for long-lived MergeTree per-part and per-table metadata. `0` disables the dedicated arena (metadata uses the default per-CPU arenas). `1` uses a single shared arena. `N > 1` creates a pool of `N` arenas and routes allocations per CPU; on many-core machines this avoids serializing metadata allocation on a single arena's locks. Capped at the number of CPUs the process may run on (its affinity mask), so a large value (or the core count) yields one arena per allowed CPU. Applied at startup.)", 0) \ DECLARE(UInt64, jemalloc_flush_profile_interval_bytes, 0, R"(Flushing jemalloc profile will be done after global peak memory usage increased by jemalloc_flush_profile_interval_bytes)", 0) \ DECLARE(Bool, jemalloc_flush_profile_on_memory_exceeded, 0, R"(Flushing jemalloc profile will be done on total memory exceeded errors)", 0) \ DECLARE(UInt64, jemalloc_flush_profile_on_memory_exceeded_interval, 0, R"(If non-zero, sets the minimum interval in seconds between flushing jemalloc profiles on total memory exceeded errors. For example, 5 means at most one profile flush every 5 seconds. Takes priority over `jemalloc_flush_profile_on_memory_exceeded`.)", 0) \ diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e939d24e0e86..d632c95ca39a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -788,6 +788,7 @@ class Context: public ContextData, public std::enable_shared_from_this MAX_PENDING_MUTATIONS_EXCEEDS_LIMIT, MAX_PENDING_MUTATIONS_OVER_THRESHOLD, MAYBE_BROKEN_TABLES, + MERGE_TREE_JEMALLOC_ARENA_POOL_DEGRADED, OBSOLETE_MONGO_TABLE_DEFINITION, OBSOLETE_SETTINGS, PROCESS_USER_MATCHES_DATA_OWNER, diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index def7f6395860..42e5f905233d 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -542,6 +544,10 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( disk->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME); } + /// The SingleDiskVolume and the DataPartStorageOnDiskFull built by `create` are stored on the + /// frozen part for its whole lifetime; route them into the dedicated MergeTree arena, like the + /// builder-owned storage path. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); /// Do not initialize storage in case of DETACH because part may be broken. @@ -598,6 +604,9 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freezeRemote( dst_disk->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME); } + /// The SingleDiskVolume and the DataPartStorageOnDiskFull built by `create` are stored on the + /// frozen part for its whole lifetime; route them into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); auto single_disk_volume = std::make_shared(dst_disk->getName(), dst_disk, 0); /// Do not initialize storage in case of DETACH because part may be broken. @@ -638,6 +647,9 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::clonePart( throw; } + /// The SingleDiskVolume and the DataPartStorageOnDiskFull built by `create` are stored on the + /// cloned part for its whole lifetime; route them into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); auto single_disk_volume = std::make_shared(dst_disk->getName(), dst_disk, 0); return create(single_disk_volume, to, dir_path, /*initialize=*/ true); } @@ -989,7 +1001,13 @@ void DataPartStorageOnDiskBase::changeRootPath(const std::string & from_root, co if (dst_size > 0 && to_root.back() == '/') --dst_size; - root_path = to_root.substr(0, dst_size) + root_path.substr(prefix_size); + /// `root_path` is part-lifetime metadata of this (arena-owned) storage, so build its new value in + /// the dedicated arena instead of the caller's default arena. Parent-part commit calls this for + /// every projection storage, so otherwise the projection path escapes back to the default arena. + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + root_path = to_root.substr(0, dst_size) + root_path.substr(prefix_size); + } /// See rename: keep a successfully-loaded (path-independent) reader, but clear a stale cached /// miss so the next access re-probes the new path. diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp index 5503d11c58d9..6061ca132972 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp @@ -38,6 +38,9 @@ MutableDataPartStoragePtr DataPartStorageOnDiskFull::create( MutableDataPartStoragePtr DataPartStorageOnDiskFull::getProjection(const std::string & name, bool use_parent_transaction) // NOLINT { + /// Not arena-scoped: most callers use this only as a short-lived filesystem handle (CHECK TABLE, + /// mutation hardlink/copy, existence probes). The part-lifetime projection storage is created via + /// `getProjectionPartBuilder`, which scopes the arena itself. return std::shared_ptr(new DataPartStorageOnDiskFull(volume, std::string(fs::path(root_path) / part_dir), name, use_parent_transaction ? transaction : nullptr)); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5a29ab7a5f91..a5b85ac740d0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -352,6 +352,10 @@ IMergeTreeDataPart::MinMaxIndexPtr IMergeTreeDataPart::getMinMaxIndex() const if (minmax_idx) return minmax_idx; + /// Build the lazily-materialized index in the parts arena. Reloaded parts and the zero-level path + /// that resets a virtual minmax column to null (see MergeTreeDataWriter) first create it here. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + if (is_temp || isEmpty()) { minmax_idx = std::make_shared(); @@ -367,6 +371,15 @@ IMergeTreeDataPart::MinMaxIndexPtr IMergeTreeDataPart::getMinMaxIndex() const void IMergeTreeDataPart::setMinMaxIndex(MinMaxIndexPtr minmax_index) const { + /// Re-home the index into the parts arena (deep copy under the scope, then adopt), same rationale as + /// `setColumns`. The index is one of the larger part-lifetime metadata structures and is built outside + /// the arena on the insert and mutation paths. + if (minmax_index && JemallocMergeTreeArena::isEnabled()) + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + minmax_index = std::make_shared(*minmax_index); + } + std::lock_guard lock(minmax_idx_mutex); minmax_idx = std::move(minmax_index); } @@ -580,6 +593,10 @@ void IMergeTreeDataPart::setIndex(Columns index_columns) if (index) throw Exception(ErrorCodes::LOGICAL_ERROR, "The index of data part can be set only once"); + /// The primary index lives for the part's whole lifetime, so build the `Index` object in the + /// dedicated arena, like `setColumns` / `setMinMaxIndex`. This covers every caller (insert / merge + /// write finalize and the mutation path) in one place. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); optimizeIndexColumns(index_granularity->getMarksCount(), index_columns); index = std::make_shared(std::move(index_columns)); } @@ -1243,18 +1260,24 @@ Estimates IMergeTreeDataPart::getEstimates() const if (estimates.has_value()) return *estimates; - Estimates new_estimates; + /// The raw statistics are transient, so load them in the default arena; only the cached + /// estimates map is long-lived (kept on the part until reload), so build it in the dedicated + /// arena, like the rest of the part's metadata. auto statistics = loadStatistics(); - for (const auto & [column_name, stats] : statistics) - new_estimates.emplace(column_name, stats->getEstimate()); - - estimates = std::move(new_estimates); + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + Estimates new_estimates; + for (const auto & [column_name, stats] : statistics) + new_estimates.emplace(column_name, stats->getEstimate()); + estimates = std::move(new_estimates); + } return *estimates; } void IMergeTreeDataPart::setEstimates(const Estimates & new_estimates) { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); std::lock_guard lock(estimates_mutex); estimates = new_estimates; } @@ -1266,13 +1289,17 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks /// Motivation: memory for index is shared between queries - not belong to the query itself. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; - /// Everything loaded here (columns, substreams, checksums, index granularity, primary index, - /// per-column sizes, rows count, partition / minmax index, TTL infos, projections, default - /// compression codec, source parts set) lives for the whole part lifetime. Route the heap - /// allocations into the dedicated parts arena. This block is on the hot server-startup path - /// (`MergeTreeData::loadDataPart` → `loadColumnsChecksumsIndexes`), so per-part metadata - /// allocated at boot also lands in the arena from the start. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + /// Long-lived per-part metadata (columns substreams, checksums, index granularity, primary + /// index, per-column sizes, partition / minmax index, TTL infos, projections) is routed into + /// the dedicated parts arena by the inner block below. Deliberately kept OUT of that arena: + /// - `loadColumns`: its file read and text/JSON parsing are short-lived scratch; the + /// persistent columns/serializations it produces are arena-scoped inside `setColumns`. + /// - `checkConsistency`: pure file-existence/size verification, allocates nothing persistent. + /// - `loadDefaultCompressionCodec`: a tiny per-part codec pointer. + /// (`loadSourcePartsSet` runs after this block but scopes itself into the arena, as its + /// patch-part metadata is part-lifetime.) + /// These paths churn many short-lived allocations; keeping them in the default per-CPU arenas + /// avoids serializing that churn on the single arena's locks under many concurrent merges. try { @@ -1280,37 +1307,46 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadUUID(); loadColumns(require_columns_checksums, load_metadata_version); - loadColumnsSubstreams(); - loadChecksums(require_columns_checksums); - loadIndexGranularity(); - /// It's important to load index after index granularity. - if (!(*storage.getSettings())[MergeTreeSetting::primary_key_lazy_load]) - index = loadIndex(); + bool has_broken_projections = false; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + + loadColumnsSubstreams(); + loadChecksums(require_columns_checksums); + loadIndexGranularity(); - if (!(*storage.getSettings())[MergeTreeSetting::columns_and_secondary_indices_sizes_lazy_calculation]) - calculateColumnsAndSecondaryIndicesSizesOnDisk(); + /// It's important to load index after index granularity. + if (!(*storage.getSettings())[MergeTreeSetting::primary_key_lazy_load]) + index = loadIndex(); - loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`. + loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`. - /// For constant granularity parts (non-adaptive marks), the last mark granularity - /// is assumed to be a full granule because the mark file does not store per-granule - /// row counts, and the final mark is not distinguished from data marks. - /// Now that we know the actual rows_count, fix the last mark and detect the final mark. - if (auto * constant_granularity = dynamic_cast(index_granularity.get())) - constant_granularity->fixFromRowsCount(rows_count); + /// For constant granularity parts (non-adaptive marks), the last mark granularity + /// is assumed to be a full granule because the mark file does not store per-granule + /// row counts, and the final mark is not distinguished from data marks. + /// Now that we know the actual rows_count, fix the last mark and detect the final mark. + if (auto * constant_granularity = dynamic_cast(index_granularity.get())) + constant_granularity->fixFromRowsCount(rows_count); - loadExistingRowsCount(); /// Must be called after loadRowsCount() as it uses the value of `rows_count`. - loadPartitionAndMinMaxIndex(); - bool has_broken_projections = false; + loadExistingRowsCount(); /// Must be called after loadRowsCount() as it uses the value of `rows_count`. + loadPartitionAndMinMaxIndex(); - if (!parent_part) - { - if (!isStoredOnReadonlyDisk()) + if (!parent_part && !isStoredOnReadonlyDisk()) loadTTLInfos(); + } + /// Projections are full sub-parts; loading each one runs its own `loadColumnsChecksumsIndexes`, + /// so keep it outside the arena scope above (each child re-enters the arena for its own + /// persistent metadata, and its transient load scratch stays on the default per-CPU arenas). + if (!parent_part) loadProjections(require_columns_checksums, check_consistency, has_broken_projections, false /* if_not_loaded */); - } + + /// Kept out of the dedicated arena scope above on purpose: the size computation is heavy + /// short-lived churn (a sample column per column/substream). The finished, part-lifetime maps + /// are re-homed into the arena inside `calculateColumnsAndSecondaryIndicesSizesOnDiskUnlocked`. + if (!(*storage.getSettings())[MergeTreeSetting::columns_and_secondary_indices_sizes_lazy_calculation]) + calculateColumnsAndSecondaryIndicesSizesOnDisk(); if (check_consistency && !has_broken_projections) checkConsistency(require_columns_checksums); @@ -1356,7 +1392,13 @@ MergeTreeDataPartBuilder IMergeTreeDataPart::getProjectionPartBuilder( const String & projection_name, ProjectionDescriptionRawPtr projection, bool is_temp_projection) { const char * projection_extension = is_temp_projection ? ".tmp_proj" : ".proj"; - auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection); + /// The projection storage is stored on the resulting projection part for its lifetime, so create + /// it in the dedicated arena (this is the part-lifetime projection-storage creation site). + MutableDataPartStoragePtr projection_storage; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection); + } MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage, getReadSettings()); return builder.withPartInfo(MergeListElement::FAKE_RESULT_PART_FOR_PROJECTION).withParentPart(this).withProjection(projection); } @@ -1371,20 +1413,20 @@ void IMergeTreeDataPart::addProjectionPart( if (projection_name != projection_part->name) throw Exception(ErrorCodes::LOGICAL_ERROR, "The name of projection part ({}) is inconsistent with the name of projection ({})", projection_part->name, projection_name); + /// The parent keeps this map node (and the copied projection name key) for its whole lifetime, + /// so build it in the dedicated arena. Scoping here covers every caller (insert, merge, + /// projection merge, load). + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); projection_parts[projection_name] = std::move(projection_part); } void IMergeTreeDataPart::loadProjections( bool require_columns_checksums, bool check_consistency, bool & has_broken_projection, bool if_not_loaded, bool only_metadata) { - /// Each loaded projection becomes its own `IMergeTreeDataPart` (via `getProjectionPartBuilder().build()`) - /// and is stored in the parent's `projection_parts` map. Both the map node insertion in - /// `addProjectionPart` and any allocation paths reached during build/load that aren't already - /// scoped by their own helpers belong in the parts arena. This is also the entry point used - /// directly by `MutateTask`, where the surrounding `loadColumnsChecksumsIndexes` scope is not - /// in effect; without this guard those calls would land in the default arena. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - + /// Each loaded projection becomes its own `IMergeTreeDataPart` whose part-lifetime allocations are + /// already routed into the arena by their own helpers (`build()`, `addProjectionPart`, and the + /// child's own `loadColumnsChecksumsIndexes`). The projection load's transient scratch is + /// deliberately left on the default per-CPU arenas, so no arena scope is taken here. auto metadata_snapshot = storage.getInMemoryMetadataPtr(storage.getContext(), false); for (const auto & projection : metadata_snapshot->projections) { @@ -1404,7 +1446,12 @@ void IMergeTreeDataPart::loadProjections( try { if (only_metadata) + { + /// The metadata-only path does not go through `loadColumnsChecksumsIndexes`, so + /// route the projection's part-lifetime checksums into the arena here. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); part->loadChecksums(require_columns_checksums); + } else part->loadColumnsChecksumsIndexes(require_columns_checksums, check_consistency); } @@ -1490,12 +1537,10 @@ std::shared_ptr IMergeTreeDataPart::loadIndex() const /// Memory for index must not be accounted as memory usage for query, because it belongs to a table. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; - /// The loaded primary-index `Columns` live for the part's lifetime when stored on the part - /// (`primary_key_lazy_load=0`, or lazy load with `use_primary_key_cache=0`), or for the - /// cache entry's lifetime when the result is handed to `PrimaryIndexCache`. `loadIndex` is - /// reachable from `loadColumnsChecksumsIndexes` (already wrapped) but also from `getIndex` - /// and `loadIndexToCache` on the lazy-load path; wrapping inside `loadIndex` itself covers - /// every entry point. + /// The loaded primary-index `Columns` are long-lived (kept on the part or handed to + /// `PrimaryIndexCache`), so route them into the dedicated parts arena. Keeping cache-owned indices + /// here too (rather than the cache arena) is deliberate: re-homing on the prewarm path would run + /// after the part is committed and could fail the INSERT, so the primary index lives in one arena. ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); auto metadata_snapshot = getMetadataSnapshot(); @@ -1624,6 +1669,10 @@ void IMergeTreeDataPart::loadSourcePartsSet() if (!info.isPatch()) return; + /// For patch parts `source_parts_set` (`min_max_versions_by_part` / `source_parts_by_version`) + /// is part-lifetime metadata, so route it into the dedicated arena like the other loaders. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + if (auto in = readFileIfExists(SourcePartsSetForPatch::FILENAME)) source_parts_set.readBinary(*in); else @@ -1787,10 +1836,9 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { - /// `MergeTreeDataPartChecksums` is a `std::map` that lives as - /// long as the part. Its tree-node allocations belong in the parts arena. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - + /// Arena-agnostic: the real part-load callers (`loadColumnsChecksumsIndexes`, the projection load) + /// already run inside the parts-arena scope, while transient checksum-only probes must stay on the + /// default arenas. So the caller picks the arena rather than pinning it here. if (auto buf = readFileIfExists("checksums.txt")) { if (checksums.read(*buf)) @@ -2151,6 +2199,26 @@ void IMergeTreeDataPart::setColumnsSubstreams(const ColumnsSubstreams & columns_ columns_substreams = columns_substreams_; } +void IMergeTreeDataPart::moveMetadataToDedicatedArena() +{ + /// Every member below is already stored; the copies exist only to move them between arenas. When the + /// feature is off `ScopedJemallocThreadArena` is a no-op, so skip the copies entirely to avoid the cost. + if (!JemallocMergeTreeArena::isEnabled()) + return; + + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + + /// Re-home the members built outside the arena into it (copy under the scope, then adopt). + reallocateByCopy(partition); + reallocateByCopy(ttl_infos); + reallocateByCopy(expired_columns); + /// The minmax index is re-homed at its population sites instead (`setMinMaxIndex`, the lazy + /// `getMinMaxIndex` build, and the in-place merge/update sites in MergeTask): here it is either not + /// yet populated (merge/mutation) or would be reset again later (zero-level virtual columns). + if (info.isPatch()) + reallocateByCopy(source_parts_set); +} + void IMergeTreeDataPart::loadColumnsSubstreams() { if (auto in = readFileIfExists(COLUMNS_SUBSTREAMS_FILE_NAME)) @@ -2652,9 +2720,25 @@ void IMergeTreeDataPart::calculateColumnsAndSecondaryIndicesSizesOnDisk() const void IMergeTreeDataPart::calculateColumnsAndSecondaryIndicesSizesOnDiskUnlocked() const { + /// The computation must run outside the dedicated arena: `calculateEachColumnSizes` resolves a + /// stream name and builds a sample column for every column and substream, which is heavy + /// short-lived churn that must stay out of the shared arena. All callers (the eager load path + /// and every lazy getter) reach this from the default arenas. calculateColumnsSizesOnDisk(); calculateSecondaryIndicesSizesOnDisk(); are_columns_and_secondary_indices_sizes_calculated = true; + + /// The size maps are cached on the part for its whole lifetime, so re-home the finished maps into + /// the dedicated arena. Done here (not in the public wrapper) so the lazy getters, which call this + /// directly, re-home too. + if (JemallocMergeTreeArena::isEnabled()) + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + if (columns_sizes) + columns_sizes = std::make_shared(*columns_sizes); + if (secondary_index_sizes) + secondary_index_sizes = std::make_shared(*secondary_index_sizes); + } } void IMergeTreeDataPart::calculateColumnsSizesOnDisk() const diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index bc9ffb8b5035..457b3a569c2f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -153,6 +153,13 @@ class IMergeTreeDataPart : public std::enable_shared_from_this #include #include +#include +#include #include namespace DB @@ -110,6 +112,11 @@ std::optional IMergeTreeDataPartWriter::releaseIndexColumns() /// We need to deallocate it in shrinkToFit without memory tracker as well. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; + /// `shrinkToFit` reallocates each index column to a right-sized buffer, and that buffer is the + /// resident primary index kept for the part's whole lifetime. Route it into the dedicated arena + /// (the reload path does the same in `loadIndex`). + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + Columns result; result.reserve(index_columns.size()); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 16edb0c10a8d..7307c448cd25 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -552,28 +552,33 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->disk = global_ctx->space_reservation->getDisk(); auto local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; - /// Same rationale as `MergeTreeData::loadDataPart`: the per-part `SingleDiskVolume` and - /// the resulting `IMergeTreeDataPart` constructed below live for the merged part's lifetime. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - - std::optional builder; - if (global_ctx->parent_part) - { - auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename, /* use parent transaction */ false); - builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage, getReadSettings()); - builder->withParentPart(global_ctx->parent_part); - } - else + /// The `SingleDiskVolume`, `DataPartStorageOnDiskFull`, and `IMergeTreeDataPart` constructed + /// here are stored on the merged part and live for its whole lifetime, so route them into the + /// dedicated arena (same as `MergeTreeData::loadDataPart` / `DataPartsExchange`). The rest of + /// `prepare` (storage snapshot, column extraction, pipeline/transform setup) is merge-lifetime + /// scratch and is deliberately left in the default per-CPU arenas. { - auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, global_ctx->disk, 0); - builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename, getReadSettings())); - builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); - } + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + + std::optional builder; + if (global_ctx->parent_part) + { + auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename, /* use parent transaction */ false); + builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage, getReadSettings()); + builder->withParentPart(global_ctx->parent_part); + } + else + { + auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, global_ctx->disk, 0); + builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename, getReadSettings())); + builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); + } - builder->withPartInfo(global_ctx->future_part->part_info); - builder->withPartType(global_ctx->future_part->part_format.part_type); + builder->withPartInfo(global_ctx->future_part->part_info); + builder->withPartType(global_ctx->future_part->part_format.part_type); - global_ctx->new_data_part = std::move(*builder).build(); + global_ctx->new_data_part = std::move(*builder).build(); + } auto data_part_storage = global_ctx->new_data_part->getDataPartStoragePtr(); if (data_part_storage->exists()) @@ -780,7 +785,12 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const if (part->isEmpty()) continue; - global_ctx->new_data_part->getMinMaxIndex()->merge(*part->getMinMaxIndex()); + { + /// Populate the merged part's minmax index in the parts arena (the object and the + /// hyperrectangle/Field allocations of `merge` both land there). + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + global_ctx->new_data_part->getMinMaxIndex()->merge(*part->getMinMaxIndex()); + } const auto & result_statistics = global_ctx->gathered_data.statistics; if (result_statistics.empty()) @@ -841,6 +851,11 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos, global_ctx->metadata_snapshot->getMetadataVersion()); + /// partition / ttl_infos / minmax / expired_columns (and patch source parts) are populated + /// above interleaved with merge-only scratch, so they were allocated in the default arenas. + /// Re-home them into the dedicated arena; the interleaved scratch stays out. + global_ctx->new_data_part->moveMetadataToDedicatedArena(); + ctx->sum_input_rows_upper_bound = global_ctx->merge_list_element_ptr->total_rows_count; ctx->sum_compressed_bytes_upper_bound = global_ctx->merge_list_element_ptr->total_size_bytes_compressed; ctx->sum_uncompressed_bytes_upper_bound = global_ctx->merge_list_element_ptr->total_size_bytes_uncompressed; @@ -1517,7 +1532,12 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeImpl() const const_cast(*global_ctx->to).write(block); if (global_ctx->merge_may_reduce_rows) + { + /// Same rationale as the horizontal-stage `merge` above: keep the row-reducing merge's + /// incrementally-built minmax index in the parts arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); global_ctx->new_data_part->getMinMaxIndex()->update(block, global_ctx->minmax_idx_columns); + } calculateProjections(block, starting_offset); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0f71e968103e..2d579a4e97f8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2121,15 +2121,20 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( auto component_guard = Coordination::setCurrentComponent("MergeTreeData::loadDataPart"); LOG_TRACE(log, "Loading {} part {} from disk {}", magic_enum::enum_name(to_state), part_name, part_disk_ptr->getName()); - /// Route the per-part `SingleDiskVolume` and `DataPartStorageOnDiskFull` clones below into - /// the MergeTree arena — both are stored on the resulting `IMergeTreeDataPart` and share its - /// lifetime. Without this scope they would land in the default arena, slipping past the - /// per-part guards in `MergeTreeDataPartBuilder::build` / `loadColumnsChecksumsIndexes`. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - LoadPartResult res; - auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); - auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, part_name); + + /// The per-part `SingleDiskVolume` is stored on the resulting part and shares its lifetime, so + /// create it in the dedicated MergeTree arena. The storage wrapper the part actually keeps is + /// created (and arena-routed) by the builder's `getPartStorageByType`; `build()` and + /// `loadColumnsChecksumsIndexes` below run OUTSIDE this scope on purpose: `build()` re-enters the + /// arena itself for the part object, while the metadata load's transient parse / consistency + /// scratch belongs in the default per-CPU arenas (it re-enters the arena only for the persistent + /// metadata it caches). + VolumePtr single_disk_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); + } String part_path = fs::path(relative_data_path) / part_name; @@ -2224,7 +2229,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( { if ((*it)->checksums.getTotalChecksumHex() == res.part->checksums.getTotalChecksumHex()) { - LOG_ERROR(log, "Duplicate part {}", data_part_storage->getFullPath()); + LOG_ERROR(log, "Duplicate part {}", res.part->getDataPartStorage().getFullPath()); res.part->is_duplicate = true; return res; } @@ -7532,12 +7537,16 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(const String & part_name, const DiskPtr & disk, const String & temp_part_dir, bool detach_if_broken) const { - /// Same rationale as `loadDataPart`: the `SingleDiskVolume` below lives for the part's lifetime. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - MutableDataPartPtr part; - auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); + /// Same rationale as `loadDataPart`: the `SingleDiskVolume` lives for the part's lifetime, so + /// create it in the dedicated arena; `build()` and the metadata load below run outside it (the + /// load's transient scratch stays on the default per-CPU arenas). + VolumePtr single_disk_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + single_disk_volume = std::make_shared(disk->getName(), disk, 0); + } fs::path full_part_dir{temp_part_dir}; String parent_part_dir = full_part_dir.parent_path(); String part_dir_name = full_part_dir.filename(); @@ -8460,15 +8469,19 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const MutableDataPartsVector loaded_parts; loaded_parts.reserve(renamed_parts.old_and_new_names.size()); - /// Same rationale as `loadDataPart`: the per-part `SingleDiskVolume` below lives for the part's lifetime. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - for (const auto & [part_name, old_dir, new_dir, disk] : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part {}", new_dir); disk->removeFileIfExists(fs::path(relative_data_path) / source_dir / new_dir / VersionMetadata::TXN_VERSION_METADATA_FILE_NAME); - auto single_disk_volume = std::make_shared("volume_" + part_name, disk); + /// The per-part `SingleDiskVolume` lives for the part's lifetime, so create it in the dedicated + /// arena; `build()` and `loadPartAndFixMetadataImpl` below run outside it (the metadata load's + /// transient scratch stays on the default per-CPU arenas). + VolumePtr single_disk_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + single_disk_volume = std::make_shared("volume_" + part_name, disk); + } auto part = getDataPartBuilder(part_name, single_disk_volume, source_dir / new_dir, getReadSettings()) .withPartFormatFromDisk() .build(); @@ -10512,7 +10525,12 @@ StorageMetadataPtr MergeTreeData::getPatchPartMetadata(const ColumnsDescription auto & metadata_snapshot = patch_parts_metadata_cache[patch_partition_id]; if (!metadata_snapshot) + { + /// This snapshot is cached per patch partition for the table's lifetime, so build it in the + /// dedicated arena like the rest of the per-table metadata. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); metadata_snapshot = DB::getPatchPartMetadata(patch_part_desc, local_context); + } return metadata_snapshot; } @@ -11157,7 +11175,12 @@ std::pair MergeTreeData::createE DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; VolumePtr volume = getStoragePolicy()->getVolume(0); ReservationPtr reservation = reserveSpacePreferringTTLRules(metadata_snapshot, 0, move_ttl_infos, time(nullptr), 0, true); - VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); + /// The `SingleDiskVolume` is stored on the part for its whole lifetime; build it in the arena. + VolumePtr data_part_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + data_part_volume = createVolumeFromReservation(reservation, volume); + } auto tmp_dir_holder = getTemporaryPartDirectoryHolder(EMPTY_PART_TMP_PREFIX + new_part_name); auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name, getReadSettings()) @@ -11186,6 +11209,8 @@ std::pair MergeTreeData::createE new_data_part->partition = partition; new_data_part->setMinMaxIndex(std::move(minmax_idx)); + /// `partition` and the minmax index were built outside the arena above; re-home them into it. + new_data_part->moveMetadataToDedicatedArena(); new_data_part->is_temp = true; /// In case of replicated merge tree with zero copy replication /// Here Clickhouse claims that this new part can be deleted in temporary state without unlocking the blobs diff --git a/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp index 39347660d4cc..2a664ddbc762 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp @@ -95,6 +95,11 @@ MutableDataPartStoragePtr MergeTreeDataPartBuilder::getPartStorageByType( if (!volume_) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part storage, because volume is not specified"); + /// The storage object and its `root_path` / `part_dir` strings live for the part's whole lifetime. + /// Create them in the dedicated arena here: on the write paths this runs while configuring the + /// builder, before `build()` enters its own scope, so the scope there would otherwise miss them. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + using Type = MergeTreeDataPartStorageType; switch (storage_type_.getValue()) { diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 4e6bccec493a..f3431f7580c7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -241,6 +243,10 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc * (observed in long INSERT SELECTs) */ MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; + /// The in-memory primary index lives on the part for its whole lifetime (freed only when the + /// part is merged away), so build it in the dedicated MergeTree arena — same rationale as the + /// memory-tracker blocker above. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); if (settings.save_primary_index_in_memory && index_columns.empty()) { @@ -327,6 +333,7 @@ void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::Dat if (write_final_mark && !last_index_block.empty()) { MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); calculateAndSerializePrimaryIndexRow(last_index_block, last_index_block.rows() - 1); } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 5a91d5dd6338..4fc3210a819b 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -865,7 +867,13 @@ MergeTreeTemporaryPartPtr MergeTreeDataWriter::writeTempPartImpl( reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, expected_size, move_ttl_infos, time(nullptr), 0, true); } - VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); + /// The `SingleDiskVolume` is stored on the part and lives for its whole lifetime, so build it in + /// the dedicated arena (`build()` below self-scopes; the tail metadata is re-homed further down). + VolumePtr data_part_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + data_part_volume = createVolumeFromReservation(reservation, volume); + } auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir, getReadSettings()) .withPartFormat(data.choosePartFormat(expected_size, block.rows(), new_part_level, /*projection =*/nullptr)) @@ -947,6 +955,11 @@ MergeTreeTemporaryPartPtr MergeTreeDataWriter::writeTempPartImpl( new_data_part->ttl_infos.update(move_ttl_infos); + /// partition / ttl_infos / minmax (and patch source parts) are built above outside any + /// dedicated-arena scope, so they were allocated in the default arenas. Re-home them into the + /// dedicated arena, matching the merge / mutation / load paths. + new_data_part->moveMetadataToDedicatedArena(); + /// Pass empty TTL infos so that `RECOMPRESS` codecs are not selected at insert time; /// recompression should happen during merges, not on the initial write path. auto compression_codec = data.getCompressionCodecForPart(0, {}, time(nullptr)); diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/src/Storages/MergeTree/MergeTreeIndexGranularity.h index 52b499a4bf14..6ff05c58fd10 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -12,6 +12,8 @@ class MergeTreeIndexGranularity { public: MergeTreeIndexGranularity() = default; + MergeTreeIndexGranularity(const MergeTreeIndexGranularity &) = default; + MergeTreeIndexGranularity & operator=(const MergeTreeIndexGranularity &) = default; virtual ~MergeTreeIndexGranularity() = default; /// Returns granularity if it is constant for whole part (except last granule). @@ -75,6 +77,9 @@ class MergeTreeIndexGranularity /// Returns new optimized index granularity structure or nullptr if no optimization is not applicable. virtual std::shared_ptr optimize() = 0; virtual std::string describe() const = 0; + + /// Deep-copy, so a written part's granularity can be re-homed into the dedicated MergeTree arena. + virtual std::shared_ptr clone() const = 0; }; using MergeTreeIndexGranularityPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.h b/src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.h index 60965d200fc1..50c6b20b74d0 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.h @@ -36,6 +36,7 @@ class MergeTreeIndexGranularityAdaptive final : public MergeTreeIndexGranularity std::shared_ptr optimize() override; std::string describe() const override; + std::shared_ptr clone() const override { return std::make_shared(*this); } private: std::vector marks_rows_partial_sums; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityConstant.h b/src/Storages/MergeTree/MergeTreeIndexGranularityConstant.h index 7d612c9f80a5..bfed57a86c6b 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityConstant.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityConstant.h @@ -52,6 +52,7 @@ class MergeTreeIndexGranularityConstant final : public MergeTreeIndexGranularity std::shared_ptr optimize() override { return nullptr; } std::string describe() const override; + std::shared_ptr clone() const override { return std::make_shared(*this); } }; } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index ce56f7121960..54914319dd26 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -246,7 +248,12 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); - new_part->checksums = checksums; + { + /// The checksums map lives on the part for its whole lifetime: copy it into the part under the + /// dedicated arena directly, rather than assigning and re-homing with a second copy later. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + new_part->checksums = checksums; + } new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk()); new_part->setBytesUncompressedOnDisk(checksums.getTotalSizeUncompressedOnDisk()); new_part->index_granularity = writer->getIndexGranularity(); @@ -278,6 +285,23 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( if (default_codec != nullptr) new_part->default_codec = default_codec; + /// The TTL infos and the index granularity are built during the write (in the default arenas) and + /// assigned above; re-home them into the dedicated MergeTree arena so a freshly written part's + /// long-lived metadata lives there, like a reloaded part's. The in-memory primary index is + /// already built in the arena by the writer, and `checksums` was copied under the arena above. + /// Runs for insert / merge / mutation, and for projections via each projection part's own + /// finalize. When the feature is off, skip the O(marks) copies rather than paying them for nothing. + if (JemallocMergeTreeArena::isEnabled()) + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + /// `TTLTransform` rebuilds `ttl_infos` from scratch during the write (in the default arenas), + /// discarding the copy re-homed at prepare time, so re-home the final maps here. `checksums` + /// was already copied into the part under the arena above. + reallocateByCopy(new_part->ttl_infos); + if (new_part->index_granularity) + new_part->index_granularity = new_part->index_granularity->clone(); + } + auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); finalizer->written_files = std::move(written_files); return Finalizer(std::move(finalizer)); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 918ead21658d..e9e923b24578 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1427,7 +1427,7 @@ static void finalizeMutatedPart( new_data_part->rows_count = source_part->rows_count; new_data_part->index_granularity = source_part->index_granularity; - new_data_part->setMinMaxIndex(std::make_shared(*source_part->getMinMaxIndex())); + new_data_part->setMinMaxIndex(source_part->getMinMaxIndex()); new_data_part->modification_time = time(nullptr); if ((*new_data_part->storage.getSettings())[MergeTreeSetting::enable_index_granularity_compression]) @@ -1453,6 +1453,21 @@ static void finalizeMutatedPart( new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); new_data_part->default_codec = codec; + + /// This hardlink / mutate-some-columns path assembles the checksums and index granularity in the + /// default arenas (the full-rewrite path re-homes them in `MergedBlockOutputStream::finalizePartAsync`). + /// Re-home the finished part-lifetime maps into the dedicated arena. The primary index set above is + /// already routed by `setIndex`; the minmax index by `setMinMaxIndex`. + if (JemallocMergeTreeArena::isEnabled()) + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + reallocateByCopy(new_data_part->checksums); + /// TTL-recalculating mutations rebuild `ttl_infos` during execution (in the default arenas), + /// so re-home the final maps here, mirroring `MergedBlockOutputStream::finalizePartAsync`. + reallocateByCopy(new_data_part->ttl_infos); + if (new_data_part->index_granularity) + new_data_part->index_granularity = new_data_part->index_granularity->clone(); + } } } @@ -3420,11 +3435,15 @@ bool MutateTask::prepare() } } - /// Same rationale as `MergeTreeData::loadDataPart`: the per-part `SingleDiskVolume` and the - /// resulting `IMergeTreeDataPart` constructed below live for the mutated part's lifetime. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - - auto single_disk_volume = std::make_shared("volume_" + ctx->future_part->name, ctx->space_reservation->getDisk(), 0); + /// Same rationale as `MergeTreeData::loadDataPart`: the per-part `SingleDiskVolume` lives for the + /// mutated part's lifetime, so create it in the dedicated arena. `build()` re-enters the arena for + /// the part object; the mutation planning below (column transforms, projection/statistics + /// collections, file lists, task construction) is transient and stays on the default per-CPU arenas. + VolumePtr single_disk_volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + single_disk_volume = std::make_shared("volume_" + ctx->future_part->name, ctx->space_reservation->getDisk(), 0); + } ctx->disk = single_disk_volume->getDisk(); std::string prefix; @@ -3457,6 +3476,11 @@ bool MutateTask::prepare() ctx->new_data_part->setColumnsSubstreams(new_columns_substreams); ctx->new_data_part->partition.assign(ctx->source_part->partition); + /// Re-home the part-lifetime metadata assigned above (partition, ttl_infos) into the dedicated + /// arena; `setColumns` / `setColumnsSubstreams` already self-scope. Everything below is transient + /// mutation planning and deliberately stays on the default per-CPU arenas. + ctx->new_data_part->moveMetadataToDedicatedArena(); + /// Don't change granularity type while mutating subset of columns ctx->mrk_extension = ctx->source_part->index_granularity_info.mark_type.getFileExtension(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 99867ac0c711..c32c2d1217fb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -227,6 +229,10 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (replica_metadata_version_exists) { auto storage_metadata_snapshot = storage.getInMemoryMetadataPtr(storage.getContext(), false); + /// This metadata snapshot lives for the table's lifetime, so route the clone into the + /// dedicated MergeTree arena like the ALTER paths (this runs on the restarting thread, + /// outside the constructor's arena scope). + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); storage.setInMemoryMetadata(storage_metadata_snapshot->withMetadataVersion(replica_metadata_version)); } else diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index b58b4e0dd1f6..48b75b997f4a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -62,6 +62,8 @@ #include #include #include +#include +#include namespace ProfileEvents @@ -487,14 +489,22 @@ void StorageMergeTree::alter( changeSettings(new_metadata.settings_changes, table_lock_holder); if (statistics_changed) + { + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); setInMemoryMetadata(new_metadata); + } /// It is safe to ignore exceptions here as only settings are changed, which is not validated in `alterTable` DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata, /*validate_new_create_query=*/true); } else if (commands.isCommentAlter()) { - setInMemoryMetadata(new_metadata); + { + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + setInMemoryMetadata(new_metadata); + } /// It is safe to ignore exceptions here as only the comment changed, which is not validated in `alterTable` DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata, /*validate_new_create_query=*/true); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ad52354eb849..8505a6ad0907 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -663,7 +663,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( * * Otherwise `metadata_version` for not first replica will be initialized with 0 by default. */ - setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_version)); + /// This metadata snapshot lives for the table's lifetime, so route the clone into the + /// dedicated MergeTree arena explicitly (rather than relying on the factory-level scope + /// in registerStorageMergeTree), so it converges with the ALTER / restart paths. + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_version)); + } metadata_snapshot = getInMemoryMetadataPtr(getContext(), true); } } @@ -2457,11 +2463,14 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo continue; } - /// Same rationale as `MergeTreeData::loadDataPart`: the per-part `SingleDiskVolume` and - /// the resulting attached `IMergeTreeDataPart` live for the part's lifetime. - ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); - - const auto volume = std::make_shared("volume_" + detached_part_info.dir_name, detached_part_info.disk); + /// Same rationale as `MergeTreeData::loadDataPart`: the per-part `SingleDiskVolume` lives for + /// the part's lifetime, so create it in the dedicated arena; `build()` and the metadata load + /// below run outside it (the load's transient scratch stays on the default per-CPU arenas). + VolumePtr volume; + { + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + volume = std::make_shared("volume_" + detached_part_info.dir_name, detached_part_info.disk); + } auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path(rename_parts.source_dir) / rename_parts.old_and_new_names.front().new_dir, getReadSettings()) .withPartFormatFromDisk() .build(); @@ -6808,7 +6817,11 @@ void StorageReplicatedMergeTree::alter( changeSettings(future_metadata.settings_changes, table_lock_holder); if (statistics_changed) + { + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); setInMemoryMetadata(future_metadata); + } /// It is safe to ignore exceptions here as only settings are changed, which is not validated in `alterTable` DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, future_metadata, /*validate_new_create_query=*/true); @@ -6817,7 +6830,11 @@ void StorageReplicatedMergeTree::alter( if (commands.isCommentAlter()) { - setInMemoryMetadata(future_metadata); + { + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + setInMemoryMetadata(future_metadata); + } /// It is safe to ignore exceptions here as only the comment is changed, which is not validated in `alterTable` DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, future_metadata, /*validate_new_create_query=*/true); @@ -6841,7 +6858,11 @@ void StorageReplicatedMergeTree::alter( for (auto & index : future_metadata.secondary_indices) index.escape_filenames = committed_metadata->escape_index_filenames; - setInMemoryMetadata(future_metadata); + { + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); + setInMemoryMetadata(future_metadata); + } /// It is safe to ignore exceptions here as only settings and comments are changed, neither of which is validated in `alterTable` DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, future_metadata, /*validate_new_create_query=*/true); @@ -6968,6 +6989,8 @@ void StorageReplicatedMergeTree::alter( if (comment_is_changed) { metadata_copy.setComment(future_metadata.comment); + /// Route the long-lived metadata snapshot clone into the dedicated MergeTree arena. + ScopedJemallocThreadArena mergetree_arena_scope(JemallocMergeTreeArena::getArenaIndex()); setInMemoryMetadata(metadata_copy); } diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/__init__.py b/tests/integration/test_jemalloc_merge_tree_arenas/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/configs/capped.xml b/tests/integration/test_jemalloc_merge_tree_arenas/configs/capped.xml new file mode 100644 index 000000000000..befc5b956dd6 --- /dev/null +++ b/tests/integration/test_jemalloc_merge_tree_arenas/configs/capped.xml @@ -0,0 +1,3 @@ + + 1000000 + diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/configs/disabled.xml b/tests/integration/test_jemalloc_merge_tree_arenas/configs/disabled.xml new file mode 100644 index 000000000000..c586a16631d8 --- /dev/null +++ b/tests/integration/test_jemalloc_merge_tree_arenas/configs/disabled.xml @@ -0,0 +1,3 @@ + + 0 + diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/configs/pool.xml b/tests/integration/test_jemalloc_merge_tree_arenas/configs/pool.xml new file mode 100644 index 000000000000..b8b925efb0b0 --- /dev/null +++ b/tests/integration/test_jemalloc_merge_tree_arenas/configs/pool.xml @@ -0,0 +1,7 @@ + + 4 + + 0 + diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/configs/single.xml b/tests/integration/test_jemalloc_merge_tree_arenas/configs/single.xml new file mode 100644 index 000000000000..200bb1a43cc8 --- /dev/null +++ b/tests/integration/test_jemalloc_merge_tree_arenas/configs/single.xml @@ -0,0 +1,3 @@ + + 1 + diff --git a/tests/integration/test_jemalloc_merge_tree_arenas/test.py b/tests/integration/test_jemalloc_merge_tree_arenas/test.py new file mode 100644 index 000000000000..c616cfb6cafd --- /dev/null +++ b/tests/integration/test_jemalloc_merge_tree_arenas/test.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 + +# Exercises the `jemalloc_merge_tree_arenas` server setting, which controls the dedicated +# jemalloc arena pool for long-lived MergeTree metadata. It is a startup-only server setting, so +# each value needs its own server instance. We assert: +# - the setting is read (system.server_settings), +# - the resulting arena count is exposed via jemalloc.mergetree_arena.count and matches the value +# (0 disabled, 1 single, N sharded, capped at the CPU core count), +# - routing follows the count (disabled -> no active_bytes metric; a pool -> arena fills up). + +import re + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node_disabled = cluster.add_instance("node_disabled", main_configs=["configs/disabled.xml"]) +node_single = cluster.add_instance("node_single", main_configs=["configs/single.xml"]) +node_pool = cluster.add_instance("node_pool", main_configs=["configs/pool.xml"]) +node_capped = cluster.add_instance("node_capped", main_configs=["configs/capped.xml"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def jemalloc_built_in(node): + return ( + node.query( + "SELECT value IN ('ON', '1') FROM system.build_options WHERE name = 'USE_JEMALLOC'" + ).strip() + == "1" + ) + + +def configured_value(node): + return int( + node.query( + "SELECT value FROM system.server_settings WHERE name = 'jemalloc_merge_tree_arenas'" + ).strip() + ) + + +def arena_count(node): + node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + return int( + node.query( + "SELECT value FROM system.asynchronous_metrics WHERE metric = 'jemalloc.mergetree_arena.count'" + ).strip() + ) + + +def num_cpus(node): + return int(node.exec_in_container(["nproc"]).strip()) + + +def test_setting_is_read(started_cluster): + assert configured_value(node_disabled) == 0 + assert configured_value(node_single) == 1 + assert configured_value(node_pool) == 4 + assert configured_value(node_capped) == 1000000 + + +def test_arena_count_matches_setting(started_cluster): + if not jemalloc_built_in(node_single): + pytest.skip("built without jemalloc") + + assert arena_count(node_disabled) == 0 + assert arena_count(node_single) == 1 + # Capped at the number of CPU cores the container sees. + assert arena_count(node_pool) == min(4, num_cpus(node_pool)) + # A value far above the core count collapses to the number of CPUs the container may run on, + # proving the cap. + capped = arena_count(node_capped) + assert capped == min(1000000, num_cpus(node_capped)) + + +def test_disabled_does_not_route_to_a_dedicated_arena(started_cluster): + if not jemalloc_built_in(node_disabled): + pytest.skip("built without jemalloc") + + node_disabled.query("DROP TABLE IF EXISTS t SYNC") + node_disabled.query("CREATE TABLE t (a UInt64, b String) ENGINE = MergeTree ORDER BY a") + node_disabled.query("INSERT INTO t SELECT number, toString(number) FROM numbers(100000)") + node_disabled.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + + # With the pool disabled, the per-arena byte metrics are not emitted at all. + assert ( + node_disabled.query( + "SELECT count() FROM system.asynchronous_metrics WHERE metric = 'jemalloc.mergetree_arena.active_bytes'" + ).strip() + == "0" + ) + + +def test_pool_arena_accumulates(started_cluster): + if not jemalloc_built_in(node_pool): + pytest.skip("built without jemalloc") + + node_pool.query("DROP TABLE IF EXISTS t SYNC") + node_pool.query("CREATE TABLE t (a UInt64, b String) ENGINE = MergeTree ORDER BY a") + node_pool.query("INSERT INTO t SELECT number, toString(number) FROM numbers(100000)") + node_pool.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + + active_bytes = int( + node_pool.query( + "SELECT value FROM system.asynchronous_metrics WHERE metric = 'jemalloc.mergetree_arena.active_bytes'" + ).strip() + ) + assert active_bytes > 0 + + +def manual_arena_nmalloc(node): + # Cumulative allocation count ("nmalloc") per manually-created arena from malloc_stats_print. + # On node_pool those are exactly the MergeTree pool arenas: the cache arena is disabled in + # pool.xml and the queries disable expression compilation so no JIT arena appears. The auto + # (per-CPU) arenas used for transient allocations are excluded. nmalloc only ever grows, so a + # per-arena delta reliably shows which pool arenas received allocations, independent of how much + # is retained or freed by background purge. + text = node.query("SELECT stats FROM system.jemalloc_stats FORMAT TSVRaw") + result = {} + for block in re.split(r"\narenas\[", text)[1:]: + index_match = re.match(r"(\d+)\]", block) + if not index_match or not re.search(r'name:\s*"manual', block): + continue + # total row columns: allocated nmalloc (#/sec) ndalloc ... + total_match = re.search(r"\ntotal:\s+\d+\s+(\d+)", block) + if total_match: + result[int(index_match.group(1))] = int(total_match.group(1)) + return result + + +def test_pool_shards_across_arenas(started_cluster): + if not jemalloc_built_in(node_pool): + pytest.skip("built without jemalloc") + if arena_count(node_pool) < 2: + pytest.skip("pool collapsed to a single arena (fewer than 2 routable CPUs)") + + node_pool.query("DROP TABLE IF EXISTS t_shard SYNC") + node_pool.query( + "CREATE TABLE t_shard (id UInt64, " + + ", ".join(f"c{i} String" for i in range(40)) + + ") ENGINE = MergeTree ORDER BY id " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" + ) + + before = manual_arena_nmalloc(node_pool) + + # Many small wide parts produce many concurrent background merges, which run on the merge thread + # pool spread across CPUs, so per-part metadata is allocated from several pool arenas rather than + # a single one. + cols = ", ".join(f"toString(number + {i})" for i in range(40)) + for batch in range(20): + node_pool.query( + f"INSERT INTO t_shard SELECT number + {batch} * 100000, {cols} FROM numbers(1500)", + settings={ + "max_insert_block_size": 150, + "min_insert_block_size_rows": 150, + # No JIT: expression compilation would lazily create the JIT arena, another + # "manual" arena that could satisfy the sharding assertion below. + "compile_expressions": 0, + }, + ) + for _ in range(5): + node_pool.query("OPTIMIZE TABLE t_shard FINAL") + + after = manual_arena_nmalloc(node_pool) + node_pool.query("DROP TABLE t_shard SYNC") + + # Pool arenas that received a non-trivial number of metadata allocations. If routing were + # broken (every allocation to arena 0) at most one would grow; a working per-CPU pool spreads the + # per-part metadata across several. + grew = sorted(idx for idx, n in after.items() if n > before.get(idx, 0) + 500) + assert len(grew) >= 2, f"metadata landed in only {len(grew)} pool arena(s): {grew}" diff --git a/tests/queries/0_stateless/03268_system_parts_index_granularity.reference b/tests/queries/0_stateless/03268_system_parts_index_granularity.reference index f66f10ced962..9f6b7ef780f9 100644 --- a/tests/queries/0_stateless/03268_system_parts_index_granularity.reference +++ b/tests/queries/0_stateless/03268_system_parts_index_granularity.reference @@ -1,2 +1,2 @@ -88 128 -25 25 +88 1 +25 1 diff --git a/tests/queries/0_stateless/03268_system_parts_index_granularity.sql b/tests/queries/0_stateless/03268_system_parts_index_granularity.sql index 3df9f6be0284..390eb09da6dc 100644 --- a/tests/queries/0_stateless/03268_system_parts_index_granularity.sql +++ b/tests/queries/0_stateless/03268_system_parts_index_granularity.sql @@ -16,6 +16,9 @@ ALTER TABLE t MODIFY SETTING enable_index_granularity_compression = 1; INSERT INTO t SELECT number, toString(number) FROM numbers(100); -SELECT index_granularity_bytes_in_memory, index_granularity_bytes_in_memory_allocated FROM system.parts where table = 't' and database = currentDatabase() ORDER BY name; +-- The reserved capacity depends on the build: on jemalloc builds it is tightened to the exact +-- in-memory size when the part is finalized, while builds without jemalloc keep the vector's growth +-- capacity (128 bytes for this adaptive part). Accept exactly those two values and nothing else. +SELECT index_granularity_bytes_in_memory, index_granularity_bytes_in_memory_allocated = index_granularity_bytes_in_memory OR index_granularity_bytes_in_memory_allocated = 128 FROM system.parts where table = 't' and database = currentDatabase() ORDER BY name; DROP TABLE IF EXISTS t; From 29075be7d3244ad94aea15346a18ed6fff16fbcd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Jul 2026 10:50:30 +0000 Subject: [PATCH 35/86] Backport #111518 to 26.6: Fix mutations with query parameters in IN PARTITION --- src/Parsers/ParserPartition.cpp | 36 +++++++-- src/Storages/MergeTree/MergeTreeData.cpp | 38 ++++++--- src/Storages/MutationCommands.cpp | 37 ++++++++- ...ion_in_partition_query_parameter.reference | 10 +++ ..._mutation_in_partition_query_parameter.sql | 80 +++++++++++++++++++ ...ition_query_parameter_replicated.reference | 3 + ...n_partition_query_parameter_replicated.sql | 31 +++++++ 7 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.reference create mode 100644 tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.sql create mode 100644 tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.reference create mode 100644 tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.sql diff --git a/src/Parsers/ParserPartition.cpp b/src/Parsers/ParserPartition.cpp index 0a5a384e8da0..cf083a5a33a0 100644 --- a/src/Parsers/ParserPartition.cpp +++ b/src/Parsers/ParserPartition.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -50,16 +51,37 @@ bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (parser_expr.parse(pos, value, expected)) { - if (const auto * tuple_ast = value->as(); tuple_ast) + if (const auto * function_ast = value->as(); function_ast) { - if (tuple_ast->name != "tuple") - return false; + if (function_ast->name == "tuple") + { + const auto * arguments_ast = function_ast->arguments->as(); + if (arguments_ast) + fields_count = arguments_ast->children.size(); + else + fields_count = 0; + } + else if (isFunctionCast(function_ast)) + { + /// A cast of a literal or of a tuple, e.g. `_CAST(20260624, 'UInt32')`. + /// Query parameter substitution (`PARTITION {param:Type}`) rewrites the + /// parameter into this form, and the result must be parseable back, e.g. + /// when mutation commands are re-read from ZooKeeper or disk. Leave + /// `fields_count` unset, the same as for an unsubstituted parameter; it is + /// deduced from the cast operand in `MergeTreeData::getPartitionIDFromQuery`. + if (!function_ast->arguments || function_ast->arguments->children.size() != 2) + return false; - const auto * arguments_ast = tuple_ast->arguments->as(); - if (arguments_ast) - fields_count = arguments_ast->children.size(); + const auto & cast_operand = function_ast->arguments->children.at(0); + const auto * inner_function = cast_operand->as(); + bool is_tuple_function = inner_function && inner_function->name == "tuple"; + if (!is_tuple_function && !cast_operand->as()) + return false; + } else - fields_count = 0; + { + return false; + } } else if (const auto * literal_ast = value->as(); literal_ast) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0f71e968103e..8a9b6d982072 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7670,7 +7670,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc auto first_arg = tuple_ast->arguments->as()->children.at(0); if (const auto * inner_tuple = first_arg->as(); inner_tuple && inner_tuple->name == "tuple") { - const auto * arguments_ast = tuple_ast->arguments->as(); + const auto * arguments_ast = inner_tuple->arguments->as(); if (arguments_ast) partition_ast_fields_count = arguments_ast->children.size(); else @@ -7737,15 +7737,21 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc { /// Function tuple(...) requires at least one argument, so empty key is a special case chassert(!partition_ast_fields_count); - chassert(typeid_cast(partition_value_ast.get())); - chassert(partition_value_ast->as()->name == "tuple"); - chassert(partition_value_ast->as()->arguments); - auto args = partition_value_ast->as()->arguments; - if (!args) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected at least one argument in partition AST"); - bool empty_tuple = partition_value_ast->as()->arguments->children.empty(); - if (!empty_tuple) - throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, "Partition key is empty, expected 'tuple()' as partition key"); + const auto * function_ast = partition_value_ast->as(); + if (function_ast && function_ast->name == "tuple") + { + if (!function_ast->arguments) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected at least one argument in partition AST"); + if (!function_ast->arguments->children.empty()) + throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, "Partition key is empty, expected 'tuple()' as partition key"); + } + else + { + /// E.g. a cast of an empty tuple, produced by a substituted query parameter of type `Tuple()`. + Field partition_key_value = evaluateConstantExpression(partition_value_ast, local_context).first; + if (partition_key_value.getType() != Field::Types::Tuple || !partition_key_value.safeGet().empty()) + throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, "Partition key is empty, expected 'tuple()' as partition key"); + } } else if (fields_count == 1) { @@ -7772,6 +7778,18 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc } /// Simple partition key, need to evaluate and cast Field partition_key_value = evaluateConstantExpression(partition_value_ast, local_context).first; + + /// A cast of a one-element tuple (e.g. a substituted query parameter of type `Tuple(T)`) + /// evaluates to a tuple; unwrap it, unless the partition key column itself is a tuple. + if (partition_key_value.getType() == Field::Types::Tuple && !isTuple(key_sample_block.getByPosition(0).type)) + { + Tuple tuple_value = partition_key_value.safeGet(); + if (tuple_value.size() != 1) + throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, + "Wrong number of fields in the partition expression: {}, must be: 1", tuple_value.size()); + partition_key_value = std::move(tuple_value[0]); + } + partition_row[0] = convertFieldToTypeOrThrow(partition_key_value, *key_sample_block.getByPosition(0).type); } else diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index f2840c076943..5f5d1f806339 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -297,9 +297,40 @@ boost::intrusive_ptr MutationCommands::ast(bool with_pure_met } +namespace +{ + +ASTPtr parseMutationCommandsList(const String & commands_str) +{ + ParserAlterCommandList p_alter_commands; + return parseQuery( + p_alter_commands, commands_str.data(), commands_str.data() + commands_str.length(), "mutation commands list", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); +} + +} + void MutationCommands::writeText(WriteBuffer & out, bool with_pure_metadata_commands) const { - writeEscapedString(ast(with_pure_metadata_commands)->formatWithSecretsOneLine(), out); + String commands_str = ast(with_pure_metadata_commands)->formatWithSecretsOneLine(); + + /// Check that the serialized commands can be parsed back the same way `readText` will parse + /// them. If they cannot (i.e. some AST is formatted in a way the parser does not accept), + /// it is much better to fail the query that creates the mutation than to persist an entry + /// that would fail to load - and prevent the whole table from loading - on every server + /// that reads it. + try + { + parseMutationCommandsList(commands_str); + } + catch (Exception & e) + { + e.addMessage( + "Serialized mutation commands cannot be parsed back (this indicates a mismatch between the AST formatter " + "and the parser; it's a bug); refusing to write them out: {}", commands_str); + throw; + } + + writeEscapedString(commands_str, out); } void MutationCommands::readText(ReadBuffer & in, bool with_pure_metadata_commands) @@ -307,9 +338,7 @@ void MutationCommands::readText(ReadBuffer & in, bool with_pure_metadata_command String commands_str; readEscapedString(commands_str, in); - ParserAlterCommandList p_alter_commands; - auto commands_ast = parseQuery( - p_alter_commands, commands_str.data(), commands_str.data() + commands_str.length(), "mutation commands list", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + auto commands_ast = parseMutationCommandsList(commands_str); for (const auto & child : commands_ast->children) { diff --git a/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.reference b/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.reference new file mode 100644 index 000000000000..7df36d63f7ea --- /dev/null +++ b/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.reference @@ -0,0 +1,10 @@ +10 20 +0 20 +0 20 +10 +5 10 +5 10 +5 10 +5 10 +5 5 +5 5 diff --git a/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.sql b/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.sql new file mode 100644 index 000000000000..438e507fb98a --- /dev/null +++ b/tests/queries/0_stateless/04626_mutation_in_partition_query_parameter.sql @@ -0,0 +1,80 @@ +-- Query parameter substitution rewrites `{param:Type}` into `_CAST(value, 'Type')`, and the +-- serialized mutation entry must be parseable back when it is re-read from disk on table load. + +DROP TABLE IF EXISTS t_mutation_param; + +CREATE TABLE t_mutation_param (id UInt64, d Date, flag Nullable(Bool)) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(d) ORDER BY id; + +INSERT INTO t_mutation_param SELECT number, '2026-06-24', NULL FROM numbers(10); +INSERT INTO t_mutation_param SELECT number + 100, '2026-06-25', NULL FROM numbers(10); + +SET param_day = '20260624'; +SET param_date = '2026-06-25'; + +ALTER TABLE t_mutation_param UPDATE flag = true IN PARTITION {day:UInt32} WHERE toYYYYMMDD(d) = {day:UInt32} SETTINGS mutations_sync = 2; + +SELECT countIf(flag), count() FROM t_mutation_param; + +-- The partition can also be specified as an explicitly written cast of a literal. +ALTER TABLE t_mutation_param UPDATE flag = false IN PARTITION _CAST(20260624, 'UInt32') WHERE 1 SETTINGS mutations_sync = 2; + +-- Arbitrary expressions are still not allowed in the partition. +ALTER TABLE t_mutation_param UPDATE flag = true IN PARTITION toYYYYMMDD({date:Date}) WHERE 1 SETTINGS mutations_sync = 2; -- { clientError SYNTAX_ERROR } + +SELECT countIf(flag), count() FROM t_mutation_param; + +-- Make sure the mutation entries written to disk can be parsed back on table load. +DETACH TABLE t_mutation_param; +ATTACH TABLE t_mutation_param; + +SELECT countIf(flag), count() FROM t_mutation_param; + +ALTER TABLE t_mutation_param DROP PARTITION {day:UInt32}; + +SELECT count() FROM t_mutation_param; + +DROP TABLE t_mutation_param; + +-- Tuple-typed query parameters for multi-, one- and zero-field partition keys. + +DROP TABLE IF EXISTS t_mutation_param_tuple2; +CREATE TABLE t_mutation_param_tuple2 (id UInt64, a UInt32, s String, flag Nullable(Bool)) +ENGINE = MergeTree PARTITION BY (a, s) ORDER BY id; +INSERT INTO t_mutation_param_tuple2 SELECT number, 1, 'x', NULL FROM numbers(5); +INSERT INTO t_mutation_param_tuple2 SELECT number + 100, 2, 'y', NULL FROM numbers(5); + +SET param_part_two = '(1,''x'')'; +ALTER TABLE t_mutation_param_tuple2 UPDATE flag = true IN PARTITION {part_two:Tuple(UInt32, String)} WHERE 1 SETTINGS mutations_sync = 2; +SELECT countIf(flag), count() FROM t_mutation_param_tuple2; +DETACH TABLE t_mutation_param_tuple2; +ATTACH TABLE t_mutation_param_tuple2; +SELECT countIf(flag), count() FROM t_mutation_param_tuple2; +DROP TABLE t_mutation_param_tuple2; + +DROP TABLE IF EXISTS t_mutation_param_tuple1; +CREATE TABLE t_mutation_param_tuple1 (id UInt64, a UInt32, flag Nullable(Bool)) +ENGINE = MergeTree PARTITION BY a ORDER BY id; +INSERT INTO t_mutation_param_tuple1 SELECT number, 1, NULL FROM numbers(5); +INSERT INTO t_mutation_param_tuple1 SELECT number + 100, 2, NULL FROM numbers(5); + +SET param_part_one = '(1)'; +ALTER TABLE t_mutation_param_tuple1 UPDATE flag = true IN PARTITION {part_one:Tuple(UInt32)} WHERE 1 SETTINGS mutations_sync = 2; +SELECT countIf(flag), count() FROM t_mutation_param_tuple1; +DETACH TABLE t_mutation_param_tuple1; +ATTACH TABLE t_mutation_param_tuple1; +SELECT countIf(flag), count() FROM t_mutation_param_tuple1; +DROP TABLE t_mutation_param_tuple1; + +DROP TABLE IF EXISTS t_mutation_param_tuple0; +CREATE TABLE t_mutation_param_tuple0 (id UInt64, flag Nullable(Bool)) +ENGINE = MergeTree PARTITION BY tuple() ORDER BY id; +INSERT INTO t_mutation_param_tuple0 SELECT number, NULL FROM numbers(5); + +SET param_part_zero = '()'; +ALTER TABLE t_mutation_param_tuple0 UPDATE flag = true IN PARTITION {part_zero:Tuple()} WHERE 1 SETTINGS mutations_sync = 2; +SELECT countIf(flag), count() FROM t_mutation_param_tuple0; +DETACH TABLE t_mutation_param_tuple0; +ATTACH TABLE t_mutation_param_tuple0; +SELECT countIf(flag), count() FROM t_mutation_param_tuple0; +DROP TABLE t_mutation_param_tuple0; diff --git a/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.reference b/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.reference new file mode 100644 index 000000000000..c160ba8107ca --- /dev/null +++ b/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.reference @@ -0,0 +1,3 @@ +10 20 +10 20 +10 20 diff --git a/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.sql b/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.sql new file mode 100644 index 000000000000..0d25b473f887 --- /dev/null +++ b/tests/queries/0_stateless/04627_mutation_in_partition_query_parameter_replicated.sql @@ -0,0 +1,31 @@ +-- Tags: zookeeper + +-- Query parameter substitution rewrites `{param:Type}` into `_CAST(value, 'Type')`, and the +-- serialized mutation entry must be parseable back when it is re-read from ZooKeeper. + +DROP TABLE IF EXISTS t_mutation_param_r SYNC; + +CREATE TABLE t_mutation_param_r (id UInt64, d Date, flag Nullable(Bool)) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_mutation_param_r', 'r1') +PARTITION BY toYYYYMMDD(d) ORDER BY id; + +INSERT INTO t_mutation_param_r SELECT number, '2026-06-24', NULL FROM numbers(10); +INSERT INTO t_mutation_param_r SELECT number + 100, '2026-06-25', NULL FROM numbers(10); + +SET param_day = '20260624'; + +ALTER TABLE t_mutation_param_r UPDATE flag = true IN PARTITION {day:UInt32} WHERE toYYYYMMDD(d) = {day:UInt32} SETTINGS mutations_sync = 2; + +SELECT countIf(flag), count() FROM t_mutation_param_r; + +-- Make sure the mutation entries written to ZooKeeper can be parsed back when the table is loaded. +DETACH TABLE t_mutation_param_r; +ATTACH TABLE t_mutation_param_r; + +SELECT countIf(flag), count() FROM t_mutation_param_r; + +SYSTEM RESTART REPLICA t_mutation_param_r; + +SELECT countIf(flag), count() FROM t_mutation_param_r; + +DROP TABLE t_mutation_param_r SYNC; From 16259aed4dd14f3b760d371d2341ce3a189b167a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Jul 2026 15:26:46 +0000 Subject: [PATCH 36/86] Backport #111425 to 26.6: Fix peer certificate leak in Poco `SecureSocketImpl::verifyPeerCertificateImpl` --- base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 8644069ab6fb..60a55fec88bb 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -468,23 +468,27 @@ long SecureSocketImpl::verifyPeerCertificateImpl(const std::string& hostName) (mode != Context::VERIFY_STRICT && isLocalHost(hostName))) return X509_V_OK; + // SSL_get1_peer_certificate returns a certificate whose reference count has + // been incremented; the caller owns that reference and must X509_free it, + // otherwise the peer certificate leaks on every verified handshake. X509* pCert = SSL_get1_peer_certificate(_pSSL); if (pCert) { + long result = X509_V_ERR_APPLICATION_VERIFICATION; if (X509_check_host(pCert, hostName.c_str(), hostName.length(), 0, nullptr) == 1) { - return X509_V_OK; + result = X509_V_OK; } else { IPAddress ip; if (IPAddress::tryParse(hostName, ip)) { - auto result = X509_check_ip_asc(pCert, hostName.c_str(), 0) == 1; - return result ? X509_V_OK : X509_V_ERR_APPLICATION_VERIFICATION; + result = X509_check_ip_asc(pCert, hostName.c_str(), 0) == 1 ? X509_V_OK : X509_V_ERR_APPLICATION_VERIFICATION; } } - return X509_V_ERR_APPLICATION_VERIFICATION;; + X509_free(pCert); + return result; } else return X509_V_OK; } From 15d0e85b421ecc88f9700e8118add4e7038bf7b3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Jul 2026 19:15:11 +0000 Subject: [PATCH 37/86] Backport #111142 to 26.6: Fix CREATE OR REPLACE of a dictionary with an object of another kind --- src/Interpreters/InterpreterCreateQuery.cpp | 5 ++++ ..._or_replace_view_over_dictionary.reference | 6 +++++ ...create_or_replace_view_over_dictionary.sql | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.reference create mode 100644 tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.sql diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7fba368de6a6..ced702fa3be4 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -2380,6 +2380,11 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, if (!interpreter_rename.renamedInsteadOfExchange()) { + /// After the exchange the temporary name holds the replaced table, which may be of a different + /// kind than the new one (e.g. a dictionary replaced by a view), so the drop must match its kind. + if (auto replaced = DatabaseCatalog::instance().tryGetTable(StorageID{create.getDatabase(), create.getTable()}, current_context)) + ast_drop->is_dictionary = replaced->isDictionary(); + /// Target table was replaced with new one, drop old table auto drop_context = make_drop_context(); InterpreterDropQuery(ast_drop, drop_context).execute(); diff --git a/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.reference b/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.reference new file mode 100644 index 000000000000..5f1bdd497672 --- /dev/null +++ b/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.reference @@ -0,0 +1,6 @@ +1 +42 +43 +1 +0 +0 diff --git a/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.sql b/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.sql new file mode 100644 index 000000000000..96e4b9c203cb --- /dev/null +++ b/tests/queries/0_stateless/04613_create_or_replace_view_over_dictionary.sql @@ -0,0 +1,23 @@ +-- The behavior of cross-kind replaces may change in the future, but each one must either succeed +-- or clearly fail before any change is committed, never leaving an orphan `_tmp_replace_*` object behind. + +CREATE TABLE src (key String, value UInt64) ENGINE = MergeTree ORDER BY key; +INSERT INTO src VALUES ('k1', 1); + +CREATE DICTIONARY dict_then_view (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +SELECT dictGet(dict_then_view, 'value', 'k1'); + +CREATE OR REPLACE VIEW dict_then_view AS SELECT 42 AS value; +SELECT value FROM dict_then_view; + +CREATE DICTIONARY dict_then_table (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +CREATE OR REPLACE TABLE dict_then_table (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO dict_then_table VALUES (43); +SELECT x FROM dict_then_table; + +CREATE DICTIONARY dict_then_dict (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +CREATE OR REPLACE DICTIONARY dict_then_dict (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +SELECT dictGet(dict_then_dict, 'value', 'k1'); + +SELECT count() FROM system.dictionaries WHERE database = currentDatabase() AND name IN ('dict_then_view', 'dict_then_table'); +SELECT count() FROM system.tables WHERE database = currentDatabase() AND startsWith(name, '_tmp_replace_'); From 7bbbe3f2638a9b17fad2083ed84c98f43a537632 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 25 Jul 2026 17:48:16 +0000 Subject: [PATCH 38/86] Backport #107739 to 26.6: Fix crash in ProtobufRowInputFormat when a valid row precedes a bad row --- .../Formats/Impl/ProtobufRowInputFormat.cpp | 10 ++++- ...rotobuf_skip_bad_row_after_valid.reference | 2 + ...04409_protobuf_skip_bad_row_after_valid.sh | 41 +++++++++++++++++++ ...09_protobuf_skip_bad_row_after_valid.proto | 5 +++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.reference create mode 100755 tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.sh create mode 100644 tests/queries/0_stateless/format_schemas/04409_protobuf_skip_bad_row_after_valid.proto diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp index 51d0120fc6d3..681a06df4f43 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp @@ -53,14 +53,22 @@ void ProtobufRowInputFormat::destroyReaderAndSerializer() bool ProtobufRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & row_read_extension) try { + bool serializer_recreated = false; if (!reader) + { createReaderAndSerializer(); + serializer_recreated = true; + } if (reader->eof()) return false; + /// Point the serializer at the current columns before reading. Besides the start + /// of a block (row_num == 0), this is also needed mid-block when error recovery + /// recreated the serializer with a valid row already buffered (row_num > 0) — + /// otherwise readRow dereferences its null column. size_t row_num = columns.empty() ? 0 : columns[0]->size(); - if (!row_num) + if (!row_num || serializer_recreated) serializer->setColumns(columns.data(), columns.size()); serializer->readRow(row_num); diff --git a/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.reference b/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.reference new file mode 100644 index 000000000000..61fd42e3ec52 --- /dev/null +++ b/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.reference @@ -0,0 +1,2 @@ +2024-01-15 +2025-06-16 diff --git a/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.sh b/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.sh new file mode 100755 index 000000000000..a7aa92940ffb --- /dev/null +++ b/tests/queries/0_stateless/04409_protobuf_skip_bad_row_after_valid.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# Regression test for a server crash (SIGSEGV) in ProtobufRowInputFormat when a valid +# message precedes a bad (skippable) message in the same block, with +# input_format_allow_errors_num > 0. After a parse error the serializer is destroyed and +# recreated on the next row; before the fix setColumns was called only when the block was +# empty (row_num == 0), so the recreated serializer kept null column pointers and +# dereferenced them in ProtobufSerializer::readRow. +# See https://github.com/ClickHouse/ClickHouse/issues/107644 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CUR_DIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eo pipefail + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS protobuf_skip_bad_row_after_valid" +$CLICKHOUSE_CLIENT --query "CREATE TABLE protobuf_skip_bad_row_after_valid (d Date) ENGINE = MergeTree ORDER BY d" + +BINARY_FILE_PATH=$(mktemp "$CLICKHOUSE_TMP/04409_protobuf_skip_bad_row_after_valid.XXXXXX.binary") +trap 'rm -f "$BINARY_FILE_PATH"' EXIT + +# Length-delimited Protobuf stream of three messages for `message Row { string d = 1; }`. +# Each message is prefixed by its length; field 1 has wire type LENGTH_DELIMITED (tag 0x0a). +# 1) valid date "2024-01-15" +# 2) bad value "bad!" (skippable: CANNOT_PARSE_DATE) +# 3) valid date "2025-06-16" +{ + printf '\x0c\x0a\x0a'; printf '2024-01-15' + printf '\x06\x0a\x04'; printf 'bad!' + printf '\x0c\x0a\x0a'; printf '2025-06-16' +} > "$BINARY_FILE_PATH" + +# Must not crash: the bad message is skipped and both valid rows are inserted. +$CLICKHOUSE_CLIENT --input_format_allow_errors_num 10 --query "INSERT INTO protobuf_skip_bad_row_after_valid SETTINGS format_schema = '$SCHEMADIR/04409_protobuf_skip_bad_row_after_valid.proto:Row' FORMAT Protobuf" < "$BINARY_FILE_PATH" + +$CLICKHOUSE_CLIENT --query "SELECT d FROM protobuf_skip_bad_row_after_valid ORDER BY d" + +$CLICKHOUSE_CLIENT --query "DROP TABLE protobuf_skip_bad_row_after_valid" diff --git a/tests/queries/0_stateless/format_schemas/04409_protobuf_skip_bad_row_after_valid.proto b/tests/queries/0_stateless/format_schemas/04409_protobuf_skip_bad_row_after_valid.proto new file mode 100644 index 000000000000..806cb9bb7e35 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/04409_protobuf_skip_bad_row_after_valid.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +message Row { + string d = 1; +} From 678beac0d4f9ec2b91fd9aaf97594eade9b22dd9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 26 Jul 2026 13:04:13 +0000 Subject: [PATCH 39/86] Backport #111279 to 26.6: Infer wildcard partition strategy from a `{_partition_id}` path on CREATE --- .../integrations/azureBlobStorage.md | 4 +- .../engines/table-engines/integrations/s3.md | 4 +- .../table-functions/azureBlobStorage.md | 4 +- docs/en/sql-reference/table-functions/s3.md | 4 +- src/Core/Settings.cpp | 2 +- .../StorageObjectStorageConfiguration.cpp | 27 ++++++++----- .../registerStorageObjectStorage.cpp | 8 ++-- ...tion_id_compatibility_validation.reference | 1 + ..._partition_id_compatibility_validation.sql | 15 ++++++-- ...ard_partition_strategy_from_path.reference | 8 ++++ ..._wildcard_partition_strategy_from_path.sql | 38 +++++++++++++++++++ 11 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.reference create mode 100644 tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.sql diff --git a/docs/en/engines/table-engines/integrations/azureBlobStorage.md b/docs/en/engines/table-engines/integrations/azureBlobStorage.md index 8417f03d7b67..41dc8789f1a9 100644 --- a/docs/en/engines/table-engines/integrations/azureBlobStorage.md +++ b/docs/en/engines/table-engines/integrations/azureBlobStorage.md @@ -29,7 +29,7 @@ CREATE TABLE azure_blob_storage_table (name String, value UInt32) - `account_key` - if storage_account_url is used, then account key can be specified here - `format` — The [format](/interfaces/formats.md) of the file. - `compression` — Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension. (same as setting to `auto`). -- `partition_strategy` – Options: `wildcard` or `hive`. `wildcard` requires a `{_partition_id}` in the path, which is replaced with the partition key. `hive` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`wildcard` under `compatibility` settings older than `26.6`, `hive` otherwise). +- `partition_strategy` – Options: `wildcard` or `hive`. `wildcard` requires a `{_partition_id}` in the path, which is replaced with the partition key. `hive` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `wildcard` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`wildcard` under `compatibility` settings older than `26.6`, `hive` otherwise). - `partition_columns_in_data_file` - Only used with `hive` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. - `extra_credentials` - Use `client_id` and `tenant_id` for authentication. If extra_credentials are provided, they are given priority over `account_name` and `account_key`. @@ -105,7 +105,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da #### Partition strategy {#partition-strategy} -`wildcard`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default only under `compatibility` settings older than `26.6`; otherwise the default is `hive` (see the `file_like_engine_default_partition_strategy` setting). +`wildcard`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `hive` (see the `file_like_engine_default_partition_strategy` setting). `hive` implements hive style partitioning for reads & writes. Reading is implemented using a recursive glob pattern. Writing generates files using the following format: `//.`. diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 926d68c0f1b7..d54b85697864 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -45,7 +45,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) - `format` — The [format](/sql-reference/formats#formats-overview) of the file. - `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). - `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension. -- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). +- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). - `partition_columns_in_data_file` - Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. - `storage_class_name` - Options: `STANDARD` or `INTELLIGENT_TIERING`, allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). - `extra_credentials` - Optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. @@ -89,7 +89,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da #### Partition strategy {#partition-strategy} -`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default only under `compatibility` settings older than `26.6`; otherwise the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). +`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). `HIVE` implements hive style partitioning for reads & writes. Reading is implemented using a recursive glob pattern, it is equivalent to `SELECT * FROM s3('table_root/**.parquet')`. Writing generates files using the following format: `//.`. diff --git a/docs/en/sql-reference/table-functions/azureBlobStorage.md b/docs/en/sql-reference/table-functions/azureBlobStorage.md index c1ef2a9645e9..52bb591f812b 100644 --- a/docs/en/sql-reference/table-functions/azureBlobStorage.md +++ b/docs/en/sql-reference/table-functions/azureBlobStorage.md @@ -64,7 +64,7 @@ azureBlobStorage(named_collection[, option=value [,..]]) | `compression` | Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension (same as setting to `auto`). | | `partition_strategy` | Optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as file names and the file format as the extension. | | `structure` | Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. | -| `partition_strategy` | Optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). | +| `partition_strategy` | Optional. Supported values: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). | | `partition_columns_in_data_file` | Optional. Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. | | `extra_credentials` | Use `client_id` and `tenant_id` for authentication. If extra_credentials are provided, they are given priority over `account_name` and `account_key`. | @@ -197,7 +197,7 @@ FROM azureBlobStorage( Supported for INSERT queries only. -`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Selected by default only under `compatibility` settings older than `26.6`; otherwise the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). +`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). `HIVE` implements hive style partitioning for reads & writes. It generates files using the following format: `//.`. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index d4c152207974..c0f052ee2c42 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -48,7 +48,7 @@ For GCS, substitute your HMAC key and HMAC secret where you see `access_key_id` | `structure` | Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. | | `compression_method` | Parameter is optional. Supported values: `none`, `gzip` or `gz`, `brotli` or `br`, `xz` or `LZMA`, `zstd` or `zst`. By default, it will autodetect compression method by file extension. | | `headers` | Parameter is optional. Allows headers to be passed in the S3 request. Pass in the format `headers(key=value)` e.g. `headers('x-amz-request-payer' = 'requester')`. | -| `partition_strategy` | Parameter is optional. Supported values: `wildcard` or `hive`. `wildcard` requires a `{_partition_id}` in the path, which is replaced with the partition key. `hive` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`wildcard` under `compatibility` settings older than `26.6`, `hive` otherwise). | +| `partition_strategy` | Parameter is optional. Supported values: `wildcard` or `hive`. `wildcard` requires a `{_partition_id}` in the path, which is replaced with the partition key. `hive` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `wildcard` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`wildcard` under `compatibility` settings older than `26.6`, `hive` otherwise). | | `partition_columns_in_data_file` | Parameter is optional. Only used with `hive` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. | | `extra_credentials` | Parameter is optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. | | `storage_class_name` | Parameter is optional. Supported values: `STANDARD` or `INTELLIGENT_TIERING`. Allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). Defaults to `STANDARD`. | @@ -248,7 +248,7 @@ FROM s3(creds, url='https://s3-object-url.csv') Supported for INSERT queries only. -`wildcard`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Selected by default only under `compatibility` settings older than `26.6`; otherwise the default is `hive` (see the `file_like_engine_default_partition_strategy` setting). +`wildcard`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `hive` (see the `file_like_engine_default_partition_strategy` setting). `hive` implements hive style partitioning for reads & writes. It generates files using the following format: `//.`. diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index adb9dac41723..22f98671627b 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7434,7 +7434,7 @@ Enables delta-kernel writes feature. Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference) )", 0) \ DECLARE(FileLikeEngineDefaultPartitionStrategy, file_like_engine_default_partition_strategy, FileLikeEngineDefaultPartitionStrategy::HIVE, R"( -Default partition strategy for file like engines. +Default partition strategy for file like engines. Applied only when the path does not contain a `{_partition_id}` placeholder: such a path is compatible only with the `wildcard` strategy, so it always implies `wildcard`. )", 0) \ DECLARE(Bool, use_iceberg_partition_pruning, true, R"( Use Iceberg partition pruning for Iceberg tables diff --git a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp index 7703a992cc52..a7c100f47af6 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.cpp @@ -190,7 +190,18 @@ void StorageObjectStorageConfiguration::initPartitionStrategy(ASTPtr partition_b /// `partition_columns_in_data_file = 0` combined with strategy `none`) keep raising. if (partition_by && partition_strategy_type == PartitionStrategyFactory::StrategyType::NONE && !isDataLakeConfiguration()) { - if (!is_create_query) + if (getRawPath().hasPartitionWildcard()) + { + /// A `{_partition_id}` placeholder in the path is valid only under the `wildcard` + /// strategy — `hive` rejects such paths. When no explicit `partition_strategy` is + /// given, the path alone therefore determines the only strategy that can work, so + /// apply it regardless of `file_like_engine_default_partition_strategy`. Consulting + /// the `hive` default here instead would reject with `BAD_ARGUMENTS` every pre-26.6 + /// CREATE statement that uses a `{_partition_id}` path, breaking existing DDL. + /// An explicit `partition_strategy = 'hive'` still rejects such paths. + partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; + } + else if (!is_create_query) { /// Backward compatibility on ATTACH / server startup / RESTORE / replicated-DDL replay: /// for a table loaded from existing metadata the implicit strategy is deterministically @@ -198,12 +209,10 @@ void StorageObjectStorageConfiguration::initPartitionStrategy(ASTPtr partition_b /// path shape — wildcard REQUIRES `{_partition_id}` in the path, hive FORBIDS it. Consulting /// the mutable `file_like_engine_default_partition_strategy` default here instead would /// refuse to load legitimately created tables whenever the default has changed since - /// creation (pre-26.6 wildcard tables under the 26.6 `hive` default, or implicit-hive - /// tables loaded under a `wildcard` default after a downgrade), aborting server startup - /// and breaking upgrades. Only a user-issued `CREATE` applies the default. - partition_strategy_type = getRawPath().hasPartitionWildcard() - ? PartitionStrategyFactory::StrategyType::WILDCARD - : PartitionStrategyFactory::StrategyType::HIVE; + /// creation (e.g. implicit-hive tables loaded under a `wildcard` default after a + /// downgrade), aborting server startup and breaking upgrades. Only a user-issued + /// `CREATE` applies the default. + partition_strategy_type = PartitionStrategyFactory::StrategyType::HIVE; } else { @@ -211,8 +220,8 @@ void StorageObjectStorageConfiguration::initPartitionStrategy(ASTPtr partition_b { case FileLikeEngineDefaultPartitionStrategy::WILDCARD: { - /// Set the strategy unconditionally; `PartitionStrategyFactory::get` will raise - /// `BAD_ARGUMENTS` if the path is missing the `{_partition_id}` placeholder. + /// The path has no `{_partition_id}` placeholder (checked above), so + /// `PartitionStrategyFactory::get` will raise `BAD_ARGUMENTS`. partition_strategy_type = PartitionStrategyFactory::StrategyType::WILDCARD; break; } diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index e52d484bd8cd..d90a89743117 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -148,7 +148,7 @@ CREATE TABLE azure_blob_storage_table (name String, value UInt32) - `account_key` - if storage_account_url is used, then account key can be specified here - `format` — The [format](/interfaces/formats.md) of the file. - `compression` — Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension. (same as setting to `auto`). -- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). +- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). - `partition_columns_in_data_file` - Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. - `extra_credentials` - Use `client_id` and `tenant_id` for authentication. If extra_credentials are provided, they are given priority over `account_name` and `account_key`. @@ -224,7 +224,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da #### Partition strategy {#partition-strategy} -`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. +`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). `HIVE` (the default) implements hive style partitioning for reads & writes. Reading is implemented using a recursive glob pattern. Writing generates files using the following format: `//.`. @@ -302,7 +302,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) - `format` — The [format](/sql-reference/formats#formats-overview) of the file. - `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). - `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension. -- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. Defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). +- `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). - `partition_columns_in_data_file` - Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. - `storage_class_name` - Options: `STANDARD` or `INTELLIGENT_TIERING`, allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). - `extra_credentials` - Optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. @@ -346,7 +346,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da #### Partition strategy {#partition-strategy} -`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. +`WILDCARD`: Replaces the `{_partition_id}` wildcard in the file path with the actual partition key. Reading is not supported. Selected by default when the path contains a `{_partition_id}` placeholder (the only strategy compatible with such a path), and otherwise under `compatibility` settings older than `26.6`; in the remaining cases the default is `HIVE` (see the `file_like_engine_default_partition_strategy` setting). `HIVE` (the default) implements hive style partitioning for reads & writes. Reading is implemented using a recursive glob pattern, it is equivalent to `SELECT * FROM s3('table_root/**.parquet')`. Writing generates files using the following format: `//.`. diff --git a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference index d24715a06873..9befde0f1eda 100644 --- a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference +++ b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.reference @@ -1,3 +1,4 @@ +0 1 1 2 diff --git a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql index 32dc5efd2bf6..4659e94c0229 100644 --- a/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql +++ b/tests/queries/0_stateless/04337_s3_hive_partition_id_compatibility_validation.sql @@ -1,9 +1,19 @@ -- Tags: no-fasttest, no-random-settings -- Tag no-fasttest: Depends on S3 +-- A `{_partition_id}` placeholder in the path is valid only under the `wildcard` strategy. +-- When no explicit `partition_strategy` is given, the path shape determines the strategy +-- regardless of the `file_like_engine_default_partition_strategy` default, so pre-26.6 DDL +-- keeps working under the 26.6 `hive` default. SET compatibility = '26.6'; CREATE TABLE old_export (d Date, x UInt64) ENGINE = S3('s3://bucket/export/data_{_partition_id}.parquet', 'Parquet') +PARTITION BY d; +SELECT 0; + +-- An explicit `partition_strategy = 'hive'` with a `{_partition_id}` path must still be rejected. +CREATE TABLE old_export_explicit_hive (d Date, x UInt64) +ENGINE = S3('s3://bucket/export/data_{_partition_id}.parquet', 'Parquet', partition_strategy='hive') PARTITION BY d; -- {serverError BAD_ARGUMENTS} SET compatibility = '26.5'; @@ -21,8 +31,7 @@ SELECT 1; -- Backward compatibility: a pre-26.6 table with a `{_partition_id}` path (implicit wildcard) -- must still load via ATTACH under the 26.6 `hive` default — the same code path the server --- takes for every such table at startup and during upgrades. Before the fix this threw --- `BAD_ARGUMENTS` and aborted server startup. +-- takes for every such table at startup and during upgrades. -- The explicit `hive` below is required: `SET compatibility` does not override the -- explicitly-set `file_like_engine_default_partition_strategy = 'wildcard'` above, and the -- ATTACH must run with the `hive` default in effect to be a real regression test. @@ -32,7 +41,7 @@ DETACH TABLE old_export_compat_265; ATTACH TABLE old_export_compat_265; SELECT 2; -DROP TABLE IF EXISTS old_export; -- never created: the first CREATE above is expected to throw +DROP TABLE old_export; DROP TABLE old_export_compat_265; DROP TABLE old_export2; diff --git a/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.reference b/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.reference new file mode 100644 index 000000000000..e0ca2de3349c --- /dev/null +++ b/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.reference @@ -0,0 +1,8 @@ +1 a +22 b +333 c +1 a +22 b +333 c +4444 d +55 e diff --git a/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.sql b/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.sql new file mode 100644 index 000000000000..6b3988072d7d --- /dev/null +++ b/tests/queries/0_stateless/04614_implicit_wildcard_partition_strategy_from_path.sql @@ -0,0 +1,38 @@ +-- Tags: no-fasttest, no-random-settings +-- Tag no-fasttest: Depends on S3 + +-- When no explicit `partition_strategy` is given, a `{_partition_id}` placeholder in the +-- path implies the `wildcard` strategy regardless of the +-- `file_like_engine_default_partition_strategy` default, because `hive` cannot work with +-- such a path anyway. This keeps pre-26.6 DDL working under the 26.6 `hive` default. + +-- All S3 keys are prefixed with `currentDatabase()` so that parallel and repeated runs +-- of this test do not see each other's objects. + +SET file_like_engine_default_partition_strategy = 'hive'; + +CREATE TABLE test_04614_implicit_wildcard (a UInt64, b String) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/tbl_{_partition_id}', format = Parquet) +PARTITION BY a; + +SET s3_truncate_on_insert = 1; +INSERT INTO test_04614_implicit_wildcard VALUES (1, 'a'), (22, 'b'), (333, 'c'); +SELECT a, b FROM s3(s3_conn, filename = currentDatabase() || '/tbl_*', format = Parquet) ORDER BY a; + +-- The table must survive DETACH / ATTACH with the `hive` default still in effect. +DETACH TABLE test_04614_implicit_wildcard; +ATTACH TABLE test_04614_implicit_wildcard; +INSERT INTO test_04614_implicit_wildcard VALUES (4444, 'd'); +SELECT a, b FROM s3(s3_conn, filename = currentDatabase() || '/tbl_*', format = Parquet) ORDER BY a; + +-- The same path with an explicit `partition_strategy = 'hive'` must still be rejected. +CREATE TABLE test_04614_explicit_hive (a UInt64, b String) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/hive_{_partition_id}', format = Parquet, partition_strategy = 'hive') +PARTITION BY a; -- {serverError BAD_ARGUMENTS} + +-- The implicit wildcard also applies to table functions (INSERT INTO FUNCTION ... PARTITION BY). +INSERT INTO FUNCTION s3(s3_conn, filename = currentDatabase() || '/fn_{_partition_id}', format = Parquet) +PARTITION BY a SELECT 55::UInt64 AS a, 'e' AS b; +SELECT a, b FROM s3(s3_conn, filename = currentDatabase() || '/fn_*', format = Parquet) ORDER BY a; + +DROP TABLE test_04614_implicit_wildcard; From 6b4f5f8c7c4a2c9aaf58df552b403650c370e38d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 26 Jul 2026 16:04:50 +0000 Subject: [PATCH 40/86] Backport #111857 to 26.6: Fix TYPE_MISMATCH in direct dictionary join with a Nullable or LowCardinality key --- src/Dictionaries/IDictionary.h | 49 ++++++++ ...ect_join_dictionary_nullable_key.reference | 40 +++++++ ...27_direct_join_dictionary_nullable_key.sql | 109 ++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.reference create mode 100644 tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.sql diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index dbd64640259a..b027d31eabcf 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include #include @@ -15,6 +17,8 @@ #include #include #include +#include +#include namespace DB { @@ -367,6 +371,40 @@ class IDictionary : public IExternalLoadable, public IKeyValueEntity key_types.emplace_back(key.type); } + /// Normalize wrapper-compatible key columns to the declared key schema via `convertKeyColumns`. + /// Track NULL rows and unwrap the outer `Nullable` first: casting a real NULL to a + /// non-Nullable declared type would throw. + PaddedPODArray key_row_is_null; + for (size_t i = 0; i < key_columns.size(); ++i) + { + if (!isNullableOrLowCardinalityNullable(key_types[i])) + continue; + + /// Drop `Const`/`Sparse`/`ColumnReplicated` and `LowCardinality` (as `HashJoin` prepares + /// probe keys) so the null map is exposed, then peel off the outer `Nullable`. + ColumnPtr full_column = recursiveRemoveLowCardinality(removeSpecialRepresentations(key_columns[i]->convertToFullColumnIfConst())); + DataTypePtr full_type = recursiveRemoveLowCardinality(key_types[i]); + + if (const auto * nullable = checkAndGetColumn(full_column.get())) + { + const auto & row_null_map = nullable->getNullMapData(); + if (key_row_is_null.empty()) + key_row_is_null.resize_fill(row_null_map.size(), 0); + for (size_t row = 0; row < row_null_map.size(); ++row) + key_row_is_null[row] |= row_null_map[row]; + + key_columns[i] = nullable->getNestedColumnPtr(); + key_types[i] = removeNullable(full_type); + } + else + { + key_columns[i] = std::move(full_column); + key_types[i] = std::move(full_type); + } + } + + convertKeyColumns(key_columns, key_types); + /// Fill null map { out_null_map.clear(); @@ -376,6 +414,12 @@ class IDictionary : public IExternalLoadable, public IKeyValueEntity out_null_map.resize(mask_data.size(), 0); std::copy(mask_data.begin(), mask_data.end(), out_null_map.begin()); + + /// A NULL key never matches, even if its unwrapped value coincides with a stored key + /// (e.g. NULL unwrapped to the empty string could otherwise hit a stored empty key). + for (size_t row = 0; row < key_row_is_null.size(); ++row) + if (key_row_is_null[row]) + out_null_map[row] = 0; } Names attribute_names; @@ -408,6 +452,11 @@ class IDictionary : public IExternalLoadable, public IKeyValueEntity Columns result_columns = getColumns(attribute_names, result_types, key_columns, key_types, default_cols); + /// Blank attributes for NULL-key rows so a coincidental match yields defaults like a miss. + if (!key_row_is_null.empty()) + for (auto & result_column : result_columns) + result_column = JoinCommon::filterWithBlanks(result_column, out_null_map); + /// Result block should consist of key columns and then attributes for (const auto & key_col : key_columns) { diff --git a/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.reference b/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.reference new file mode 100644 index 000000000000..4cafe154d88d --- /dev/null +++ b/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.reference @@ -0,0 +1,40 @@ +Nullable(String) key, LEFT, join_use_nulls=0 +a A +b B +x +\N +Nullable(String) key, LEFT, join_use_nulls=1 +a A +b B +x \N +\N \N +Nullable(String) key, INNER +a A +b B +LowCardinality(String) key, LEFT +a A +x +LowCardinality(Nullable(String)) key, LEFT +a A +b B +\N +NULL key does not match a real empty-string dictionary key +a A +\N +A genuine empty-string key still matches + EMPTYHIT +A constant NULL key does not match the empty-string dictionary key +\N +Dictionary with Nullable(UInt64) key, Nullable probe with NULL +1 one +2 two +\N +Nullable key carried by a sparse column, LEFT +a A +\N +Nullable key replicated by ARRAY JOIN, LEFT +a A +a A +\N +Direct join is still chosen for the Nullable key +1 diff --git a/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.sql b/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.sql new file mode 100644 index 000000000000..2c4355265c5d --- /dev/null +++ b/tests/queries/0_stateless/04627_direct_join_dictionary_nullable_key.sql @@ -0,0 +1,109 @@ +-- Tags: no-parallel-replicas +-- DirectKeyValueJoin (the algorithm under test) cannot be chosen with parallel replicas, so the +-- ParallelReplicas runner variant would throw NOT_IMPLEMENTED instead of exercising the fix. +-- Direct JOIN onto a dictionary with a wrapped (Nullable / LowCardinality / LowCardinality(Nullable)) +-- left join key used to throw "Key type for complex key ... does not match" (TYPE_MISMATCH). +-- The direct-join lookup now normalizes the key to the dictionary's declared key schema, and a NULL +-- key never matches. See https://github.com/ClickHouse/ClickHouse/issues/111829 + +DROP DICTIONARY IF EXISTS dict_str; +DROP DICTIONARY IF EXISTS dict_empty; +DROP DICTIONARY IF EXISTS dict_nullable_key; +DROP TABLE IF EXISTS src_str; +DROP TABLE IF EXISTS src_empty; +DROP TABLE IF EXISTS src_nullable_key; + +CREATE TABLE src_str (k String, v String) ENGINE = MergeTree ORDER BY k; +INSERT INTO src_str VALUES ('a', 'A'), ('b', 'B'), ('c', 'C'); +CREATE DICTIONARY dict_str (k String, v String) +PRIMARY KEY k SOURCE(CLICKHOUSE(TABLE 'src_str' DB currentDatabase())) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(0); + +-- Dictionary that has a real empty-string key: a NULL probe key must not coincide with it. +CREATE TABLE src_empty (k String, v String) ENGINE = MergeTree ORDER BY k; +INSERT INTO src_empty VALUES ('', 'EMPTYHIT'), ('a', 'A'); +CREATE DICTIONARY dict_empty (k String, v String) +PRIMARY KEY k SOURCE(CLICKHOUSE(TABLE 'src_empty' DB currentDatabase())) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(0); + +-- Dictionary whose declared key is itself Nullable: normalizing the probe wrapper must not break it. +CREATE TABLE src_nullable_key (k Nullable(UInt64), v String) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO src_nullable_key VALUES (1, 'one'), (2, 'two'); +CREATE DICTIONARY dict_nullable_key (k Nullable(UInt64), v String) +PRIMARY KEY k SOURCE(CLICKHOUSE(TABLE 'src_nullable_key' DB currentDatabase())) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(0); + +-- A Nullable key can also be carried by a sparse column or replicated by ARRAY JOIN; the lookup must +-- expose the null map through those special representations too. +CREATE TABLE src_sparse (k Nullable(String)) ENGINE = MergeTree ORDER BY tuple() +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.0; +INSERT INTO src_sparse VALUES ('a'), (NULL); + +-- `k` is carried (not array-joined) through ARRAY JOIN of `arr`, so with lazy replication it becomes +-- a ColumnReplicated wrapping a Nullable column. +CREATE TABLE src_array (k Nullable(String), arr Array(UInt8)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO src_array VALUES ('a', [1, 2]), (NULL, [3]); + +SET join_algorithm = 'direct'; + +SELECT 'Nullable(String) key, LEFT, join_use_nulls=0'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', 'b', 'x', NULL]::Array(Nullable(String))) AS pref) AS t +LEFT JOIN dict_str AS dd ON t.pref = dd.k ORDER BY t.pref NULLS LAST, dd.v SETTINGS join_use_nulls = 0; + +SELECT 'Nullable(String) key, LEFT, join_use_nulls=1'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', 'b', 'x', NULL]::Array(Nullable(String))) AS pref) AS t +LEFT JOIN dict_str AS dd ON t.pref = dd.k ORDER BY t.pref NULLS LAST, dd.v SETTINGS join_use_nulls = 1; + +SELECT 'Nullable(String) key, INNER'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', 'b', 'x', NULL]::Array(Nullable(String))) AS pref) AS t +INNER JOIN dict_str AS dd ON t.pref = dd.k ORDER BY t.pref, dd.v; + +-- The analyzer casts LowCardinality away before the join; enable_analyzer = 0 keeps the +-- LowCardinality key so it survives to the dictionary lookup and exercises the stripping in getByKeys. +SELECT 'LowCardinality(String) key, LEFT'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', 'x']::Array(LowCardinality(String))) AS pref) AS t +LEFT JOIN dict_str AS dd ON t.pref = dd.k ORDER BY t.pref, dd.v SETTINGS enable_analyzer = 0; + +SELECT 'LowCardinality(Nullable(String)) key, LEFT'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', 'b', NULL]::Array(LowCardinality(Nullable(String)))) AS pref) AS t +LEFT JOIN dict_str AS dd ON t.pref = dd.k ORDER BY t.pref NULLS LAST, dd.v SETTINGS enable_analyzer = 0; + +SELECT 'NULL key does not match a real empty-string dictionary key'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin(['a', NULL]::Array(Nullable(String))) AS pref) AS t +LEFT JOIN dict_empty AS dd ON t.pref = dd.k ORDER BY t.pref NULLS LAST, dd.v; + +SELECT 'A genuine empty-string key still matches'; +SELECT t.pref, dd.v FROM (SELECT CAST('', 'Nullable(String)') AS pref) AS t +LEFT JOIN dict_empty AS dd ON t.pref = dd.k; + +-- A constant NULL arrives as a ColumnConst, unlike the arrayJoin NULLs above; it must still not match. +SELECT 'A constant NULL key does not match the empty-string dictionary key'; +SELECT t.pref, dd.v FROM (SELECT CAST(NULL, 'Nullable(String)') AS pref) AS t +LEFT JOIN dict_empty AS dd ON t.pref = dd.k; + +-- Nullable-declared dictionary key with a Nullable probe: direct join is selected here, so the +-- wrapper normalization must keep both non-null lookups and the NULL non-match correct. +SELECT 'Dictionary with Nullable(UInt64) key, Nullable probe with NULL'; +SELECT t.pref, dd.v FROM (SELECT arrayJoin([1, 2, NULL]::Array(Nullable(UInt64))) AS pref) AS t +LEFT JOIN dict_nullable_key AS dd ON t.pref = dd.k ORDER BY t.pref NULLS LAST, dd.v; + +SELECT 'Nullable key carried by a sparse column, LEFT'; +SELECT s.k, dd.v FROM src_sparse AS s +LEFT JOIN dict_str AS dd ON s.k = dd.k ORDER BY s.k NULLS LAST, dd.v; + +SELECT 'Nullable key replicated by ARRAY JOIN, LEFT'; +SELECT p.k, dd.v FROM (SELECT k FROM src_array ARRAY JOIN arr) AS p +LEFT JOIN dict_str AS dd ON p.k = dd.k ORDER BY p.k NULLS LAST, dd.v +SETTINGS enable_lazy_columns_replication = 1; + +SELECT 'Direct join is still chosen for the Nullable key'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 + SELECT count() FROM (SELECT CAST('a', 'Nullable(String)') AS pref) AS t + LEFT JOIN dict_str AS dd ON t.pref = dd.k) +WHERE explain ILIKE '%Algorithm: DirectKeyValueJoin%'; + +DROP DICTIONARY dict_str; +DROP DICTIONARY dict_empty; +DROP DICTIONARY dict_nullable_key; +DROP TABLE src_str; +DROP TABLE src_empty; +DROP TABLE src_nullable_key; +DROP TABLE src_sparse; +DROP TABLE src_array; From 23c0887c0a18f6eb715b279f73642ab9506a1fc8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 26 Jul 2026 20:45:38 +0000 Subject: [PATCH 41/86] Backport #111589 to 26.6: Add setting `analyzer_compatibility_apply_final_to_all_joined_tables` --- src/Core/Settings.cpp | 8 ++++ src/Core/SettingsChangesHistory.cpp | 2 + src/Planner/Utils.cpp | 3 ++ src/Storages/SelectQueryInfo.cpp | 2 +- src/Storages/SelectQueryInfo.h | 6 +++ ...tibility_final_all_joined_tables.reference | 12 ++++++ ..._compatibility_final_all_joined_tables.sql | 41 +++++++++++++++++++ 7 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.reference create mode 100644 tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 22f98671627b..1e6a02bcb7fd 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7639,6 +7639,14 @@ When enabled, the new analyzer mimics the legacy behavior of moving non-aggregat DECLARE(Bool, analyzer_compatibility_prefer_alias_over_subcolumn, false, R"( When a multi-part identifier like `b.id` could refer to either the column `id` of a table aliased `b` or to a Tuple subcolumn `b.id` of some other column, prefer the alias-prefix interpretation (column `id` of `b`). By default the new analyzer prefers the subcolumn. Enable to match the old analyzer's resolution. )", 0) \ + DECLARE(Bool, analyzer_compatibility_apply_final_to_all_joined_tables, false, R"( +Restores the behavior of versions before 26.6, where the `FINAL` modifier specified on the left-most table of a JOIN was incorrectly applied to all other joined tables as well (for engines that support `FINAL`, e.g. `ReplacingMergeTree`). By default `FINAL` applies only to the table it is written on. Enable for compatibility with queries that rely on the old behavior; the recommended fix is to write `FINAL` explicitly on every table that needs it. + +Possible values: + +- 0 - `FINAL` applies only to the table it is specified on. +- 1 - `FINAL` on the left-most table of a JOIN is applied to all joined tables. +)", 0) \ DECLARE(Bool, enable_identifier_resolve_cache, true, R"( Enable the identifier resolution cache in the query analyzer. The cache shares resolved alias nodes to prevent AST explosion when the same alias is referenced multiple times. Set to false to disable caching if incorrect results are suspected. )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index de1418ea0de9..dcf7f8814427 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,6 +41,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.6", { + {"analyzer_compatibility_apply_final_to_all_joined_tables", true, false, "Fixed a bug in the analyzer where FINAL on the left-most table of a JOIN was incorrectly applied to the other joined tables as well. previous_value=true so `compatibility` with versions before 26.6 restores the old behavior."}, {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the new analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, {"output_format_image_width", 1024, 1024, "New setting controlling the width of the output image for image output formats such as PNG."}, @@ -148,6 +149,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.4", { + {"analyzer_compatibility_apply_final_to_all_joined_tables", true, true, "New compatibility setting controlling whether FINAL on the left-most table of a JOIN is applied to the other joined tables. Introduced with default true (the old behavior) for backports to versions before 26.6."}, {"max_bytes_before_external_join", 0, 0, "New setting to control automatic spilling of hash joins to disk. Non-zero value enables spilling and sets the byte threshold."}, {"allow_iceberg_remove_orphan_files", false, false, "New setting to gate Iceberg orphan file removal"}, {"iceberg_orphan_files_older_than_seconds", 259200, 259200, "New setting for default orphan file age threshold"}, diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index e250a66002cf..f883c452dcf8 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -59,6 +59,7 @@ namespace DB namespace Setting { extern const SettingsString additional_result_filter; + extern const SettingsBool analyzer_compatibility_apply_final_to_all_joined_tables; extern const SettingsUInt64 max_bytes_to_read; extern const SettingsUInt64 max_bytes_to_read_leaf; extern const SettingsSeconds max_estimated_execution_time; @@ -559,6 +560,8 @@ SelectQueryInfo buildSelectQueryInfo(const QueryTreeNodePtr & query_tree, const select_query_info.query = queryNodeToSelectQuery(query_tree); select_query_info.query_tree = query_tree; select_query_info.planner_context = planner_context; + select_query_info.apply_query_level_final_if_no_modifiers + = planner_context->getQueryContext()->getSettingsRef()[Setting::analyzer_compatibility_apply_final_to_all_joined_tables]; return select_query_info; } diff --git a/src/Storages/SelectQueryInfo.cpp b/src/Storages/SelectQueryInfo.cpp index 95d3c01979a3..1b2f024d4661 100644 --- a/src/Storages/SelectQueryInfo.cpp +++ b/src/Storages/SelectQueryInfo.cpp @@ -15,7 +15,7 @@ bool SelectQueryInfo::isFinal() const if (table_expression_modifiers) return table_expression_modifiers->hasFinal(); - if (query_tree) + if (query_tree && !apply_query_level_final_if_no_modifiers) return false; const auto & select = query->as(); diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 1aab52c1fd86..bcad61aeebbf 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -140,6 +140,12 @@ struct SelectQueryInfo /// Table expression modifiers for storage std::optional table_expression_modifiers; + /// Value of the `analyzer_compatibility_apply_final_to_all_joined_tables` setting. + /// When true, `isFinal` falls back to the query-level FINAL (the left-most table's modifier) + /// for table expressions without their own modifiers, restoring the pre-26.6 behavior + /// where FINAL on one table of a JOIN leaked onto the other joined tables. + bool apply_query_level_final_if_no_modifiers = false; + std::shared_ptr storage_limits; /// Local storage limits diff --git a/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.reference b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.reference new file mode 100644 index 000000000000..718266507fd7 --- /dev/null +++ b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.reference @@ -0,0 +1,12 @@ +FINAL on the left table only, default behavior +2 +FINAL on the left table only, compatibility setting enabled +1 +FINAL on the left table only, compatibility with an older version +1 +FINAL on both tables is unaffected by the setting +1 +1 +FINAL on the right table only does not leak to the left table even with the setting +2 +2 diff --git a/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql new file mode 100644 index 000000000000..9e93ef62edfb --- /dev/null +++ b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql @@ -0,0 +1,41 @@ +-- Compatibility setting for the fix of FINAL leaking onto other tables of a JOIN +-- (https://github.com/ClickHouse/ClickHouse/pull/108979). + +DROP TABLE IF EXISTS t_left; +DROP TABLE IF EXISTS t_right; + +CREATE TABLE t_left (id Int64, right_id Int64, ver UInt64) ENGINE = ReplacingMergeTree(ver) ORDER BY id; +CREATE TABLE t_right (id Int64, attr String, ver UInt64) ENGINE = ReplacingMergeTree(ver) ORDER BY id; + +SYSTEM STOP MERGES t_left; +SYSTEM STOP MERGES t_right; + +-- One logical row per table, two unmerged versions each (separate parts). +INSERT INTO t_left VALUES (1, 10, 1); +INSERT INTO t_left VALUES (1, 10, 2); +INSERT INTO t_right VALUES (10, 'car', 1); +INSERT INTO t_right VALUES (10, 'car', 2); + +SELECT 'FINAL on the left table only, default behavior'; +SELECT count() FROM t_left AS l FINAL INNER JOIN t_right AS r ON r.id = l.right_id; + +SELECT 'FINAL on the left table only, compatibility setting enabled'; +SELECT count() FROM t_left AS l FINAL INNER JOIN t_right AS r ON r.id = l.right_id +SETTINGS analyzer_compatibility_apply_final_to_all_joined_tables = 1; + +SELECT 'FINAL on the left table only, compatibility with an older version'; +SELECT count() FROM t_left AS l FINAL INNER JOIN t_right AS r ON r.id = l.right_id +SETTINGS compatibility = '26.5'; + +SELECT 'FINAL on both tables is unaffected by the setting'; +SELECT count() FROM t_left AS l FINAL INNER JOIN t_right AS r FINAL ON r.id = l.right_id; +SELECT count() FROM t_left AS l FINAL INNER JOIN t_right AS r FINAL ON r.id = l.right_id +SETTINGS analyzer_compatibility_apply_final_to_all_joined_tables = 1; + +SELECT 'FINAL on the right table only does not leak to the left table even with the setting'; +SELECT count() FROM t_left AS l INNER JOIN t_right AS r FINAL ON r.id = l.right_id; +SELECT count() FROM t_left AS l INNER JOIN t_right AS r FINAL ON r.id = l.right_id +SETTINGS analyzer_compatibility_apply_final_to_all_joined_tables = 1; + +DROP TABLE t_left; +DROP TABLE t_right; From e88249fa072b0c18b5d500131471c7c2bae1a74c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 27 Jul 2026 10:04:39 +0000 Subject: [PATCH 42/86] Backport #110292 to 26.6: Refresh S3Queue bucket locks so the TTL cleanup does not remove them mid-processing --- src/Common/ProfileEvents.cpp | 2 + .../ObjectStorageQueueMetadata.cpp | 7 +- .../ObjectStorageQueueOrderedFileMetadata.cpp | 130 ++++++- .../ObjectStorageQueueOrderedFileMetadata.h | 15 + .../ObjectStorageQueueSource.cpp | 53 ++- .../ObjectStorageQueueSource.h | 11 + .../StorageObjectStorageQueue.cpp | 6 +- .../test_file_iterator_lost_lock.py | 368 ++++++++++++++++++ .../test_file_iterator_ttl.py | 92 +++++ 9 files changed, 664 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_storage_s3_queue/test_file_iterator_lost_lock.py create mode 100644 tests/integration/test_storage_s3_queue/test_file_iterator_ttl.py diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7f35ef869e92..1caf1b8a6525 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1101,6 +1101,8 @@ The server successfully detected this situation and will download merged part fr M(ObjectStorageQueueTaggedObjects, "Number of objects tagged as part of after_processing = tag", ValueType::Number) \ M(ObjectStorageQueueInsertIterations, "Number of insert iterations", ValueType::Number) \ M(ObjectStorageQueueCommitRequests, "Number of keeper requests to commit files as either failed or processed", ValueType::Number) \ + M(ObjectStorageQueueBucketLockLostOwnership, "Number of times ownership of a bucket lock was detected as lost in S3(Azure)Queue. Non-zero value indicates too small persistent_processing_node_ttl_seconds or a bug", ValueType::Number) \ + M(ObjectStorageQueueBucketLockRefreshes, "Number of successful bucket lock refreshes in S3(Azure)Queue", ValueType::Number) \ M(ObjectStorageQueueSuccessfulCommits, "Number of successful keeper commits", ValueType::Number) \ M(ObjectStorageQueueUnsuccessfulCommits, "Number of unsuccessful keeper commits", ValueType::Number) \ M(ObjectStorageQueueCancelledFiles, "Number cancelled files in StorageS3(Azure)Queue", ValueType::Number) \ diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index ea28acb10ce6..c871ddeee2a8 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -325,7 +325,8 @@ std::optional ObjectStorageQueueMetadata::getStartAfterForListing() ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr ObjectStorageQueueMetadata::tryAcquireBucket(const Bucket & bucket) { - return ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket(zookeeper_path, bucket, use_persistent_processing_nodes, zookeeper_name, log); + return ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket( + zookeeper_path, bucket, use_persistent_processing_nodes, persistent_processing_node_ttl_seconds, zookeeper_name, log); } void ObjectStorageQueueMetadata::alterSettings(const SettingsChanges & changes, const ContextPtr & context) @@ -1238,7 +1239,9 @@ void ObjectStorageQueueMetadata::cleanupThreadFuncImpl() return; } - if (cleanup_processing_files) + /// Check the TTL as well: it is changeable at runtime and zero disables + /// the cleanup (otherwise every node would be treated as stale). + if (cleanup_processing_files && persistent_processing_node_ttl_seconds) cleanupPersistentProcessingNodes(); if (table_metadata.hasTrackedFilesLimit()) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp index e0ab4f3502a0..2a06d5797478 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,6 +11,12 @@ #include +namespace ProfileEvents +{ + extern const Event ObjectStorageQueueBucketLockLostOwnership; + extern const Event ObjectStorageQueueBucketLockRefreshes; +} + namespace DB { namespace ErrorCodes @@ -205,6 +212,7 @@ ObjectStorageQueueOrderedFileMetadata::BucketHolder::BucketHolder( const Bucket & bucket_, const std::string & bucket_lock_path_, const std::string & processor_info_, + const std::atomic & persistent_processing_node_ttl_seconds_, LoggerPtr log_, const std::string & zookeeper_name_) : bucket_info(std::make_shared(BucketInfo{ @@ -212,6 +220,7 @@ ObjectStorageQueueOrderedFileMetadata::BucketHolder::BucketHolder( .bucket_lock_path = bucket_lock_path_, .processor_info = processor_info_, .zookeeper_name = zookeeper_name_ })) + , persistent_processing_node_ttl_seconds(persistent_processing_node_ttl_seconds_) , log(log_) { #ifdef DEBUG_OR_SANITIZER_BUILD @@ -244,6 +253,64 @@ std::optional ObjectStorageQueueOrderedFileMetadata::BucketHolder:: return std::nullopt; } +void ObjectStorageQueueOrderedFileMetadata::BucketHolder::refresh() +{ + /// Released holders are removed from the iterator's bucket_holders, + /// so refresh is not expected to be called on a released holder. + chassert(!released); + if (released) + return; + + bool ownership_lost = false; + std::optional current_owner; + auto zk_retry = ObjectStorageQueueMetadata::getKeeperRetriesControl(log); + zk_retry.retryLoop([&] + { + auto zk_client = ObjectStorageQueueMetadata::getZooKeeper(log, bucket_info->zookeeper_name); + + Coordination::Stat stat; + std::string data; + if (!zk_client->tryGet(bucket_info->bucket_lock_path, data, &stat) || data != bucket_info->processor_info) + { + ownership_lost = true; + if (!data.empty()) + current_owner = data; + return; + } + + /// Rewrite the same data to update mtime of the lock node. + /// Version check protects from updating a lock re-created by another server. + Coordination::Stat set_stat; + auto code = zk_client->trySet(bucket_info->bucket_lock_path, data, stat.version, &set_stat); + if (code == Coordination::Error::ZOK) + { + bucket_lock_version = set_stat.version; + return; + } + if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZNONODE) + ownership_lost = true; + else + throw zkutil::KeeperException::fromPath(code, bucket_info->bucket_lock_path); + }); + + if (ownership_lost) + { + /// Must never happen: the lock was removed as abandoned by the TTL cleanup and possibly + /// acquired by another server (`persistent_processing_node_ttl_seconds` too small, or a + /// bug), which can cause duplicates. released is set to not remove someone else's lock. + released = true; + ProfileEvents::increment(ProfileEvents::ObjectStorageQueueBucketLockLostOwnership); + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Lost ownership of bucket lock {} (processor: {}, current owner: {})", + bucket_info->bucket_lock_path, bucket_info->processor_info, current_owner.value_or("none")); + } + + ProfileEvents::increment(ProfileEvents::ObjectStorageQueueBucketLockRefreshes); + LOG_TEST(log, "Refreshed bucket lock {}", bucket_info->bucket_lock_path); + age_watch.restart(); +} + void ObjectStorageQueueOrderedFileMetadata::BucketHolder::release() { if (released) @@ -254,40 +321,69 @@ void ObjectStorageQueueOrderedFileMetadata::BucketHolder::release() LOG_TEST(log, "Releasing bucket {}", bucket_info->bucket); Coordination::Error code = {}; + bool ownership_lost = false; auto zk_retry = ObjectStorageQueueMetadata::getKeeperRetriesControl(log); zk_retry.retryLoop([&] { auto zk_client = ObjectStorageQueueMetadata::getZooKeeper(log, bucket_info->zookeeper_name); - if (zk_retry.isRetry()) + + if (zk_retry.isRetry() && !checkBucketOwnership(zk_client)) { - /// It is possible that we fail "after operation", - /// e.g. we successfully removed the node, but did not get confirmation, - /// but then if we retry - we can remove a newly recreated node, - /// therefore avoid this with this check. - if (!checkBucketOwnership(zk_client)) - { - LOG_TEST(log, "Will not remove bucket lock node, ownership changed"); - code = Coordination::Error::ZOK; - return; - } + /// We could have failed "after operation": the node was removed without + /// confirmation and could be re-created by another server since then. + LOG_TEST(log, "Will not remove bucket lock node, ownership changed"); + code = Coordination::Error::ZOK; + return; } - else + /// A lock not refreshed for longer than the TTL could have been removed by the + /// cleanup and re-created by another server colliding on the version (e.g. both + /// at the creation version 0), so the version check alone cannot prove ownership. + const size_t ttl_seconds = persistent_processing_node_ttl_seconds.load(); + if (ttl_seconds + && age_watch.elapsedSeconds() >= static_cast(ttl_seconds) + && !checkBucketOwnership(zk_client)) { - chassert(checkBucketOwnership(zk_client)); + ownership_lost = true; + return; + } + /// The version check below protects from removing a lock re-created by + /// another server; on the first attempt also assert ownership in debug builds. + chassert(zk_retry.isRetry() || checkBucketOwnership(zk_client)); + code = zk_client->tryRemove(bucket_info->bucket_lock_path, bucket_lock_version); + if (code == Coordination::Error::ZBADVERSION) + { + /// The version is stale if a refresh succeeded without confirmation. + /// Re-read to distinguish that from a lock re-created by another server. + Coordination::Stat stat; + std::string data; + if (zk_client->tryGet(bucket_info->bucket_lock_path, data, &stat) + && data == bucket_info->processor_info) + { + bucket_lock_version = stat.version; + code = zk_client->tryRemove(bucket_info->bucket_lock_path, bucket_lock_version); + } } - code = zk_client->tryRemove(bucket_info->bucket_lock_path); }); - if (code == Coordination::Error::ZOK) + if (!ownership_lost && code == Coordination::Error::ZOK) { LOG_TEST(log, "Released bucket {}", bucket_info->bucket); return; } - else if (zk_retry.isRetry() && code == Coordination::Error::ZNONODE) + else if (!ownership_lost && zk_retry.isRetry() && code == Coordination::Error::ZNONODE) { LOG_TEST(log, "Released bucket {} (has zk session loss)", bucket_info->bucket); return; } + else if (ownership_lost || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZBADVERSION) + { + ProfileEvents::increment(ProfileEvents::ObjectStorageQueueBucketLockLostOwnership); + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Lost ownership of bucket lock {} detected during release (processor: {}, error: {})", + bucket_info->bucket_lock_path, bucket_info->processor_info, + ownership_lost ? "ownership check failed" : Coordination::errorMessage(code)); + } throw zkutil::KeeperException::fromPath(code, bucket_info->bucket_lock_path); } @@ -567,6 +663,7 @@ ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr ObjectStorageQueueOrdered const std::filesystem::path & zk_path, const Bucket & bucket, bool /*use_persistent_processing_nodes_*/, + const std::atomic & persistent_processing_node_ttl_seconds_, const std::string & zookeeper_name_, LoggerPtr log_) { @@ -614,6 +711,7 @@ ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr ObjectStorageQueueOrdered bucket, bucket_lock_path, processor_info, + persistent_processing_node_ttl_seconds_, log_, zookeeper_name_); } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h index 6b213cb73a55..2496b4a91027 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,7 @@ class ObjectStorageQueueOrderedFileMetadata : public ObjectStorageQueueIFileMeta const std::filesystem::path & zk_path, const Bucket & bucket, bool use_persistent_processing_nodes_, + const std::atomic & persistent_processing_node_ttl_seconds_, const std::string & zookeeper_name_, LoggerPtr log_); @@ -167,6 +169,7 @@ struct ObjectStorageQueueOrderedFileMetadata::BucketHolder : private boost::nonc const Bucket & bucket_, const std::string & bucket_lock_path_, const std::string & processor_info_, + const std::atomic & persistent_processing_node_ttl_seconds_, LoggerPtr log_, const std::string & zookeeper_name_); @@ -178,6 +181,13 @@ struct ObjectStorageQueueOrderedFileMetadata::BucketHolder : private boost::nonc void setFinished() { finished = true; } bool isFinished() const { return finished; } + /// Time since the bucket lock node was created or last refreshed. + double getAgeSeconds() const { return age_watch.elapsedSeconds(); } + + /// Update mtime of the bucket lock node, so that it is not removed as abandoned by + /// the TTL cleanup. Throws a logical error on lost ownership, marking the holder released. + void refresh(); + bool checkBucketOwnership(std::shared_ptr zk_client); std::optional getProcessorInfo(std::shared_ptr zk_client); @@ -185,6 +195,11 @@ struct ObjectStorageQueueOrderedFileMetadata::BucketHolder : private boost::nonc private: BucketInfoPtr bucket_info; + Stopwatch age_watch; + int32_t bucket_lock_version = 0; + /// A reference, not a snapshot: the setting is changeable at runtime, + /// and release must use the same TTL as the cleanup. + const std::atomic & persistent_processing_node_ttl_seconds; bool released = false; bool finished = false; LoggerPtr log; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index 6e4e97fb0d1e..adcb519416d6 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -169,6 +169,9 @@ ObjectStorageQueueSource::FileIterator::FileIterator( bool ObjectStorageQueueSource::FileIterator::isFinished() { + if (iterator_invalidated) + return true; + std::lock_guard lock(mutex); LOG_TEST(log, "Iterator finished: {}, objects to retry: {}", iterator_finished.load(), objects_to_retry.size()); return iterator_finished @@ -502,7 +505,15 @@ ObjectInfoPtr ObjectStorageQueueSource::FileIterator::next(size_t processor) if (use_buckets_for_processing) { + refreshExpiringBucketLocks(); + std::lock_guard lock(mutex); + + if (iterator_invalidated) + { + LOG_WARNING(log, "Bucket lock refresh failed, stopping the file iterator"); + return {}; + } auto result = getNextKeyFromAcquiredBucket(processor); object_info = result.object_info; file_metadata = result.file_metadata; @@ -602,6 +613,38 @@ void ObjectStorageQueueSource::FileIterator::returnForRetry(ObjectInfoPtr object } } +void ObjectStorageQueueSource::FileIterator::refreshExpiringBucketLocks() +{ + const size_t ttl_seconds = metadata->getPersistentProcessingNodeTTLSeconds(); + if (!ttl_seconds) + return; + + std::lock_guard lock(mutex); + + /// Already invalidated (possibly by another thread), nothing to refresh. + /// Checked under the mutex to never hit the released holder of the lost lock. + if (iterator_invalidated) + return; + for (auto & [processor, holders] : bucket_holders) + { + for (auto & holder : *holders) + { + if (holder->getAgeSeconds() >= static_cast(ttl_seconds) / 4) + { + try + { + holder->refresh(); + } + catch (...) + { + iterator_invalidated = true; + throw; + } + } + } + } +} + void ObjectStorageQueueSource::FileIterator::releaseFinishedBuckets() { std::lock_guard lock(mutex); @@ -627,7 +670,15 @@ void ObjectStorageQueueSource::FileIterator::releaseFinishedBuckets() chassert(holder->isFinished()); /// Release bucket lock. - holder->release(); + try + { + holder->release(); + } + catch (...) + { + iterator_invalidated = true; + throw; + } ++released_holders; /// Reset bucket processor in cached state. diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h index 380622de68f8..233b0343287f 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -73,6 +74,11 @@ class ObjectStorageQueueSource final : public ISource, WithContext /// because we want to be able to rethrow exceptions if they might happen. void releaseFinishedBuckets(); + /// Refresh bucket locks which were not refreshed for more than a quarter of + /// the TTL, after which the cleanup removes them as abandoned (the TTL is + /// meant to remove locks of dead servers). + void refreshExpiringBucketLocks(); + bool useBucketsForProcessing() const { return use_buckets_for_processing; } private: @@ -124,6 +130,10 @@ class ObjectStorageQueueSource final : public ISource, WithContext /// Is glob_iterator finished? std::atomic_bool iterator_finished = false; + /// Set when a bucket lock refresh or release fails (e.g. lost ownership): + /// next() stops returning keys, isFinished returns true. + std::atomic_bool iterator_invalidated = false; + bool is_path_with_hive_partitioning = false; /// Only for processing without buckets. @@ -137,6 +147,7 @@ class ObjectStorageQueueSource final : public ISource, WithContext }; NextKeyFromBucket getNextKeyFromAcquiredBucket(size_t processor) TSA_REQUIRES(mutex); std::string bucketHoldersToString() const TSA_REQUIRES(mutex); + BucketHolderPtr tryAcquireBucket( size_t bucket, BucketInfo & bucket_info, diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 343fb3b8408b..792b447aa311 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -825,6 +825,8 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) // Create a stream for each consumer and join them in a union stream // Only insert into dependent views and expect that input blocks contain virtual columns + Stopwatch watch; + auto table_id = getStorageID(); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); if (!table) @@ -966,6 +968,7 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) getCurrentExceptionCode()); file_iterator->releaseFinishedBuckets(); + file_iterator->refreshExpiringBucketLocks(); /// Halve the global batch size so that on the next iteration the bad file /// ends up in a smaller batch, eventually alone (batch size 1), @@ -997,11 +1000,12 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) commit(/*insert_succeeded=*/ true, rows, sources, transaction_start_time); file_iterator->releaseFinishedBuckets(); + file_iterator->refreshExpiringBucketLocks(); max_files_override = 0; total_rows += rows; } - LOG_TEST(log, "Processed rows: {}", total_rows); + LOG_TEST(log, "Processed rows: {}, elapsed: {} ms", total_rows, watch.elapsedMilliseconds()); return total_rows > 0; } diff --git a/tests/integration/test_storage_s3_queue/test_file_iterator_lost_lock.py b/tests/integration/test_storage_s3_queue/test_file_iterator_lost_lock.py new file mode 100644 index 000000000000..ba5a6524c89c --- /dev/null +++ b/tests/integration/test_storage_s3_queue/test_file_iterator_lost_lock.py @@ -0,0 +1,368 @@ +import logging +import time +import uuid + +import pytest +from kazoo.exceptions import BadVersionError, NoNodeError + +from helpers.cluster import ClickHouseCluster +from helpers.s3_queue_common import ( + generate_random_files, + create_table, +) + + +def wait_for_processed_files_in_log(node, table_name, expected_count): + """A file row reaches the destination table before the file is committed + as processed in Keeper, so the s3queue log can lag behind the data.""" + count = 0 + for _ in range(60): + node.query("SYSTEM FLUSH LOGS") + count = int( + node.query( + f"SELECT uniqExact(file_name) FROM system.s3queue_log " + f"WHERE table = '{table_name}' AND status = 'Processed'" + ) + ) + if count == expected_count: + break + time.sleep(1) + return count + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "instance", + user_configs=["configs/users.xml"], + with_minio=True, + with_zookeeper=True, + main_configs=[ + "configs/zookeeper.xml", + "configs/s3queue_log.xml", + ], + stay_alive=True, + ) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + # The test deliberately provokes a logical error + # about lost bucket lock ownership. + cluster.shutdown(ignore_logical_errors=True) + + +def test_streaming_recovers_after_lost_bucket_lock(started_cluster): + node = started_cluster.instances["instance"] + + if node.is_debug_build() or node.is_built_with_sanitizer(): + pytest.skip( + "Debug and sanitizer builds abort on the deliberately provoked " + "logical error about lost bucket lock ownership" + ) + + table_name = f"test_lost_bucket_lock_{uuid.uuid4().hex[:8]}" + dst_table_name = f"{table_name}_dst" + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + files_to_generate = 300 + + # With TTL = 2 sec bucket locks are refreshed once they were not + # refreshed for 0.5 sec, so a stolen lock is detected quickly. + create_table( + started_cluster, + node, + table_name, + "ordered", + files_path, + additional_settings={ + "keeper_path": keeper_path, + "buckets": 3, + "processing_threads_num": 3, + "persistent_processing_node_ttl_seconds": 2, + # Keep the TTL cleanup away from the test: with TTL = 2 sec it could + # remove processing nodes of files which take longer than the TTL and + # bucket locks which stop being refreshed after the provoked ownership loss. + "cleanup_interval_min_ms": 600000, + "cleanup_interval_max_ms": 600000, + "max_processed_files_before_commit": 1, + "polling_min_timeout_ms": 100, + # Files in flight at the moment the lock loss is detected fail + # together with the batch and must be retried afterwards. + "s3queue_loading_retries": 10, + }, + ) + + generate_random_files( + started_cluster, files_path, files_to_generate, start_ind=0, row_num=1 + ) + + node.query( + f""" + CREATE TABLE {dst_table_name} + (column1 UInt32, column2 UInt32, column3 UInt32, _path String) + ENGINE = MergeTree ORDER BY column1 + """ + ) + # Throttle the processing with sleepEachRow, so that it certainly + # takes long enough to steal a bucket lock in the middle of it. + node.query( + f""" + CREATE MATERIALIZED VIEW {table_name}_mv TO {dst_table_name} AS + SELECT column1, column2, column3, _path + FROM {table_name} + WHERE ignore(sleepEachRow(0.2)) = 0 + """ + ) + + def get_processed_count(): + return int(node.query(f"SELECT uniqExact(_path) FROM {dst_table_name}")) + + # Wait until streaming is in progress and holds bucket locks. + for _ in range(150): + if get_processed_count() >= files_to_generate // 10: + break + time.sleep(1) + assert get_processed_count() >= files_to_generate // 10 + + # Steal one bucket lock, as if it was removed by the TTL cleanup + # and the bucket was acquired by another server. A single set is + # atomic, unlike delete + create, so it cannot hit a lock which is + # concurrently released and re-acquired by the server. + zk = started_cluster.get_kazoo_client("zoo1") + stolen_lock_path = None + for _ in range(10): + for bucket in zk.get_children(f"{keeper_path}/buckets"): + lock_path = f"{keeper_path}/buckets/{bucket}/lock" + try: + stat = zk.exists(lock_path) + # Steal a lock acquired at least a second ago, so that the + # debug ownership check in the BucketHolder constructor + # is certainly over. + if stat is None or time.time() - stat.created < 1: + continue + zk.set(lock_path, b"another_server") + stolen_lock_path = lock_path + break + except NoNodeError: + continue + if stolen_lock_path: + break + time.sleep(1) + assert stolen_lock_path + + # The refresh must detect the lost ownership exactly once + # and invalidate the file iterator. + for _ in range(150): + if node.contains_in_log("Lost ownership of bucket lock"): + break + time.sleep(1) + assert node.contains_in_log("Lost ownership of bucket lock") + + # Return the stolen lock, so that the bucket can be acquired again. + # By this time the TTL cleanup could have removed the stolen lock as + # abandoned and the server could have re-acquired the bucket, + # so make sure not to remove a lock owned by the server. + try: + data, stat = zk.get(stolen_lock_path) + if data == b"another_server": + zk.delete(stolen_lock_path, version=stat.version) + except (NoNodeError, BadVersionError): + pass + + # Streaming must recover with a fresh file iterator + # and process all the files. + for _ in range(300): + if get_processed_count() == files_to_generate: + break + time.sleep(1) + assert get_processed_count() == files_to_generate + + # The ownership loss must have been detected exactly once: after the + # iterator invalidation the fresh iterator must not fail on it again. + # ForcedCriticalErrorsLogger mirrors every logical error into the log, + # so filter its duplicate of the same single exception out. + detections = [ + line + for line in node.grep_in_log("Lost ownership of bucket lock").splitlines() + if "(processor: " in line and "ForcedCriticalErrorsLogger" not in line + ] + assert 1 == len(detections) + + # The profile event must count exactly one ownership loss as well. + assert 1 == int( + node.query( + "SELECT value FROM system.events " + "WHERE event = 'ObjectStorageQueueBucketLockLostOwnership'" + ) + ) + + # uniqExact(_path) above proves at-least-once processing. Prove exactly-once + # commits via the s3queue log: every file was set as processed exactly once. + # A duplicates check on the destination table would be flaky by design: + # a file which was in flight when the batch failed can legitimately + # re-insert an already inserted row when it is retried. + assert files_to_generate == wait_for_processed_files_in_log( + node, table_name, files_to_generate + ) + assert "" == node.query( + f"SELECT file_name, count() FROM system.s3queue_log " + f"WHERE table = '{table_name}' AND status = 'Processed' " + f"GROUP BY file_name HAVING count() > 1" + ) + + +def test_lost_bucket_lock_detected_during_release(started_cluster): + node = started_cluster.instances["instance"] + + if node.is_debug_build() or node.is_built_with_sanitizer(): + pytest.skip( + "Debug and sanitizer builds abort on the deliberately provoked " + "logical error about lost bucket lock ownership" + ) + + table_name = f"test_lost_lock_release_{uuid.uuid4().hex[:8]}" + dst_table_name = f"{table_name}_dst" + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + files_to_generate = 30 + + def get_lost_ownership_events(): + return int( + node.query( + "SELECT sum(value) FROM system.events " + "WHERE event = 'ObjectStorageQueueBucketLockLostOwnership'" + ) + ) + + events_before = get_lost_ownership_events() + + create_table( + started_cluster, + node, + table_name, + "ordered", + files_path, + additional_settings={ + "keeper_path": keeper_path, + "buckets": 3, + "processing_threads_num": 3, + "persistent_processing_node_ttl_seconds": 2, + # Keep the TTL cleanup away from the test, same as above. + "cleanup_interval_min_ms": 600000, + "cleanup_interval_max_ms": 600000, + "max_processed_files_before_commit": 1, + "polling_min_timeout_ms": 100, + "s3queue_loading_retries": 10, + }, + ) + + generate_random_files( + started_cluster, files_path, files_to_generate, start_ind=0, row_num=1 + ) + + node.query( + f""" + CREATE TABLE {dst_table_name} + (column1 UInt32, column2 UInt32, column3 UInt32, _path String) + ENGINE = MergeTree ORDER BY column1 + """ + ) + # With TTL = 2 sec, 2.5 sec of sleep per one-row file keeps all the + # processing threads inside the pipeline for longer than the TTL, + # where nothing can refresh the bucket locks. + node.query( + f""" + CREATE MATERIALIZED VIEW {table_name}_mv TO {dst_table_name} AS + SELECT column1, column2, column3, _path + FROM {table_name} + WHERE ignore(sleepEachRow(2.5)) = 0 + """ + ) + + # Steal two bucket locks while all the threads are sleeping inside the + # pipeline and the locks age past the TTL without a refresh. The first + # stolen lock is detected by the refresh scan, which throws immediately + # and invalidates the iterator, so the second one is only discovered by + # release when the invalidated iterator holders are destroyed - the + # release-time detection this test is about. + zk = started_cluster.get_kazoo_client("zoo1") + stolen_lock_paths = [] + for _ in range(100): + for bucket in zk.get_children(f"{keeper_path}/buckets"): + lock_path = f"{keeper_path}/buckets/{bucket}/lock" + if lock_path in stolen_lock_paths: + continue + try: + stat = zk.exists(lock_path) + # Steal locks acquired at least a second ago, so that the + # processing threads are certainly sleeping already. + if stat is None or time.time() - stat.created < 1: + continue + zk.set(lock_path, b"another_server") + stolen_lock_paths.append(lock_path) + if len(stolen_lock_paths) == 2: + break + except NoNodeError: + continue + if len(stolen_lock_paths) == 2: + break + time.sleep(0.2) + assert 2 == len(stolen_lock_paths) + + # One loss must be detected by the refresh, the other by the release. + def get_detections(): + lines = [ + line + for line in node.grep_in_log( + f"Lost ownership of bucket lock {keeper_path}" + ).splitlines() + if "ForcedCriticalErrorsLogger" not in line + ] + return ( + [line for line in lines if "current owner:" in line], + [line for line in lines if "detected during release" in line], + ) + + for _ in range(150): + refresh_detections, release_detections = get_detections() + if refresh_detections and release_detections: + break + time.sleep(1) + refresh_detections, release_detections = get_detections() + assert 1 == len(refresh_detections) + assert 1 == len(release_detections) + assert events_before + 2 == get_lost_ownership_events() + + # Return the stolen locks, so that the buckets can be acquired again. + for lock_path in stolen_lock_paths: + try: + data, stat = zk.get(lock_path) + if data == b"another_server": + zk.delete(lock_path, version=stat.version) + except (NoNodeError, BadVersionError): + pass + + # Streaming must recover and process all the files exactly once. + def get_processed_count(): + return int(node.query(f"SELECT uniqExact(_path) FROM {dst_table_name}")) + + for _ in range(300): + if get_processed_count() == files_to_generate: + break + time.sleep(1) + assert get_processed_count() == files_to_generate + + assert files_to_generate == wait_for_processed_files_in_log( + node, table_name, files_to_generate + ) + assert "" == node.query( + f"SELECT file_name, count() FROM system.s3queue_log " + f"WHERE table = '{table_name}' AND status = 'Processed' " + f"GROUP BY file_name HAVING count() > 1" + ) diff --git a/tests/integration/test_storage_s3_queue/test_file_iterator_ttl.py b/tests/integration/test_storage_s3_queue/test_file_iterator_ttl.py new file mode 100644 index 000000000000..e77021f4c341 --- /dev/null +++ b/tests/integration/test_storage_s3_queue/test_file_iterator_ttl.py @@ -0,0 +1,92 @@ +import logging +import time +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.s3_queue_common import ( + generate_random_files, + create_table, + create_mv, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "instance", + user_configs=["configs/users.xml"], + with_minio=True, + with_zookeeper=True, + main_configs=[ + "configs/zookeeper.xml", + "configs/s3queue_log.xml", + ], + stay_alive=True, + ) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def test_refresh_bucket_locks_on_ttl(started_cluster): + node = started_cluster.instances["instance"] + + table_name = f"test_file_iterator_ttl_{uuid.uuid4().hex[:8]}" + dst_table_name = f"{table_name}_dst" + # A unique path is necessary for repeatable tests + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + files_to_generate = 300 + + # With TTL = 2 sec a bucket lock must be refreshed once it was not + # refreshed for 1 sec, otherwise the TTL cleanup can remove it as + # abandoned. Commit after every file makes processing slow enough + # for the locks to certainly be refreshed several times + # before all files are done. + create_table( + started_cluster, + node, + table_name, + "ordered", + files_path, + additional_settings={ + "keeper_path": keeper_path, + "buckets": 3, + "processing_threads_num": 3, + "persistent_processing_node_ttl_seconds": 2, + "max_processed_files_before_commit": 1, + "polling_min_timeout_ms": 100, + }, + ) + + generate_random_files( + started_cluster, files_path, files_to_generate, start_ind=0, row_num=1 + ) + + create_mv(node, table_name, dst_table_name) + + def get_count(): + return int(node.query(f"SELECT count() FROM {dst_table_name}")) + + for _ in range(150): + if get_count() == files_to_generate: + break + time.sleep(1) + assert get_count() == files_to_generate + + # Bucket locks must have been refreshed at least once. + assert node.contains_in_log("Refreshed bucket lock") + + # Every file must be processed exactly once. + assert files_to_generate == int( + node.query(f"SELECT uniqExact(_path) FROM {dst_table_name}") + ) From 9c9cc19fb1b4ed0e65ee6dbac2af66c7bdc83d09 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 27 Jul 2026 10:07:24 +0000 Subject: [PATCH 43/86] Backport #107562 to 26.6: Fix missing variant_discr stream for Dynamic columns after mutation --- src/Storages/MergeTree/ColumnsSubstreams.cpp | 10 +++ src/Storages/MergeTree/ColumnsSubstreams.h | 3 + src/Storages/MergeTree/MutateTask.cpp | 63 ++++++++++++++++++- ...c_mutation_variant_discr_streams.reference | 8 +++ ...dynamic_mutation_variant_discr_streams.sql | 31 +++++++++ ...utation_dummy_placeholder_stream.reference | 3 + ...amic_mutation_dummy_placeholder_stream.sql | 28 +++++++++ ...tion_old_part_no_substreams_file.reference | 8 +++ ...ic_mutation_old_part_no_substreams_file.sh | 61 ++++++++++++++++++ ...ation_old_part_basic_map_partial.reference | 6 ++ ...412_mutation_old_part_basic_map_partial.sh | 61 ++++++++++++++++++ 11 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.reference create mode 100644 tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.sql create mode 100644 tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.reference create mode 100644 tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.sql create mode 100644 tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.reference create mode 100755 tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.sh create mode 100644 tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.reference create mode 100755 tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.sh diff --git a/src/Storages/MergeTree/ColumnsSubstreams.cpp b/src/Storages/MergeTree/ColumnsSubstreams.cpp index 6b7ca57946f9..98cb99db9421 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.cpp +++ b/src/Storages/MergeTree/ColumnsSubstreams.cpp @@ -130,6 +130,16 @@ const std::vector & ColumnsSubstreams::getColumnSubstreams(size_t column return columns_substreams[column_position].second; } +const std::vector * ColumnsSubstreams::tryGetColumnSubstreams(const String & column_name) const +{ + for (const auto & [name, substreams] : columns_substreams) + { + if (name == column_name) + return &substreams; + } + return nullptr; +} + ColumnsSubstreams ColumnsSubstreams::merge(const ColumnsSubstreams & left, const ColumnsSubstreams & right, const std::vector & columns_order) { std::unordered_map left_column_to_position; diff --git a/src/Storages/MergeTree/ColumnsSubstreams.h b/src/Storages/MergeTree/ColumnsSubstreams.h index 50f00c23777f..26dcec9e270a 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.h +++ b/src/Storages/MergeTree/ColumnsSubstreams.h @@ -36,6 +36,9 @@ class ColumnsSubstreams const std::vector & getColumnSubstreams(size_t column_position) const; + /// Returns the recorded substreams for a column by name, or nullptr if the column is not present. + const std::vector * tryGetColumnSubstreams(const String & column_name) const; + void writeText(WriteBuffer & buf) const; void readText(ReadBuffer & buf); String toString() const; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index e9e923b24578..6c0d7c0585a1 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -123,6 +123,11 @@ enum class ExecuteTTLType : uint8_t namespace MutationHelpers { +/// Placeholder substream that `getColumnsForNewDataPart` records for a column that will be written +/// later by the mutation and is therefore not yet present in the part. It is not a real stream and +/// must never be resolved against the source part's checksums (a part may happen to contain a real +/// column whose name collides with this sentinel). +static const String NOT_YET_WRITTEN_COLUMN_SUBSTREAM_PLACEHOLDER = "dummy"; static bool haveMutationsOfDynamicColumns(const MergeTreeData::DataPartPtr & data_part, const MutationCommands & commands) { @@ -150,6 +155,36 @@ static bool haveMutationsOfDynamicColumns(const MergeTreeData::DataPartPtr & dat return false; } +/// Wide parts written before `columns_substreams.txt` was introduced (in 25.8) can contain a column +/// with a dynamic structure (`Dynamic`, `JSON`, ...) whose data-dependent substreams (`variant_discr`, +/// the variant element streams, ...) are not recorded anywhere we can enumerate without a +/// deserialization state. State-less `serialization->enumerateStreams` stops after `dynamic_structure` +/// for such a column (see `getStreamCounts`), so a partial mutation cannot account for all of its +/// streams and could leave one neither rewritten nor hardlinked into the new part. To stay safe we +/// rewrite the whole part in that case (the resulting part gets a `columns_substreams.txt`, so later +/// mutations can take the partial path again). The file is also discarded when found corrupted, which +/// lands here too. +/// +/// The guard is `hasDynamicStructure`, not the broader `hasDynamicSubcolumns`: only a data-dependent +/// dynamic structure (`Dynamic`, `JSON`) makes state-less enumeration incomplete. A plain `Map` (and a +/// plain `Variant`) reports `hasDynamicSubcolumns` too, but its serialization enumerates all physical +/// streams without a column/state, so forcing a full rewrite for it would be needless (it would turn a +/// cheap single-column mutation of an old part into a rewrite of all the `Map` data). +static bool hasDynamicColumnsWithoutRecordedSubstreams(const MergeTreeData::DataPartPtr & data_part) +{ + if (!isWidePart(data_part)) + return false; + + const auto & columns_substreams = data_part->getColumnsSubstreams(); + for (const auto & column : data_part->getColumns()) + { + if (column.type->hasDynamicStructure() && !columns_substreams.tryGetColumnSubstreams(column.name)) + return true; + } + + return false; +} + static UInt64 getExistingRowsCount(const Block & block) { auto column = block.getByName(RowExistsColumn::name).column; @@ -208,7 +243,8 @@ static void splitAndModifyMutationCommands( auto part_columns = part->getColumnsDescription(); const auto & table_columns = metadata_snapshot->getColumns(); - if (haveMutationsOfDynamicColumns(part, commands) || !isWidePart(part) || !isFullPartStorage(part->getDataPartStorage())) + if (haveMutationsOfDynamicColumns(part, commands) || hasDynamicColumnsWithoutRecordedSubstreams(part) + || !isWidePart(part) || !isFullPartStorage(part->getDataPartStorage())) { NameSet mutated_columns; NameSet dropped_columns; @@ -828,7 +864,7 @@ getColumnsForNewDataPart( if (fill_columns_substreams) { new_columns_substreams.addColumn(it->name); - new_columns_substreams.addSubstreamToLastColumn("dummy"); + new_columns_substreams.addSubstreamToLastColumn(NOT_YET_WRITTEN_COLUMN_SUBSTREAM_PLACEHOLDER); } ++it; @@ -986,9 +1022,31 @@ static std::unordered_map getStreamCounts( const Names & column_names) { std::unordered_map stream_counts; + const auto & columns_substreams = data_part->getColumnsSubstreams(); for (const auto & column_name : column_names) { + /// When columns_substreams.txt is available, prefer its recorded substreams over + /// enumerateStreams. The file is the ground truth of what streams exist on disk, and + /// for columns with a data-dependent dynamic structure (Dynamic, JSON) a state-less + /// enumerateStreams is incomplete: it stops after `dynamic_structure` and never reports + /// data-dependent substreams like `variant_discr`. + const auto * recorded_substreams = columns_substreams.tryGetColumnSubstreams(column_name); + + /// A not-yet-written column in a new part carries only a single placeholder substream + /// (see getColumnsForNewDataPart). It has no real streams on disk yet, so we fall back + /// to enumerateStreams to preserve correct shared-stream accounting for regular columns + /// (e.g. Nested array sizes). + if (recorded_substreams && !(recorded_substreams->size() == 1 && (*recorded_substreams)[0] == NOT_YET_WRITTEN_COLUMN_SUBSTREAM_PLACEHOLDER)) + { + for (const auto & substream : *recorded_substreams) + { + if (auto stream_name = IMergeTreeDataPart::getStreamNameOrHash(substream, ".bin", source_part_checksums)) + ++stream_counts[*stream_name]; + } + continue; + } + if (auto serialization = data_part->tryGetSerialization(column_name)) { auto callback = [&](const ISerialization::SubstreamPath & substream_path) @@ -3510,6 +3568,7 @@ bool MutateTask::prepare() /// Also currently mutations of types with dynamic subcolumns in Wide part are possible only by /// rewriting the whole part. if (MutationHelpers::haveMutationsOfDynamicColumns(ctx->source_part, ctx->commands_for_part) + || MutationHelpers::hasDynamicColumnsWithoutRecordedSubstreams(ctx->source_part) || !isWidePart(ctx->source_part) || !isFullPartStorage(ctx->source_part->getDataPartStorage()) || (ctx->interpreter && ctx->interpreter->isAffectingAllColumns())) diff --git a/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.reference b/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.reference new file mode 100644 index 000000000000..bd694ec35dcd --- /dev/null +++ b/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.reference @@ -0,0 +1,8 @@ +5000 5000 +Array(UInt64) 500 +Decimal(18, 2) 2000 +Float64 500 +Int64 500 +Map(UInt64, UInt64) 1000 +String 500 +1 diff --git a/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.sql b/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.sql new file mode 100644 index 000000000000..f061b17adf4c --- /dev/null +++ b/tests/queries/0_stateless/04341_dynamic_mutation_variant_discr_streams.sql @@ -0,0 +1,31 @@ +-- Mutations that rewrite a Dynamic column (which has data-dependent substreams such as +-- `variant_discr`) interleaved with merges, on Wide parts. Regression coverage for the +-- mutation stream-accounting of dynamic-structure columns; CHECK TABLE validates that +-- every recorded substream of every part is present on disk. +-- See https://github.com/ClickHouse/ClickHouse/issues/107561 + +DROP TABLE IF EXISTS t_dyn_mut; +CREATE TABLE t_dyn_mut (id UInt64, s UInt64, y Dynamic(max_types=3)) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; + +INSERT INTO t_dyn_mut SELECT number, number, number::Int64 FROM numbers(1000); +INSERT INTO t_dyn_mut SELECT number, number, 's' || number FROM numbers(1000); +INSERT INTO t_dyn_mut SELECT number, number, number::Float64 FROM numbers(1000); +INSERT INTO t_dyn_mut SELECT number, number, [number] FROM numbers(1000); +OPTIMIZE TABLE t_dyn_mut FINAL; + +-- Mutation that rewrites the Dynamic column. +ALTER TABLE t_dyn_mut UPDATE y = id::Decimal64(2) WHERE id % 2 = 0 SETTINGS mutations_sync = 2; +-- Mutation that does NOT touch the Dynamic column (its streams must be hardlinked intact). +ALTER TABLE t_dyn_mut UPDATE s = s + 1 WHERE id % 3 = 0 SETTINGS mutations_sync = 2; +-- Type change that re-decides the variant structure. +ALTER TABLE t_dyn_mut MODIFY COLUMN y Dynamic(max_types = 1) SETTINGS mutations_sync = 2; +INSERT INTO t_dyn_mut SELECT number, number, map(number, number) FROM numbers(1000); +OPTIMIZE TABLE t_dyn_mut FINAL; + +SELECT count(), countIf(y IS NOT NULL) FROM t_dyn_mut; +SELECT dynamicType(y) AS t, count() FROM t_dyn_mut GROUP BY t ORDER BY t; +CHECK TABLE t_dyn_mut SETTINGS check_query_single_value_result = 1; + +DROP TABLE t_dyn_mut; diff --git a/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.reference b/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.reference new file mode 100644 index 000000000000..a39ffcb5e499 --- /dev/null +++ b/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.reference @@ -0,0 +1,3 @@ +1000 1000 +Int64 1000 +1 diff --git a/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.sql b/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.sql new file mode 100644 index 000000000000..a1b69eabf3ab --- /dev/null +++ b/tests/queries/0_stateless/04342_dynamic_mutation_dummy_placeholder_stream.sql @@ -0,0 +1,28 @@ +-- Regression for the "dummy" placeholder substream of a not-yet-written Dynamic column. +-- During a partial mutation `getColumnsForNewDataPart` records a placeholder substream for a column +-- that the mutation will write but that is not yet present in the source part. The mutation +-- stream-accounting must not resolve that placeholder against the source part's checksums: if the +-- table happens to contain an unrelated column literally named `dummy`, resolving the placeholder +-- to that column's real `dummy.bin` stream would mark the unchanged stream as skipped and drop it +-- from the new part. +-- See https://github.com/ClickHouse/ClickHouse/issues/107561 + +DROP TABLE IF EXISTS t_dyn_dummy; +CREATE TABLE t_dyn_dummy (id UInt64, dummy String) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; + +INSERT INTO t_dyn_dummy SELECT number, 'value_' || number FROM numbers(1000); + +-- Add a Dynamic column and materialize it on the existing part via a partial mutation. The mutation +-- does not rewrite the unrelated `dummy` column, so its stream must be hardlinked into the new part +-- intact rather than skipped because of a placeholder/name collision. +ALTER TABLE t_dyn_dummy ADD COLUMN y Dynamic(max_types = 3) DEFAULT id::Int64 SETTINGS mutations_sync = 2; +ALTER TABLE t_dyn_dummy MATERIALIZE COLUMN y SETTINGS mutations_sync = 2; + +-- The unchanged `dummy` column must still be present and consistent. +SELECT count(), countIf(dummy = 'value_' || toString(id)) FROM t_dyn_dummy; +SELECT dynamicType(y) AS t, count() FROM t_dyn_dummy GROUP BY t ORDER BY t; +CHECK TABLE t_dyn_dummy SETTINGS check_query_single_value_result = 1; + +DROP TABLE t_dyn_dummy; diff --git a/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.reference b/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.reference new file mode 100644 index 000000000000..6a69f9852004 --- /dev/null +++ b/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.reference @@ -0,0 +1,8 @@ +columns_substreams.txt present before: 1 +columns_substreams.txt present after delete+attach: 0 +Data after mutation: +2000 2000 2000 +Int64 1000 +String 1000 +CHECK TABLE result: 1 +columns_substreams.txt regenerated with variant_discr: 1 diff --git a/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.sh b/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.sh new file mode 100755 index 000000000000..34fae7820982 --- /dev/null +++ b/tests/queries/0_stateless/04401_dynamic_mutation_old_part_no_substreams_file.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree, no-object-storage + +# Regression for a Wide part that has a Dynamic column but no columns_substreams.txt, as written by +# servers from before that file existed (Dynamic became production-ready in 25.3, the file was added +# to Wide parts in 25.8). For such a part the mutation stream-accounting cannot enumerate the +# data-dependent substreams of the Dynamic column (variant_discr, ...) without a deserialization +# state, so a partial mutation could leave one of those streams neither rewritten nor hardlinked. +# The mutation must instead rewrite the whole part. We simulate the old part by deleting +# columns_substreams.txt and reloading the table, then run a partial mutation that does NOT touch the +# Dynamic column and validate the resulting part with CHECK TABLE. +# See https://github.com/ClickHouse/ClickHouse/issues/107561 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t_dyn_old_part" + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE t_dyn_old_part (id UInt64, s UInt64, y Dynamic(max_types=3)) + ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +" + +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_dyn_old_part SELECT number, number, number::Int64 FROM numbers(1000)" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_dyn_old_part SELECT number, number, 's' || number FROM numbers(1000)" +${CLICKHOUSE_CLIENT} --query "OPTIMIZE TABLE t_dyn_old_part FINAL" + +DATA_PATH=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_dyn_old_part' AND active") + +echo -n "columns_substreams.txt present before: " +test -f "${DATA_PATH}columns_substreams.txt" && echo 1 || echo 0 + +# Detach the table, delete columns_substreams.txt to simulate a part written before the file existed, +# then reload it from disk. +${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_dyn_old_part" +rm -f "${DATA_PATH}columns_substreams.txt" +${CLICKHOUSE_CLIENT} --query "ATTACH TABLE t_dyn_old_part" + +echo -n "columns_substreams.txt present after delete+attach: " +test -f "${DATA_PATH}columns_substreams.txt" && echo 1 || echo 0 + +# Partial mutation that does NOT touch the Dynamic column. Because the source part has a Dynamic +# column with no recorded substreams, the whole part must be rewritten. +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_dyn_old_part UPDATE s = s + 1 WHERE id % 2 = 0 SETTINGS mutations_sync = 2" + +echo "Data after mutation:" +${CLICKHOUSE_CLIENT} --query "SELECT count(), countIf(y IS NOT NULL), countIf(s = id + (id % 2 = 0)) FROM t_dyn_old_part" +${CLICKHOUSE_CLIENT} --query "SELECT dynamicType(y) AS t, count() FROM t_dyn_old_part GROUP BY t ORDER BY t" + +echo -n "CHECK TABLE result: " +${CLICKHOUSE_CLIENT} --query "CHECK TABLE t_dyn_old_part SETTINGS check_query_single_value_result = 1" + +# The rewritten part is in the modern format again: columns_substreams.txt exists and records the +# Dynamic column's variant_discr substream. +NEW_DATA_PATH=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_dyn_old_part' AND active") +echo -n "columns_substreams.txt regenerated with variant_discr: " +grep -q "variant_discr" "${NEW_DATA_PATH}columns_substreams.txt" 2>/dev/null && echo 1 || echo 0 + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t_dyn_old_part" diff --git a/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.reference b/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.reference new file mode 100644 index 000000000000..ca066842d596 --- /dev/null +++ b/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.reference @@ -0,0 +1,6 @@ +columns_substreams.txt present before: 1 +columns_substreams.txt present after delete+attach: 0 +Data after mutation: +1000 1000 1000 +CHECK TABLE result: 1 +Stayed on partial path (no columns_substreams.txt regenerated): 1 diff --git a/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.sh b/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.sh new file mode 100755 index 000000000000..73b88a32981d --- /dev/null +++ b/tests/queries/0_stateless/04412_mutation_old_part_basic_map_partial.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree, no-object-storage + +# Companion to 04401: a Wide part with a plain Map column but no columns_substreams.txt (as written by +# servers from before that file existed) must NOT be force-rewritten by a partial mutation of an +# unrelated column. A basic Map serialization enumerates all of its physical streams without a +# deserialization state (unlike Dynamic/JSON, whose data-dependent substreams require the state), so a +# partial mutation can account for every stream and stay on the cheap partial path. We simulate the old +# part by deleting columns_substreams.txt and reloading the table, run a partial mutation that does NOT +# touch the Map column, validate with CHECK TABLE, and assert the rewritten part did NOT regenerate +# columns_substreams.txt (which would have meant a needless full rewrite of all the Map data). +# See https://github.com/ClickHouse/ClickHouse/issues/107561 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t_map_old_part" + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE t_map_old_part (id UInt64, s UInt64, m Map(String, String)) + ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +" + +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_map_old_part SELECT number, number, map('id', toString(number), 'k', toString(number * 2)) FROM numbers(1000)" +${CLICKHOUSE_CLIENT} --query "OPTIMIZE TABLE t_map_old_part FINAL" + +DATA_PATH=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_map_old_part' AND active") + +echo -n "columns_substreams.txt present before: " +test -f "${DATA_PATH}columns_substreams.txt" && echo 1 || echo 0 + +# Detach the table, delete columns_substreams.txt to simulate a part written before the file existed, +# then reload it from disk. +${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_map_old_part" +rm -f "${DATA_PATH}columns_substreams.txt" +${CLICKHOUSE_CLIENT} --query "ATTACH TABLE t_map_old_part" + +echo -n "columns_substreams.txt present after delete+attach: " +test -f "${DATA_PATH}columns_substreams.txt" && echo 1 || echo 0 + +# Partial mutation that does NOT touch the Map column. The Map's streams are fully enumerable without a +# deserialization state, so there is no correctness need to rewrite the whole part: it must stay on the +# partial path. +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_map_old_part UPDATE s = s + 1 WHERE id % 2 = 0 SETTINGS mutations_sync = 2" + +echo "Data after mutation:" +${CLICKHOUSE_CLIENT} --query "SELECT count(), countIf(s = id + (id % 2 = 0)), countIf(m['id'] = toString(id) AND m['k'] = toString(id * 2)) FROM t_map_old_part" + +echo -n "CHECK TABLE result: " +${CLICKHOUSE_CLIENT} --query "CHECK TABLE t_map_old_part SETTINGS check_query_single_value_result = 1" + +# A partial mutation of a part with no columns_substreams.txt does not write one (it is only filled from +# a non-empty source). So the absence of the file proves the part stayed on the partial path; if the Map +# column had wrongly forced a full rewrite, the part would have regained columns_substreams.txt. +NEW_DATA_PATH=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_map_old_part' AND active") +echo -n "Stayed on partial path (no columns_substreams.txt regenerated): " +test -f "${NEW_DATA_PATH}columns_substreams.txt" && echo 0 || echo 1 + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t_map_old_part" From c1f8b750e75b72acc4bf0a8983e4b4edf09f62f1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 27 Jul 2026 15:45:24 +0000 Subject: [PATCH 44/86] Backport #111417 to 26.6: Lower max_snapshot_commit_thread_pool_size to 16 --- src/Core/ServerSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index ceb9d58d28e5..2aa39554953e 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -160,7 +160,7 @@ namespace )", 0) \ DECLARE(UInt64, max_fetch_partition_thread_pool_size, 64, R"(The number of threads for ALTER TABLE FETCH PARTITION.)", 0) \ DECLARE(UInt64, max_active_parts_loading_thread_pool_size, 64, R"(The number of threads to load active set of data parts (Active ones) at startup.)", 0) \ - DECLARE(UInt64, max_snapshot_commit_thread_pool_size, 64, R"(The number of threads to commit snapshot.)", 0) \ + DECLARE(UInt64, max_snapshot_commit_thread_pool_size, 16, R"(The number of threads to commit snapshot.)", 0) \ DECLARE(UInt64, max_snapshot_commit_thread_pool_free_size, 0, R"(If the number of idle threads in the snapshot commit thread pool exceeds `max_snapshot_commit_thread_pool_free_size`, ClickHouse will release resources occupied by idling threads and decrease the pool size. Threads can be created again if necessary.)", 0) \ DECLARE(UInt64, max_outdated_parts_loading_thread_pool_size, 32, R"(The number of threads to load inactive set of data parts (Outdated ones) at startup.)", 0) \ DECLARE(UInt64, max_unexpected_parts_loading_thread_pool_size, 8, R"(The number of threads to load inactive set of data parts (Unexpected ones) at startup.)", 0) \ From db835f2d5aaaf77dc7c44a7dba19bbc86e56e7f3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 27 Jul 2026 19:34:29 +0000 Subject: [PATCH 45/86] Backport #112007 to 26.6: Fix AMBIGUOUS_COLUMN_NAME when a JOIN ON condition is repeated in WHERE --- src/Processors/QueryPlan/FilterStep.cpp | 5 +- ...s_repeated_on_condition_in_where.reference | 31 +++ ...e_nulls_repeated_on_condition_in_where.sql | 186 ++++++++++++++++++ 3 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.reference create mode 100644 tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index f147353ae14b..ae754e6e7286 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -196,7 +196,10 @@ struct ActionsAndName static ActionsAndName splitSingleAndFilter(ActionsDAG & dag, const ActionsDAG::Node * filter_node) { - auto split_result = dag.split({filter_node}, true); + /// avoid_duplicate_inputs: the split promotes the atom into an input of the remainder DAG, copying its + /// name verbatim. Duplicate names are legal inside a DAG but break the `Block` invariant, so a colliding + /// input has to be renamed (`split` adds an `ALIAS` so the original name stays resolvable). + auto split_result = dag.split({filter_node}, true, true); dag = std::move(split_result.second); const auto * split_filter_node = split_result.split_nodes_mapping[filter_node]; diff --git a/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.reference b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.reference new file mode 100644 index 000000000000..abf29619569f --- /dev/null +++ b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.reference @@ -0,0 +1,31 @@ +left join, right-side ON condition repeated in WHERE +1 100 +the colliding AND atom is renamed +1 +the AND chain is split at runtime +1 +right join, left-side ON condition repeated in WHERE +1 100 +full join +1 100 +inner join +1 100 +left join, non-Bool repeated condition +1 100 +left join, ARRAY JOIN above the join +1 100 1 +1 100 2 +left join, ORDER BY an expression above the join +1 100 +left join, String repeated condition +1 100 +left join, LowCardinality repeated condition +1 100 +left join, plain UInt8 repeated condition +1 100 +left join, already Nullable repeated condition (non-regression) +1 100 +left join, LowCardinality(Nullable) repeated condition (no promotion, non-regression) +1 100 +left join over ReplacingMergeTree with FINAL, prewhere disabled +1 100 diff --git a/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql new file mode 100644 index 000000000000..45569615715d --- /dev/null +++ b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql @@ -0,0 +1,186 @@ +-- The plan assertions below match analyzer-generated column identifiers (`__table2.`) in the legacy +-- `EXPLAIN` output, so both are pinned for the whole file. The old analyzer does not build the plan +-- shape that triggers this bug, so nothing is lost by pinning. +SET enable_analyzer = 1; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t2_string; +DROP TABLE IF EXISTS t2_lc; +DROP TABLE IF EXISTS t2_uint8; +DROP TABLE IF EXISTS t2_lc_nullable; +DROP TABLE IF EXISTS t2_nullable; +DROP TABLE IF EXISTS mt1; +DROP TABLE IF EXISTS mt2; + +CREATE TABLE t1 (id Int64, grp Int64) ENGINE = Memory; +CREATE TABLE t2 (id Int64, reviewer Int64, enabled Bool) ENGINE = Memory; +INSERT INTO t1 VALUES (1, 10), (2, 20); +INSERT INTO t2 VALUES (1, 100, true), (2, 200, false); + +SELECT 'left join, right-side ON condition repeated in WHERE'; +SELECT t1.id, t2.reviewer +FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.enabled = true +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +-- The `AND`-chain split has to rename the colliding boundary input. Assert the rename is in the +-- plan: a plan shape that never reaches the split would make every case below pass vacuously. +-- The extra settings are pinned (all randomized in CI) because the assertion needs both colliding +-- atoms to stay on one shared filter step below the join. +SELECT 'the colliding AND atom is renamed'; +SELECT count() > 0 FROM ( + EXPLAIN actions = 1, pretty = 0 + SELECT t1.id, t2.reviewer + FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.enabled = true + WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true + SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1, + query_plan_remove_unused_columns = 1, query_plan_merge_filter_into_join_condition = 0, + query_plan_optimize_join_order_randomize = 0, optimize_move_to_prewhere = 0, + query_plan_optimize_prewhere = 0 +) WHERE position(explain, 'AND column: equals(__table2.enabled, 1_Bool)_0') > 0; + +-- The rename above is printed by `FilterStep::describeActions`, which splits a clone of the DAG and +-- never builds a pipeline. Assert the `AND` chain is also split on the real execution path. One +-- `FilterTransform` is emitted per extracted atom plus one for the remainder, so a multiplicity of 3 +-- means two atoms were extracted: the collision this fixes needs that second, iterated split. +-- `max_threads` is pinned so the multiplicity counts split atoms, not parallel streams. +SELECT 'the AND chain is split at runtime'; +SELECT max(toUInt32OrZero(extract(explain, 'FilterTransform[^0-9]+([0-9]+)'))) >= 3 FROM ( + EXPLAIN PIPELINE + SELECT t1.id, t2.reviewer + FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.enabled = true + WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true + SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1, + query_plan_remove_unused_columns = 1, query_plan_merge_filter_into_join_condition = 0, + query_plan_optimize_join_order_randomize = 0, optimize_move_to_prewhere = 0, + query_plan_optimize_prewhere = 0, max_threads = 1 +); + +SELECT 'right join, left-side ON condition repeated in WHERE'; +SELECT t1.id, t2.reviewer +FROM t2 RIGHT JOIN t1 ON t1.id = t2.id AND t1.grp = 10 +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +SELECT 'full join'; +SELECT t1.id, t2.reviewer +FROM t1 FULL JOIN t2 ON t1.id = t2.id AND t2.enabled = true +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +SELECT 'inner join'; +SELECT t1.id, t2.reviewer +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t2.enabled = true +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +SELECT 'left join, non-Bool repeated condition'; +SELECT t1.id, t2.reviewer +FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.reviewer = 100 +WHERE t1.grp = 10 AND t2.enabled = true AND t2.reviewer = 100 +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +-- Two further affected shapes: with an `ARRAY JOIN` above the join, and with `ORDER BY` on an +-- expression. Both throw the same 352 on master and return the correct result with the fix. +SELECT 'left join, ARRAY JOIN above the join'; +SELECT t1.id, t2.reviewer, x +FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.enabled = true +ARRAY JOIN [1, 2] AS x +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t1.id, x +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +SELECT 'left join, ORDER BY an expression above the join'; +SELECT t1.id, t2.reviewer +FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND t2.enabled = true +WHERE t1.grp = 10 AND t2.reviewer = 100 AND t2.enabled = true +ORDER BY t2.reviewer + 1 +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +CREATE TABLE t2_string (id Int64, reviewer Int64, tag String) ENGINE = Memory; +INSERT INTO t2_string VALUES (1, 100, 'x'), (2, 200, 'y'); + +SELECT 'left join, String repeated condition'; +SELECT t1.id, t2_string.reviewer +FROM t1 LEFT JOIN t2_string ON t1.id = t2_string.id AND t2_string.tag = 'x' +WHERE t1.grp = 10 AND t2_string.reviewer = 100 AND t2_string.tag = 'x' +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +CREATE TABLE t2_lc (id Int64, reviewer Int64, tag LowCardinality(String)) ENGINE = Memory; +INSERT INTO t2_lc VALUES (1, 100, 'x'), (2, 200, 'y'); + +SELECT 'left join, LowCardinality repeated condition'; +SELECT t1.id, t2_lc.reviewer +FROM t1 LEFT JOIN t2_lc ON t1.id = t2_lc.id AND t2_lc.tag = 'x' +WHERE t1.grp = 10 AND t2_lc.reviewer = 100 AND t2_lc.tag = 'x' +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +-- `Bool` is a `UInt8` domain (a display name over `UInt8`), so it does not cover a plain `UInt8` +-- column: the colliding name carries the type spelling (`1_Bool` vs `1_UInt8`). Pin both. +CREATE TABLE t2_uint8 (id Int64, reviewer Int64, flag UInt8) ENGINE = Memory; +INSERT INTO t2_uint8 VALUES (1, 100, 1), (2, 200, 0); + +SELECT 'left join, plain UInt8 repeated condition'; +SELECT t1.id, t2_uint8.reviewer +FROM t1 LEFT JOIN t2_uint8 ON t1.id = t2_uint8.id AND t2_uint8.flag = 1 +WHERE t1.grp = 10 AND t2_uint8.reviewer = 100 AND t2_uint8.flag = 1 +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +CREATE TABLE t2_nullable (id Int64, reviewer Int64, enabled Nullable(Bool)) ENGINE = Memory; +INSERT INTO t2_nullable VALUES (1, 100, true), (2, 200, false), (3, 300, NULL); + +SELECT 'left join, already Nullable repeated condition (non-regression)'; +SELECT t1.id, t2_nullable.reviewer +FROM t1 LEFT JOIN t2_nullable ON t1.id = t2_nullable.id AND t2_nullable.enabled = true +WHERE t1.grp = 10 AND t2_nullable.reviewer = 100 AND t2_nullable.enabled = true +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +-- Also non-regression, for the same reason as the case above: `makeNullableOrLowCardinalityNullable` +-- early-returns on `isLowCardinalityNullable`, so promoting this column is a no-op and the `ON` and +-- `WHERE` copies of the predicate are computed on the identical type. Kept as the negative half of +-- the `LowCardinality` wrapper matrix. +CREATE TABLE t2_lc_nullable (id Int64, reviewer Int64, tag LowCardinality(Nullable(String))) ENGINE = Memory; +INSERT INTO t2_lc_nullable VALUES (1, 100, 'x'), (2, 200, NULL); + +SELECT 'left join, LowCardinality(Nullable) repeated condition (no promotion, non-regression)'; +SELECT t1.id, t2_lc_nullable.reviewer +FROM t1 LEFT JOIN t2_lc_nullable ON t1.id = t2_lc_nullable.id AND t2_lc_nullable.tag = 'x' +WHERE t1.grp = 10 AND t2_lc_nullable.reviewer = 100 AND t2_lc_nullable.tag = 'x' +ORDER BY t1.id +SETTINGS join_use_nulls = 1, query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +CREATE TABLE mt1 (id Int64, grp Int64) ENGINE = ReplacingMergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 1.0; +CREATE TABLE mt2 (id Int64, reviewer Int64, enabled Bool) ENGINE = ReplacingMergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 1.0; +INSERT INTO mt1 VALUES (1, 10), (2, 20); +INSERT INTO mt2 VALUES (1, 100, true), (2, 200, false); + +SELECT 'left join over ReplacingMergeTree with FINAL, prewhere disabled'; +SELECT mt1.id, mt2.reviewer +FROM mt1 LEFT JOIN mt2 ON mt1.id = mt2.id AND mt2.enabled = true +WHERE mt1.grp = 10 AND mt2.reviewer = 100 AND mt2.enabled = true +ORDER BY mt1.id +SETTINGS join_use_nulls = 1, final = 1, optimize_move_to_prewhere = 0, query_plan_optimize_prewhere = 0, + query_plan_merge_filters = 1, query_plan_convert_outer_join_to_inner_join = 1; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t2_string; +DROP TABLE t2_lc; +DROP TABLE t2_uint8; +DROP TABLE t2_lc_nullable; +DROP TABLE t2_nullable; +DROP TABLE mt1; +DROP TABLE mt2; From 02063a1404cd40049a19fff4bfd73b130aa0b49d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 11:15:33 +0000 Subject: [PATCH 46/86] Backport #111059 to 26.6: Properly fix primary key index analysis for reverse sorting keys --- src/Core/Field.h | 37 + .../QueryPlan/ReadFromMergeTree.cpp | 3 +- src/Storages/AlterCommands.cpp | 3 +- src/Storages/KeyDescription.cpp | 19 + src/Storages/KeyDescription.h | 12 + src/Storages/MergeTree/KeyCondition.cpp | 133 +++- src/Storages/MergeTree/KeyCondition.h | 23 +- src/Storages/MergeTree/KeyOrder.cpp | 36 + src/Storages/MergeTree/KeyOrder.h | 66 ++ .../MergeTree/MergeTreeDataSelectExecutor.cpp | 71 +- .../MergeTree/MergeTreeSequentialSource.cpp | 3 +- src/Storages/MergeTree/PartitionPruner.cpp | 3 +- .../ReplicatedMergeTreeTableMetadata.cpp | 5 +- .../MergeTree/registerStorageMergeTree.cpp | 3 +- ...04612_reverse_key_index_analysis.reference | 637 ++++++++++++++++++ .../04612_reverse_key_index_analysis.sql | 274 ++++++++ ..._reverse_key_replicated_metadata.reference | 9 + .../04613_reverse_key_replicated_metadata.sql | 35 + 18 files changed, 1299 insertions(+), 73 deletions(-) create mode 100644 src/Storages/MergeTree/KeyOrder.cpp create mode 100644 src/Storages/MergeTree/KeyOrder.h create mode 100644 tests/queries/0_stateless/04612_reverse_key_index_analysis.reference create mode 100644 tests/queries/0_stateless/04612_reverse_key_index_analysis.sql create mode 100644 tests/queries/0_stateless/04613_reverse_key_replicated_metadata.reference create mode 100644 tests/queries/0_stateless/04613_reverse_key_replicated_metadata.sql diff --git a/src/Core/Field.h b/src/Core/Field.h index c3e863ca0f5f..b78b5efb1707 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -315,6 +315,43 @@ class Field || which == Types::Decimal256; } + /// Whether values of the type are single scalar values with a plain value comparison, as opposed + /// to composite values (Array, Tuple, Map, Object — compared element-wise, where elements of + /// different types are ordered by type index rather than by value) and opaque values + /// (AggregateFunctionState, CustomType). + static bool isScalar(Types::Which which) + { + switch (which) + { + case Types::Null: + case Types::UInt64: + case Types::Int64: + case Types::Float64: + case Types::UInt128: + case Types::Int128: + case Types::String: + case Types::Decimal32: + case Types::Decimal64: + case Types::Decimal128: + case Types::Decimal256: + case Types::UInt256: + case Types::Int256: + case Types::UUID: + case Types::Bool: + case Types::IPv4: + case Types::IPv6: + return true; + case Types::Array: + case Types::Tuple: + case Types::Map: + case Types::Object: + case Types::CustomType: + case Types::AggregateFunctionState: + return false; + } + UNREACHABLE(); + } + Field() : Field(Null{}) {} /** Despite the presence of a template constructor, this constructor is still needed, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 8f2653729ab6..8602db328b77 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2030,8 +2030,7 @@ void ReadFromMergeTree::buildIndexes( ReadFromMergeTree::Indexes{KeyCondition{ filter_dag, query_context, - primary_key_column_names, - primary_key.expression, + primary_key, /* single_point_ = */ false, /* skip_analysis_ = */ !settings[Setting::use_primary_key]}}); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 0526d66d378c..bd15a140a863 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1386,7 +1386,8 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context metadata_copy.sorting_key.recalculateWithNewAST(metadata_copy.sorting_key.definition_ast, metadata_copy.columns, metadata_copy.virtuals, context); if (metadata_copy.primary_key.definition_ast != nullptr) { - metadata_copy.primary_key.recalculateWithNewAST(metadata_copy.primary_key.definition_ast, metadata_copy.columns, metadata_copy.virtuals, context); + metadata_copy.primary_key = KeyDescription::getPrimaryKeyFromAST( + metadata_copy.primary_key.definition_ast, metadata_copy.sorting_key, metadata_copy.columns, metadata_copy.virtuals, context); } else { diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 3a8218611e30..39e0eadb43db 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -224,6 +224,25 @@ ASTPtr KeyDescription::getOriginalExpressionList() const return expr_list; } +KeyDescription KeyDescription::getPrimaryKeyFromAST( + const ASTPtr & definition_ast, + const KeyDescription & sorting_key, + const ColumnsDescription & columns, + const VirtualColumnsDescription & virtuals, + const ContextPtr & context) +{ + KeyDescription result = getKeyFromAST(definition_ast, columns, virtuals, context); + + /// The primary key is a prefix of the sorting key (validated in MergeTreeData::checkProperties), + /// so its per-column directions are the corresponding prefix of the sorting key's. + if (!sorting_key.reverse_flags.empty()) + result.reverse_flags.assign( + sorting_key.reverse_flags.begin(), + sorting_key.reverse_flags.begin() + std::min(result.column_names.size(), sorting_key.reverse_flags.size())); + + return result; +} + KeyDescription KeyDescription::buildEmptyKey() { KeyDescription result; diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index 15534b22416a..da1cee4959ca 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -61,6 +61,18 @@ struct KeyDescription const ContextPtr & context, const NamesAndTypesList & additional_columns = {}); + /// Build a primary key description from an explicit PRIMARY KEY. The PRIMARY KEY names a + /// prefix of the sorting key but cannot express per-column directions (`DESC`); the physical + /// order of the named columns is defined by ORDER BY, so the directions are inherited from the + /// sorting key. An implicitly defined primary key is built from the ORDER BY AST itself and + /// carries the directions naturally. + static KeyDescription getPrimaryKeyFromAST( + const ASTPtr & definition_ast, + const KeyDescription & sorting_key, + const ColumnsDescription & columns, + const VirtualColumnsDescription & virtuals, + const ContextPtr & context); + /// Build an empty key description. It's different from the default constructor with some /// additional initializations. static KeyDescription buildEmptyKey(); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index d2c7e41ba87d..07ed28a93371 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -1663,6 +1664,17 @@ KeyCondition::KeyCondition( relaxed = true; } +KeyCondition::KeyCondition( + const ActionsDAGWithInversionPushDown & filter_dag, + ContextPtr context, + const KeyDescription & key_description, + bool single_point_, + bool skip_analysis_) + : KeyCondition(filter_dag, context, key_description.column_names, key_description.expression, single_point_, skip_analysis_) +{ + key_order = KeyOrder(key_description.reverse_flags); +} + KeyCondition::KeyCondition( ThisIsPrivate, ColumnIndices key_columns_, size_t num_key_columns_, bool single_point_, bool date_time_overflow_behavior_ignore_, bool relaxed_) @@ -4514,6 +4526,35 @@ KeyCondition::Description KeyCondition::getDescription() const * This is important because it is easy for us to check the feasibility of the condition over the hyperrectangle, * and therefore, feasibility of condition on the range of tuples will be checked by feasibility of condition * over at least one hyperrectangle from which this range consists. + * + * A key column may be sorted in reverse (`ORDER BY (x, y DESC)`, see KeyOrder). The boundary + * tuples are still the physical values at the marks, and the decomposition produces the same three + * groups of rows; what changes is the y-interval covering the first and the last group. To see how, + * revisit why the ascending decomposition above is correct: + * + * - The rows with x == x1 lie between the left boundary (x1, y1) and the end of the x1 group. Within + * the group they are ordered by y, so their y values start at y1 and move in y's sort direction: + * upward for ascending y, giving [x1] × [y1 .. +inf), but downward for descending y, giving + * [x1] × (-inf .. y1]. + * - Symmetrically, the rows with x == x2 lie before the right boundary (x2, y2), so their y values + * approach y2 from the opposite side: [x2] × (-inf .. y2] for ascending y, but [x2] × [y2 .. +inf) + * for descending y. + * - The middle rectangle is unchanged: it covers rows whose x lies strictly between x1 and x2 with + * any y, and whether a value lies strictly between two others does not depend on sort direction. + * + * So for descending y, the same range [ x1 y1 .. x2 y2 ] given x1 != x2 is the union of: + * [x1] × (-inf .. y1] + * (x1 .. x2) × (-inf .. +inf) + * [x2] × [y2 .. +inf) + * + * The same rule covers a descending column in any position. At the first column where the boundaries + * differ, the two boundary values delimit that column's interval, and on a descending column the left + * boundary holds the larger value, so the operands swap: with x descending, the middle rectangle is + * (x2 .. x1); and when the boundaries agree on x and differ first at a descending last column y, the + * interval is the closed [y2 .. y1]. This is why every Range built from boundary values below picks + * its operands and its bounded side through KeyOrder (see values_between, values_after_left_boundary + * and values_before_right_boundary): a boundary value stays attached to its physical boundary, and + * the column's direction decides which side of the value interval it bounds. */ /** For the range between tuples, determined by left_keys, left_bounded, right_keys, right_bounded, @@ -4529,10 +4570,31 @@ static BoolMask forAnyHyperrectangle( bool right_bounded, Hyperrectangle & hyperrectangle, /// This argument is modified in-place for the callback const DataTypes & data_types, + const KeyOrder & key_order, size_t prefix_size, BoolMask initial_mask, F && callback) { + auto values_between = [&](size_t col, bool included) -> Range + { + return key_order.isReversed(col) ? Range(right_keys[col], included, left_keys[col], included) + : Range(left_keys[col], included, right_keys[col], included); + }; + + auto values_after_left_boundary = [&](size_t col, bool included) -> Range + { + const bool with_null = isNullableOrLowCardinalityNullable(data_types[col]); + return key_order.isReversed(col) ? Range::createRightBounded(left_keys[col], included, with_null) + : Range::createLeftBounded(left_keys[col], included, with_null); + }; + + auto values_before_right_boundary = [&](size_t col, bool included) -> Range + { + const bool with_null = isNullableOrLowCardinalityNullable(data_types[col]); + return key_order.isReversed(col) ? Range::createLeftBounded(right_keys[col], included, with_null) + : Range::createRightBounded(right_keys[col], included, with_null); + }; + if (!left_bounded && !right_bounded) return callback(hyperrectangle); @@ -4558,13 +4620,11 @@ static BoolMask forAnyHyperrectangle( if (prefix_size + 1 == key_size) { if (left_bounded && right_bounded) - hyperrectangle[prefix_size] = Range(left_keys[prefix_size], true, right_keys[prefix_size], true); + hyperrectangle[prefix_size] = values_between(prefix_size, true); else if (left_bounded) - hyperrectangle[prefix_size] - = Range::createLeftBounded(left_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size])); + hyperrectangle[prefix_size] = values_after_left_boundary(prefix_size, true); else if (right_bounded) - hyperrectangle[prefix_size] - = Range::createRightBounded(right_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size])); + hyperrectangle[prefix_size] = values_before_right_boundary(prefix_size, true); return callback(hyperrectangle); } @@ -4572,13 +4632,11 @@ static BoolMask forAnyHyperrectangle( /// (x1 .. x2) × (-inf .. +inf) if (left_bounded && right_bounded) - hyperrectangle[prefix_size] = Range(left_keys[prefix_size], false, right_keys[prefix_size], false); + hyperrectangle[prefix_size] = values_between(prefix_size, false); else if (left_bounded) - hyperrectangle[prefix_size] - = Range::createLeftBounded(left_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size])); + hyperrectangle[prefix_size] = values_after_left_boundary(prefix_size, false); else if (right_bounded) - hyperrectangle[prefix_size] - = Range::createRightBounded(right_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size])); + hyperrectangle[prefix_size] = values_before_right_boundary(prefix_size, false); for (size_t i = prefix_size + 1; i < key_size; ++i) { @@ -4603,7 +4661,8 @@ static BoolMask forAnyHyperrectangle( result = BoolMask::combine( result, forAnyHyperrectangle( - key_size, left_keys, right_keys, true, false, hyperrectangle, data_types, prefix_size + 1, initial_mask, callback)); + key_size, left_keys, right_keys, true, false, hyperrectangle, data_types, key_order, + prefix_size + 1, initial_mask, callback)); if (result.isComplete()) return result; @@ -4617,7 +4676,8 @@ static BoolMask forAnyHyperrectangle( result = BoolMask::combine( result, forAnyHyperrectangle( - key_size, left_keys, right_keys, false, true, hyperrectangle, data_types, prefix_size + 1, initial_mask, callback)); + key_size, left_keys, right_keys, false, true, hyperrectangle, data_types, key_order, + prefix_size + 1, initial_mask, callback)); } return result; @@ -4656,12 +4716,35 @@ static BoolMask forAnySparseHyperrectangle( bool right_bounded, Hyperrectangle & sparse_hyperrectangle, const DataTypes & sparse_data_types, + const KeyOrder & key_order, size_t prefix_size, BoolMask initial_mask, F && callback) { const size_t key_size = equal_boundaries_mask.size(); + auto values_between = [&](size_t key_index, size_t sparse_pos, bool included) -> Range + { + return key_order.isReversed(key_index) ? Range(sparse_right_keys[sparse_pos], included, sparse_left_keys[sparse_pos], included) + : Range(sparse_left_keys[sparse_pos], included, sparse_right_keys[sparse_pos], included); + }; + + auto values_after_left_boundary = [&](size_t key_index, size_t sparse_pos, bool included) -> Range + { + const bool with_null = isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos]); + return key_order.isReversed(key_index) + ? Range::createRightBounded(sparse_left_keys[sparse_pos], included, with_null) + : Range::createLeftBounded(sparse_left_keys[sparse_pos], included, with_null); + }; + + auto values_before_right_boundary = [&](size_t key_index, size_t sparse_pos, bool included) -> Range + { + const bool with_null = isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos]); + return key_order.isReversed(key_index) + ? Range::createLeftBounded(sparse_right_keys[sparse_pos], included, with_null) + : Range::createRightBounded(sparse_right_keys[sparse_pos], included, with_null); + }; + #ifndef NDEBUG const size_t sparse_keys_size = sparse_key_indices.size(); @@ -4709,17 +4792,15 @@ static BoolMask forAnySparseHyperrectangle( const size_t sparse_pos = static_cast(key_col_to_sparse_pos[prefix_size]); if (left_bounded && right_bounded) { - sparse_hyperrectangle[sparse_pos] = Range(sparse_left_keys[sparse_pos], true, sparse_right_keys[sparse_pos], true); + sparse_hyperrectangle[sparse_pos] = values_between(prefix_size, sparse_pos, true); } else if (left_bounded) { - sparse_hyperrectangle[sparse_pos] = Range::createLeftBounded( - sparse_left_keys[sparse_pos], true, isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos])); + sparse_hyperrectangle[sparse_pos] = values_after_left_boundary(prefix_size, sparse_pos, true); } else if (right_bounded) { - sparse_hyperrectangle[sparse_pos] = Range::createRightBounded( - sparse_right_keys[sparse_pos], true, isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos])); + sparse_hyperrectangle[sparse_pos] = values_before_right_boundary(prefix_size, sparse_pos, true); } } @@ -4733,17 +4814,15 @@ static BoolMask forAnySparseHyperrectangle( const size_t sparse_pos = static_cast(key_col_to_sparse_pos[prefix_size]); if (left_bounded && right_bounded) { - sparse_hyperrectangle[sparse_pos] = Range(sparse_left_keys[sparse_pos], false, sparse_right_keys[sparse_pos], false); + sparse_hyperrectangle[sparse_pos] = values_between(prefix_size, sparse_pos, false); } else if (left_bounded) { - sparse_hyperrectangle[sparse_pos] = Range::createLeftBounded( - sparse_left_keys[sparse_pos], false, isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos])); + sparse_hyperrectangle[sparse_pos] = values_after_left_boundary(prefix_size, sparse_pos, false); } else if (right_bounded) { - sparse_hyperrectangle[sparse_pos] = Range::createRightBounded( - sparse_right_keys[sparse_pos], false, isNullableOrLowCardinalityNullable(sparse_data_types[sparse_pos])); + sparse_hyperrectangle[sparse_pos] = values_before_right_boundary(prefix_size, sparse_pos, false); } } @@ -4787,6 +4866,7 @@ static BoolMask forAnySparseHyperrectangle( false, sparse_hyperrectangle, sparse_data_types, + key_order, prefix_size + 1, initial_mask, callback)); @@ -4817,6 +4897,7 @@ static BoolMask forAnySparseHyperrectangle( true, sparse_hyperrectangle, sparse_data_types, + key_order, prefix_size + 1, initial_mask, callback)); @@ -4832,6 +4913,8 @@ BoolMask KeyCondition::checkInRange( const DataTypes & data_types, BoolMask initial_mask) const { + chassert(key_order.compareTuples(left_keys, right_keys, used_key_size) <= 0); + Hyperrectangle key_ranges; key_ranges.reserve(used_key_size); @@ -4843,7 +4926,7 @@ BoolMask KeyCondition::checkInRange( key_ranges.push_back(Range::createWholeUniverseWithoutNull()); } - return forAnyHyperrectangle(used_key_size, left_keys, right_keys, true, true, key_ranges, data_types, 0, initial_mask, + return forAnyHyperrectangle(used_key_size, left_keys, right_keys, true, true, key_ranges, data_types, key_order, 0, initial_mask, [&] (const Hyperrectangle & key_ranges_hyperrectangle) { return checkInHyperrectangle(key_ranges_hyperrectangle, data_types); @@ -4903,6 +4986,7 @@ BoolMask KeyCondition::checkInRange( /*right_bounded*/ true, sparse_key_ranges, sparse_data_types, + key_order, /*prefix_size*/ 0, initial_mask, [&](const Hyperrectangle & key_ranges_hyperrectangle) @@ -6843,6 +6927,9 @@ void KeyCondition::extractSingleColumnConditions(std::vector(ThisIsPrivate(), std::move(one_key_column), num_key_columns, single_point, date_time_overflow_behavior_ignore, relaxed); + + /// The split conditions keep the original key column positions, so the key order carries over. + condition->key_order = key_order; add_rpn_ranges(*condition, *this, ranges); out_column_conditions.emplace_back(i, std::move(condition)); } diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index a349230dc25c..734d6757b9c0 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -12,6 +12,7 @@ #include #include +#include #include @@ -25,6 +26,7 @@ class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; struct ActionDAGNodes; class MergeTreeSetIndex; +struct KeyDescription; /// Canonize the predicate @@ -67,7 +69,10 @@ class KeyCondition struct ThisIsPrivate {}; public: - /// Construct key condition from ActionsDAG nodes + /// Construct key condition from ActionsDAG nodes. + /// This overload takes the key column names and expression without any direction information, + /// so the condition treats the key as ascending in every column. Use it only for keys that + /// cannot be reverse-sorted (e.g. skip index expressions, virtual row-offset columns). KeyCondition( const ActionsDAGWithInversionPushDown & filter_dag, ContextPtr context, @@ -76,6 +81,17 @@ class KeyCondition bool single_point_ = false, bool skip_analysis_ = false); /// Toggled by `use_primary_key`, `use_partition_key` setting. Useful for testing. + /// Same as above, but takes the key's KeyDescription. The condition honors the key's per-column + /// sort directions (reverse flags; an empty vector means all-ascending, e.g. a partition key). + /// Any condition over a key that can be reverse-sorted (a MergeTree primary key) must be + /// constructed this way, otherwise a reverse key would be analyzed as ascending. + KeyCondition( + const ActionsDAGWithInversionPushDown & filter_dag, + ContextPtr context, + const KeyDescription & key_description, + bool single_point_ = false, + bool skip_analysis_ = false); + struct BloomFilterData { using HashesForColumns = std::vector>; @@ -158,6 +174,8 @@ class KeyCondition const std::vector & equal_boundaries_mask, BoolMask initial_mask) const; + const KeyOrder & getKeyOrder() const { return key_order; } + /// Same as checkInRange, but calculate only may_be_true component of a result. /// This is more efficient than checkInRange(...).can_be_true. bool mayBeTrueInRange( @@ -594,6 +612,9 @@ class KeyCondition /// Used to check toDateTime monotonicity. bool date_time_overflow_behavior_ignore; + /// Holds whether the key columns are sorted in reverse (ORDER BY ... DESC) or not. + KeyOrder key_order; + /// If true, this key condition is relaxed. When a key condition is relaxed, it /// is considered weakened. This is because keys may not always align perfectly /// with the condition specified in the query, and the aim is to enhance the diff --git a/src/Storages/MergeTree/KeyOrder.cpp b/src/Storages/MergeTree/KeyOrder.cpp new file mode 100644 index 000000000000..231a0ddfa989 --- /dev/null +++ b/src/Storages/MergeTree/KeyOrder.cpp @@ -0,0 +1,36 @@ +#include + +#include + +namespace DB +{ + +int KeyOrder::compareTuples(const FieldRef * left, const FieldRef * right, size_t size) const +{ + for (size_t i = 0; i < size; ++i) + { + /// Field comparison of non-scalar values can diverge from the columnar sort order (composite + /// values compare NULL or NaN elements by type index), so nothing can be concluded about + /// such coordinates. + if (!Field::isScalar(left[i].getType()) || !Field::isScalar(right[i].getType())) + return 0; + + /// The (physicalStartExtreme, physicalEndExtreme) pair marks a column whose value is unknown + /// at both boundaries; nothing can be concluded about the order of tuples that differ first + /// at such a column. + if (left[i] == physicalStartExtreme(i) && right[i] == physicalEndExtreme(i)) + return 0; + + /// Equality must follow the total sort order, not value-space semantics: accurateLess places + /// NaN above every number (the writer sorts with the same convention), so two NaNs are equal + /// here, while accurateEquals would report them as unequal. + if (accurateLess(left[i], right[i])) + return isReversed(i) ? 1 : -1; + + if (accurateLess(right[i], left[i])) + return isReversed(i) ? -1 : 1; + } + return 0; +} + +} diff --git a/src/Storages/MergeTree/KeyOrder.h b/src/Storages/MergeTree/KeyOrder.h new file mode 100644 index 000000000000..4712fa873c69 --- /dev/null +++ b/src/Storages/MergeTree/KeyOrder.h @@ -0,0 +1,66 @@ +#pragma once + +#include + +#include +#include + +namespace DB +{ + +/// Which key columns are sorted in reverse (`ORDER BY (g, r DESC)`). +class KeyOrder +{ +public: + /// All columns ascending. + KeyOrder() = default; + + explicit KeyOrder(std::vector reverse_flags_) + : reverse_flags(std::move(reverse_flags_)) + , has_any_reversed(std::find(reverse_flags.begin(), reverse_flags.end(), true) != reverse_flags.end()) + { + } + + bool hasAnyReversed() const { return has_any_reversed; } + + /// Positions beyond the stored flags are ascending; an empty vector means the whole key is ascending. + bool isReversed(size_t column) const { return column < reverse_flags.size() && reverse_flags[column]; } + + /// Value-space stand-in for the value at the unknown physical end of a part (the side past the + /// last mark). Values ascend toward +inf on an ascending column and descend toward -inf on a + /// descending one. + FieldRef physicalEndExtreme(size_t column) const + { + return isReversed(column) ? FieldRef(NEGATIVE_INFINITY) : FieldRef(POSITIVE_INFINITY); + } + + /// Mirror of physicalEndExtreme: the stand-in for an unknown value on the physical-start side. + FieldRef physicalStartExtreme(size_t column) const + { + return isReversed(column) ? FieldRef(POSITIVE_INFINITY) : FieldRef(NEGATIVE_INFINITY); + } + + /// Whether NULLs are stored physically last on this column: they are on an ascending column and + /// physically first on a descending one. + /// mark range starting with a NULL known to be NULL up to the end of the part. + bool nullsAreStoredLast(size_t column) const { return !isReversed(column); } + + /// Debug helper for the `checkInRange` contract: boundary tuples must be points of storage order, + /// left before right. + int compareTuples(const FieldRef * left, const FieldRef * right, size_t size) const; + + bool matchesPrefix(const std::vector & flags, size_t num_columns) const + { + for (size_t i = 0; i < num_columns; ++i) + if (isReversed(i) != (i < flags.size() && flags[i])) + return false; + return true; + } + +private: + /// Empty means all columns ascending. + std::vector reverse_flags; + bool has_any_reversed = false; +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index a66972cb2c59..11e24e70d325 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1668,9 +1668,11 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( exact_ranges = nullptr; const auto & primary_key = metadata_snapshot->getPrimaryKey(); - const auto & sorting_key = metadata_snapshot->getSortingKey(); auto index_columns = std::make_shared(); - std::vector reverse_flags; + + /// Which key columns are reverse-sorted. + const KeyOrder & key_order = key_condition.getKeyOrder(); + chassert(key_order.matchesPrefix(metadata_snapshot->getSortingKey().reverse_flags, primary_key.column_names.size())); const auto index = part->getIndex(); const bool use_sparse_pk_representation @@ -1709,7 +1711,6 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( chassert(i < index->size()); chassert(index->at(i)); index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); - reverse_flags.push_back(!sorting_key.reverse_flags.empty() && sorting_key.reverse_flags[i]); } //// Get PK columns potentially used in `KeyCondition` Filter @@ -1751,16 +1752,10 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( for (size_t i = 0; i < num_key_columns; ++i) { if (i < index->size()) - { index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); - reverse_flags.push_back(!sorting_key.reverse_flags.empty() && sorting_key.reverse_flags[i]); - } else - { /// The column of the primary key was not loaded in memory - we'll skip it. index_columns->emplace_back(); - reverse_flags.push_back(false); - } key_types.emplace_back(primary_key.data_types[i]); } @@ -1817,27 +1812,24 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( if (range.end == marks_count) { - /// Last mark: the right boundary of every key column is +inf. The left and right - /// boundaries are equal only when the left boundary value is also +inf, i.e. when the - /// value at range.begin is NULL (create_field_ref maps NULL to +inf for NULL_LAST - /// ordering). A non-nullable column is never NULL, so its boundaries are never equal. + /// Last mark: the boundary at the unknown physical end of the part is the column's + /// directional extreme. The boundaries are known to be equal only when the range + /// starts with a NULL and NULLs are stored physically last on the column — then the + /// column is NULL up to the end of the part (NULL's value-space stand-in, +inf, + /// equals the extreme). A non-nullable column is never NULL, so its boundaries are + /// never known to be equal. for (size_t i = 0; i < num_key_columns; ++i) { const auto & col = (*index_columns)[i].column; chassert(col); - equal_boundaries_mask[i] = col->isNullAt(range.begin); + equal_boundaries_mask[i] = col->isNullAt(range.begin) && key_order.nullsAreStoredLast(i); } for (size_t sparse_pos = 0; sparse_pos < sparse_keys_size; ++sparse_pos) { const size_t key_col = used_key_indices[sparse_pos]; - - auto & left = reverse_flags[key_col] ? sparse_key_right[sparse_pos] : sparse_key_left[sparse_pos]; - auto & right = reverse_flags[key_col] ? sparse_key_left[sparse_pos] : sparse_key_right[sparse_pos]; - - create_field_ref(range.begin, key_col, left); - - right = POSITIVE_INFINITY; + create_field_ref(range.begin, key_col, sparse_key_left[sparse_pos]); + sparse_key_right[sparse_pos] = key_order.physicalEndExtreme(key_col); } } else @@ -1856,12 +1848,8 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( for (size_t sparse_pos = 0; sparse_pos < sparse_keys_size; ++sparse_pos) { const size_t key_col = used_key_indices[sparse_pos]; - - auto & left = reverse_flags[key_col] ? sparse_key_right[sparse_pos] : sparse_key_left[sparse_pos]; - auto & right = reverse_flags[key_col] ? sparse_key_left[sparse_pos] : sparse_key_right[sparse_pos]; - - create_field_ref(range.begin, key_col, left); - create_field_ref(range.end, key_col, right); + create_field_ref(range.begin, key_col, sparse_key_left[sparse_pos]); + create_field_ref(range.end, key_col, sparse_key_right[sparse_pos]); } } @@ -1878,32 +1866,35 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { for (size_t i = 0; i < used_key_size; ++i) { - auto & left = reverse_flags[i] ? index_right[i] : index_left[i]; - auto & right = reverse_flags[i] ? index_left[i] : index_right[i]; if ((*index_columns)[i].column) - create_field_ref(range.begin, i, left); + { + create_field_ref(range.begin, i, index_left[i]); + /// The value at the unknown physical end of the part is the directional extreme. + index_right[i] = key_order.physicalEndExtreme(i); + } else - left = NEGATIVE_INFINITY; - - right = POSITIVE_INFINITY; + { + /// Key column not loaded in the in-memory index: unknown at both boundaries. + /// The (start, end) extreme pair makes the decomposition take the whole + /// universe at this column, a safe over-approximation. + index_left[i] = key_order.physicalStartExtreme(i); + index_right[i] = key_order.physicalEndExtreme(i); + } } } else { for (size_t i = 0; i < used_key_size; ++i) { - auto & left = reverse_flags[i] ? index_right[i] : index_left[i]; - auto & right = reverse_flags[i] ? index_left[i] : index_right[i]; if ((*index_columns)[i].column) { - create_field_ref(range.begin, i, left); - create_field_ref(range.end, i, right); + create_field_ref(range.begin, i, index_left[i]); + create_field_ref(range.end, i, index_right[i]); } else { - /// If the PK column was not loaded in memory - exclude it from the analysis. - left = NEGATIVE_INFINITY; - right = POSITIVE_INFINITY; + index_left[i] = key_order.physicalStartExtreme(i); + index_right[i] = key_order.physicalEndExtreme(i); } } } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 3b02eb002daa..6ebd84807485 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -436,9 +436,8 @@ class ReadFromPart final : public ISourceStep if (filter && metadata_snapshot->hasPrimaryKey()) { const auto & primary_key = storage_snapshot->metadata->getPrimaryKey(); - const Names & primary_key_column_names = primary_key.column_names; ActionsDAGWithInversionPushDown filter_dag(filter->getOutputs().front(), context, /* boolean_context */ true); - KeyCondition key_condition(filter_dag, context, primary_key_column_names, primary_key.expression); + KeyCondition key_condition(filter_dag, context, primary_key); LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); if (!key_condition.alwaysFalse()) diff --git a/src/Storages/MergeTree/PartitionPruner.cpp b/src/Storages/MergeTree/PartitionPruner.cpp index 730a2f0b3bca..8eeffaac630e 100644 --- a/src/Storages/MergeTree/PartitionPruner.cpp +++ b/src/Storages/MergeTree/PartitionPruner.cpp @@ -14,8 +14,7 @@ PartitionPruner::PartitionPruner( , partition_condition( filter_dag, context, - partition_key.column_names, - partition_key.expression, + partition_key, true /* single_point */, skip_analysis) , useless((strict && partition_condition.isRelaxed()) || partition_condition.alwaysUnknownOrTrue()) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index d0a22630d044..1069091962e1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -635,7 +635,10 @@ StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(c /// Primary key is special, it exists even if not defined if (new_metadata.primary_key.definition_ast != nullptr) { - new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, virtuals, context); + /// An explicitly defined primary key cannot express per-column directions (`DESC`), so it + /// inherits them from the sorting key, which is already recalculated above. + new_metadata.primary_key = KeyDescription::getPrimaryKeyFromAST( + new_metadata.primary_key.definition_ast, new_metadata.sorting_key, new_metadata.columns, virtuals, context); } else { diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index d60293269ec7..753944860beb 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -718,7 +718,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// If primary key explicitly defined, than get it from AST if (args.storage_def->primary_key) { - metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, metadata.virtuals, context); + metadata.primary_key = KeyDescription::getPrimaryKeyFromAST( + args.storage_def->primary_key->ptr(), metadata.sorting_key, metadata.columns, metadata.virtuals, context); } else /// Otherwise we don't have explicit primary key and copy it from order by. { diff --git a/tests/queries/0_stateless/04612_reverse_key_index_analysis.reference b/tests/queries/0_stateless/04612_reverse_key_index_analysis.reference new file mode 100644 index 000000000000..378e06fc6eff --- /dev/null +++ b/tests/queries/0_stateless/04612_reverse_key_index_analysis.reference @@ -0,0 +1,637 @@ +-- { echo } + +SELECT 'equality on both key columns, DESC Enum second'; +equality on both key columns, DESC Enum second +DROP TABLE IF EXISTS t_enum_rev; +CREATE TABLE t_enum_rev (g String, r Enum8('poor' = 1, 'ok' = 2, 'great' = 3)) +ENGINE = MergeTree ORDER BY (g, r DESC); +INSERT INTO t_enum_rev VALUES ('manual', 'ok'), ('manual', 'poor'), ('novel', 'great'), ('novel', 'great'); +SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 1; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 0; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +DROP TABLE t_enum_rev; +SELECT 'bounds on the DESC column in each direction'; +bounds on the DESC column in each direction +DROP TABLE IF EXISTS t_str_int; +CREATE TABLE t_str_int (g String, r Int8) +ENGINE = MergeTree ORDER BY (g, r DESC); +INSERT INTO t_str_int VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g = 'novel' AND r >= 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r >= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, +Inf)), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g = 'novel' AND r <= 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r <= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in (-Inf, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g >= 'novel' AND r = 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g >= 'novel' AND r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', +Inf))) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: generic exclusion search +SELECT count() FROM t_str_int WHERE g = 'manual' AND r > 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'manual' AND r > 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [2, +Inf)), (g in [\'manual\', \'manual\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g = 'manual' AND r = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'manual' AND r = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [1, 1]), (g in [\'manual\', \'manual\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 2; +0 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [2, 2]), (g in [\'novel\', \'novel\'])) +Parts: 0/1 +Granules: 0/1 +Search Algorithm: binary search +DROP TABLE t_str_int; +SELECT 'DESC column first'; +DESC column first +DROP TABLE IF EXISTS t_rev_first; +CREATE TABLE t_rev_first (r Int8, g String) +ENGINE = MergeTree ORDER BY (r DESC, g); +INSERT INTO t_rev_first VALUES (2, 'manual'), (1, 'manual'), (3, 'novel'), (3, 'zzz'); +SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((g in [\'novel\', \'novel\']), (r in [3, 3])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((g in [\'novel\', \'novel\']), (r in [3, 3])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_rev_first WHERE r = 1 AND g = 'manual'; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 1 AND g = 'manual') WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((g in [\'manual\', \'manual\']), (r in [1, 1])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_rev_first WHERE r >= 2 AND g >= 'a'; +3 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r >= 2 AND g >= 'a') WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((g in [\'a\', +Inf)), (r in [2, +Inf))) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: generic exclusion search +SELECT count() FROM t_rev_first WHERE r = 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: (r in [3, 3]) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +DROP TABLE t_rev_first; +SELECT 'two DESC columns'; +two DESC columns +DROP TABLE IF EXISTS t_both_rev; +CREATE TABLE t_both_rev (a UInt8, b UInt8) +ENGINE = MergeTree ORDER BY (a DESC, b DESC) SETTINGS index_granularity = 2; +INSERT INTO t_both_rev SELECT 1 + intDiv(number, 3), 1 + number % 3 FROM numbers(9); +SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((b in [2, 2]), (a in [2, 2])) +Parts: 1/1 +Granules: 2/5 +Search Algorithm: binary search +SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((b in [2, 2]), (a in [2, 2])) +Parts: 1/1 +Granules: 2/5 +Search Algorithm: binary search +SELECT count() FROM t_both_rev WHERE a = 2 AND b >= 2; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b >= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((b in [2, +Inf)), (a in [2, 2])) +Parts: 1/1 +Granules: 2/5 +Search Algorithm: binary search +SELECT count() FROM t_both_rev WHERE a >= 2 AND b <= 2; +4 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a >= 2 AND b <= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((b in (-Inf, 2]), (a in [2, +Inf))) +Parts: 1/1 +Granules: 3/5 +Search Algorithm: generic exclusion search +DROP TABLE t_both_rev; +SELECT 'NULLs sit physically first on a DESC column'; +NULLs sit physically first on a DESC column +DROP TABLE IF EXISTS t_null; +CREATE TABLE t_null (g String, r Nullable(Int8)) +ENGINE = MergeTree ORDER BY (g, r DESC) +SETTINGS allow_nullable_key = 1, index_granularity = 2; +INSERT INTO t_null VALUES ('a', NULL), ('a', 5), ('a', 3), ('b', NULL), ('b', NULL), ('b', 7); +SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [7, 7]), (g in [\'b\', \'b\'])) +Parts: 1/1 +Granules: 1/3 +Search Algorithm: binary search +SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [7, 7]), (g in [\'b\', \'b\'])) +Parts: 1/1 +Granules: 1/3 +Search Algorithm: binary search +SELECT count() FROM t_null WHERE g = 'a' AND r >= 4; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'a' AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [4, +Inf)), (g in [\'a\', \'a\'])) +Parts: 1/1 +Granules: 1/3 +Search Algorithm: binary search +SELECT count() FROM t_null WHERE g = 'b' AND r IS NULL; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r IS NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r isNull), (g in [\'b\', \'b\'])) +Parts: 1/1 +Granules: 2/3 +Search Algorithm: generic exclusion search +SELECT count() FROM t_null WHERE g = 'b' AND r IS NOT NULL; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r IS NOT NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r isNotNull), (g in [\'b\', \'b\'])) +Parts: 1/1 +Granules: 1/3 +Search Algorithm: generic exclusion search +SELECT count() FROM t_null WHERE g = 'a' AND r IS NOT NULL; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'a' AND r IS NOT NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r isNotNull), (g in [\'a\', \'a\'])) +Parts: 1/1 +Granules: 2/3 +Search Algorithm: generic exclusion search +SELECT count() FROM t_null WHERE r IS NULL; +3 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE r IS NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: (r isNull) +Parts: 1/1 +Granules: 3/3 +Search Algorithm: generic exclusion search +DROP TABLE t_null; +SELECT 'binary search over many granules'; +binary search over many granules +DROP TABLE IF EXISTS t_big; +CREATE TABLE t_big (g UInt32, r UInt32) +ENGINE = MergeTree ORDER BY (g, r DESC) SETTINGS index_granularity = 4; +INSERT INTO t_big SELECT number % 10, 1000 - number FROM numbers(1000); +SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [995, 995]), (g in [5, 5])) +Parts: 1/1 +Granules: 2/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [995, 995]), (g in [5, 5])) +Parts: 1/1 +Granules: 2/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [945, 945]), (g in [5, 5])) +Parts: 1/1 +Granules: 1/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [945, 945]), (g in [5, 5])) +Parts: 1/1 +Granules: 1/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r >= 900; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r >= 900) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [900, +Inf)), (g in [5, 5])) +Parts: 1/1 +Granules: 4/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r BETWEEN 500 AND 600; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r BETWEEN 500 AND 600) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and(and((r in (-Inf, 600]), (r in [500, +Inf))), (g in [5, 5])) +Parts: 1/1 +Granules: 4/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 9 AND r < 100; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 9 AND r < 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in (-Inf, 99]), (g in [9, 9])) +Parts: 1/1 +Granules: 3/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE r = 995; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE r = 995) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: (r in [995, 995]) +Parts: 1/1 +Granules: 15/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_big WHERE g = 5 AND toInt64(r) >= 900; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND toInt64(r) >= 900) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((toInt64(r) in [900, +Inf)), (g in [5, 5])) +Parts: 1/1 +Granules: 4/250 +Search Algorithm: binary search +SELECT count() FROM t_big WHERE g = 5 AND r IN (995, 5, 123); +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r IN (995, 5, 123)) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in 3-element set), (g in [5, 5])) +Parts: 1/1 +Granules: 4/250 +Search Algorithm: generic exclusion search +DROP TABLE t_big; +SELECT 'part without a final mark'; +part without a final mark +DROP TABLE IF EXISTS t_nofinal; +CREATE TABLE t_nofinal (g UInt8, r UInt8) +ENGINE = MergeTree ORDER BY (g, r DESC) +SETTINGS index_granularity = 3, index_granularity_bytes = 0, min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; +INSERT INTO t_nofinal SELECT 1 + intDiv(number, 5), 5 - number % 5 FROM numbers(10); +SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [1, 1]), (g in [2, 2])) +Parts: 1/1 +Granules: 2/4 +Search Algorithm: binary search +SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [1, 1]), (g in [2, 2])) +Parts: 1/1 +Granules: 2/4 +Search Algorithm: binary search +SELECT count() FROM t_nofinal WHERE g = 2 AND r <= 2; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r <= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in (-Inf, 2]), (g in [2, 2])) +Parts: 1/1 +Granules: 2/4 +Search Algorithm: binary search +SELECT count() FROM t_nofinal WHERE g = 2 AND r >= 4; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [4, +Inf)), (g in [2, 2])) +Parts: 1/1 +Granules: 2/4 +Search Algorithm: binary search +SELECT count() FROM t_nofinal WHERE g = 1 AND r = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 1 AND r = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [1, 1]), (g in [1, 1])) +Parts: 1/1 +Granules: 1/4 +Search Algorithm: binary search +DROP TABLE t_nofinal; +SELECT 'middle DESC column not referenced by the filter'; +middle DESC column not referenced by the filter +DROP TABLE IF EXISTS t_skip; +CREATE TABLE t_skip (a UInt16, b UInt16, c UInt16) +ENGINE = MergeTree ORDER BY (a, b DESC, c) SETTINGS index_granularity = 4; +INSERT INTO t_skip SELECT intDiv(number, 100), 9 - intDiv(number % 100, 10), number % 10 FROM numbers(1000); +SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((c in [7, 7]), (a in [5, 5])) +Parts: 1/1 +Granules: 16/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((c in [7, 7]), (a in [5, 5])) +Parts: 1/1 +Granules: 16/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c = 7; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c = 7) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((c in [7, 7]), and((b in [3, 3]), (a in [5, 5]))) +Parts: 1/1 +Granules: 1/250 +Search Algorithm: binary search +SELECT count() FROM t_skip WHERE a = 5 AND b >= 8 AND c <= 1; +4 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b >= 8 AND c <= 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((c in (-Inf, 1]), and((b in [8, +Inf)), (a in [5, 5]))) +Parts: 1/1 +Granules: 3/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c >= 8; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c >= 8) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((c in [8, +Inf)), and((b in [3, 3]), (a in [5, 5]))) +Parts: 1/1 +Granules: 2/250 +Search Algorithm: binary search +DROP TABLE t_skip; +SELECT 'DESC key column not loaded in the in-memory index'; +DESC key column not loaded in the in-memory index +DROP TABLE IF EXISTS t_unloaded; +CREATE TABLE t_unloaded (a UInt16, b UInt16, c UInt16) +ENGINE = MergeTree ORDER BY (a, b DESC, c) +SETTINGS index_granularity = 4, + primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns = 0.01; +INSERT INTO t_unloaded SELECT intDiv(number, 100), 9 - intDiv(number % 100, 10), number % 10 FROM numbers(1000); +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((b in [3, 3]), (a in [5, 5]))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0; +10 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((b in [3, 3]), (a in [5, 5]))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1; +20 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((b in [8, +Inf)), (a in [5, 5]))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0; +20 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((b in [8, +Inf)), (a in [5, 5]))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3 AND c = 7) OR a >= 100; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3 AND c = 7) OR a >= 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((c in [7, 7]), and((b in [3, 3]), (a in [5, 5])))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +SELECT count() FROM t_unloaded WHERE (a = 5 AND b <= 1) OR a >= 100; +20 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b <= 1) OR a >= 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: or((a in [100, +Inf)), and((b in (-Inf, 1]), (a in [5, 5]))) +Parts: 1/1 +Granules: 26/250 +Search Algorithm: generic exclusion search +DROP TABLE t_unloaded; +SELECT 'explicit PRIMARY KEY clause inherits ORDER BY directions'; +explicit PRIMARY KEY clause inherits ORDER BY directions +DROP TABLE IF EXISTS t_explicit_pk; +CREATE TABLE t_explicit_pk (g String, r Int8) +ENGINE = MergeTree ORDER BY (g, r DESC) PRIMARY KEY (g, r); +INSERT INTO t_explicit_pk VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r >= 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r >= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, +Inf)), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +DROP TABLE t_explicit_pk; +SELECT 'ascending control'; +ascending control +DROP TABLE IF EXISTS t_asc; +CREATE TABLE t_asc (g String, r Int8) ENGINE = MergeTree ORDER BY (g, r); +INSERT INTO t_asc VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_asc WHERE g = 'novel' AND r = 3; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_asc WHERE g = 'novel' AND r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [3, 3]), (g in [\'novel\', \'novel\'])) +Parts: 1/1 +Granules: 1/1 +Search Algorithm: binary search +DROP TABLE t_asc; +SELECT 'NaN runs in a float key column form mark boundaries'; +NaN runs in a float key column form mark boundaries +SET allow_suspicious_primary_key = 1; +DROP TABLE IF EXISTS t_nan; +CREATE TABLE t_nan (g UInt32, r Float64) +ENGINE = MergeTree ORDER BY (g, r DESC) SETTINGS index_granularity = 1; +INSERT INTO t_nan VALUES (1, 0/0), (1, 0/0), (1, 5), (1, 3), (2, 0/0), (2, 7); +SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [4., +Inf)), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [4., +Inf)), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan WHERE g = 1 AND r = 5; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r = 5) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [5., 5.]), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan WHERE g = 1 AND r <= 3; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r <= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in (-Inf, 3.]), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan WHERE g = 2; +2 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: (g in [2, 2]) +Parts: 1/1 +Granules: 3/6 +Search Algorithm: binary search +SELECT count() FROM t_nan WHERE g = 1 AND isNaN(r); +2 +DROP TABLE t_nan; +DROP TABLE IF EXISTS t_nan_asc; +CREATE TABLE t_nan_asc (g UInt32, r Float64) +ENGINE = MergeTree ORDER BY (g, r) SETTINGS index_granularity = 1; +INSERT INTO t_nan_asc VALUES (1, 3), (1, 5), (1, 0/0), (1, 0/0), (2, 7), (2, 0/0); +SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 1; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [5., 5.]), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 0; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [5., 5.]), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan_asc WHERE g = 1 AND r >= 4; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [4., +Inf)), (g in [1, 1])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan_asc WHERE g = 2 AND r = 7; +1 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 2 AND r = 7) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((r in [7., 7.]), (g in [2, 2])) +Parts: 1/1 +Granules: 2/6 +Search Algorithm: binary search +SELECT count() FROM t_nan_asc WHERE g = 1 AND isNaN(r); +2 +DROP TABLE t_nan_asc; +SELECT 'realistic ranges across merged parts'; +realistic ranges across merged parts +-- Several inserts merged with FINAL exercise the merge path writing reverse-sorted keys, and the +-- per-group time ranges are nested (the middle group is denser and wider), so range conditions cut +-- granule boundaries at different positions in every group. The ground truth countIf scans without +-- using the primary key, so its value must literally equal the indexed counts below it. +DROP TABLE IF EXISTS t_ranges; +CREATE TABLE t_ranges (org String, dt DateTime('UTC'), id UInt64) +ENGINE = MergeTree ORDER BY (org, dt DESC, id) SETTINGS index_granularity = 128; +INSERT INTO t_ranges SELECT 'org_a', toDateTime('2026-06-01', 'UTC') + intDiv(number * 2592000, 2000), number FROM numbers(2000); +INSERT INTO t_ranges SELECT 'org_m', toDateTime('2026-06-01', 'UTC') + intDiv(number * 2592000, 8000), number FROM numbers(8000); +INSERT INTO t_ranges SELECT 'org_z', toDateTime('2026-06-10', 'UTC') + intDiv(number * 1814400, 2000), number FROM numbers(2000); +OPTIMIZE TABLE t_ranges FINAL; +SELECT countIf(org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC')) FROM t_ranges; +2933 +SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1; +2933 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in [1781913601, +Inf)), (org in [\'org_m\', \'org_m\'])) +Parts: 1/1 +Granules: 24/95 +Search Algorithm: binary search +SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0; +2933 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in [1781913601, +Inf)), (org in [\'org_m\', \'org_m\'])) +Parts: 1/1 +Granules: 24/95 +Search Algorithm: binary search +SELECT countIf(org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC')) FROM t_ranges; +267 +SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1; +267 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in (-Inf, 1780617599]), (org in [\'org_a\', \'org_a\'])) +Parts: 1/1 +Granules: 3/95 +Search Algorithm: binary search +SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0; +267 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in (-Inf, 1780617599]), (org in [\'org_a\', \'org_a\'])) +Parts: 1/1 +Granules: 3/95 +Search Algorithm: binary search +SELECT countIf(org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC')) FROM t_ranges; +285 +SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC'); +285 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC')) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in [1782604801, +Inf)), (org in [\'org_z\', \'org_z\'])) +Parts: 1/1 +Granules: 4/95 +Search Algorithm: binary search +SELECT countIf(org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC')) FROM t_ranges; +0 +SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC'); +0 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC')) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in (-Inf, 1781049599]), (org in [\'org_z\', \'org_z\'])) +Parts: 0/1 +Granules: 0/95 +Search Algorithm: binary search +-- Reading in order with a limit must see the same rows the index analysis selects. +SELECT dt, id FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1; +2026-06-04 23:45:36 266 +2026-06-04 23:24:00 265 +2026-06-04 23:02:24 264 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT dt, id FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in (-Inf, 1780617599]), (org in [\'org_a\', \'org_a\'])) +Parts: 1/1 +Granules: 3/95 +Search Algorithm: binary search +SELECT dt, id FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1; +2026-06-30 23:44:52 1999 +2026-06-30 23:29:45 1998 +2026-06-30 23:14:38 1997 +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT dt, id FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +Condition: and((dt in [1782604801, +Inf)), (org in [\'org_z\', \'org_z\'])) +Parts: 1/1 +Granules: 4/95 +Search Algorithm: binary search +DROP TABLE t_ranges; diff --git a/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql b/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql new file mode 100644 index 000000000000..369c0a0ca608 --- /dev/null +++ b/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql @@ -0,0 +1,274 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: EXPLAIN output may differ with random settings. + +-- Primary key pruning on tables with reverse-sorted key columns must never drop granules that +-- contain matching rows. The shapes below are the tricky ones: a condition that bounds a DESC +-- column from below while another key column also participates, mark-range boundaries where an +-- earlier key column changes between marks, NULLs (stored physically first on a DESC column), +-- parts without a final mark (non-adaptive granularity), key columns not loaded in the in-memory +-- index, and key columns skipped by the sparse analysis. The core queries run through both index +-- analysis paths (use_lightweight_primary_key_index_analysis 1 and 0), and every query has an +-- EXPLAIN companion pinning the pruning decision (Parts/Granules), not only the result. + +-- { echo } + +SELECT 'equality on both key columns, DESC Enum second'; +DROP TABLE IF EXISTS t_enum_rev; +CREATE TABLE t_enum_rev (g String, r Enum8('poor' = 1, 'ok' = 2, 'great' = 3)) +ENGINE = MergeTree ORDER BY (g, r DESC); +INSERT INTO t_enum_rev VALUES ('manual', 'ok'), ('manual', 'poor'), ('novel', 'great'), ('novel', 'great'); +SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_enum_rev WHERE g = 'novel' AND r = 'great' SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_enum_rev; + +SELECT 'bounds on the DESC column in each direction'; +DROP TABLE IF EXISTS t_str_int; +CREATE TABLE t_str_int (g String, r Int8) +ENGINE = MergeTree ORDER BY (g, r DESC); +INSERT INTO t_str_int VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'novel' AND r >= 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r >= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'novel' AND r <= 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r <= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g >= 'novel' AND r = 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g >= 'novel' AND r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'manual' AND r > 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'manual' AND r > 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'manual' AND r = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'manual' AND r = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 2; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_str_int WHERE g = 'novel' AND r = 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_str_int; + +SELECT 'DESC column first'; +DROP TABLE IF EXISTS t_rev_first; +CREATE TABLE t_rev_first (r Int8, g String) +ENGINE = MergeTree ORDER BY (r DESC, g); +INSERT INTO t_rev_first VALUES (2, 'manual'), (1, 'manual'), (3, 'novel'), (3, 'zzz'); +SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3 AND g = 'novel' SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_rev_first WHERE r = 1 AND g = 'manual'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 1 AND g = 'manual') WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_rev_first WHERE r >= 2 AND g >= 'a'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r >= 2 AND g >= 'a') WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_rev_first WHERE r = 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_rev_first WHERE r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_rev_first; + +SELECT 'two DESC columns'; +DROP TABLE IF EXISTS t_both_rev; +CREATE TABLE t_both_rev (a UInt8, b UInt8) +ENGINE = MergeTree ORDER BY (a DESC, b DESC) SETTINGS index_granularity = 2; +INSERT INTO t_both_rev SELECT 1 + intDiv(number, 3), 1 + number % 3 FROM numbers(9); +SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b = 2 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_both_rev WHERE a = 2 AND b >= 2; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a = 2 AND b >= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_both_rev WHERE a >= 2 AND b <= 2; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_both_rev WHERE a >= 2 AND b <= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_both_rev; + +SELECT 'NULLs sit physically first on a DESC column'; +DROP TABLE IF EXISTS t_null; +CREATE TABLE t_null (g String, r Nullable(Int8)) +ENGINE = MergeTree ORDER BY (g, r DESC) +SETTINGS allow_nullable_key = 1, index_granularity = 2; +INSERT INTO t_null VALUES ('a', NULL), ('a', 5), ('a', 3), ('b', NULL), ('b', NULL), ('b', 7); +SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE g = 'a' AND r >= 4; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'a' AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE g = 'b' AND r IS NULL; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r IS NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE g = 'b' AND r IS NOT NULL; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'b' AND r IS NOT NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE g = 'a' AND r IS NOT NULL; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE g = 'a' AND r IS NOT NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_null WHERE r IS NULL; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_null WHERE r IS NULL) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_null; + +SELECT 'binary search over many granules'; +DROP TABLE IF EXISTS t_big; +CREATE TABLE t_big (g UInt32, r UInt32) +ENGINE = MergeTree ORDER BY (g, r DESC) SETTINGS index_granularity = 4; +INSERT INTO t_big SELECT number % 10, 1000 - number FROM numbers(1000); +SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 995 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r = 945 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r >= 900; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r >= 900) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r BETWEEN 500 AND 600; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r BETWEEN 500 AND 600) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 9 AND r < 100; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 9 AND r < 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE r = 995; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE r = 995) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND toInt64(r) >= 900; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND toInt64(r) >= 900) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_big WHERE g = 5 AND r IN (995, 5, 123); +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_big WHERE g = 5 AND r IN (995, 5, 123)) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_big; + +SELECT 'part without a final mark'; +DROP TABLE IF EXISTS t_nofinal; +CREATE TABLE t_nofinal (g UInt8, r UInt8) +ENGINE = MergeTree ORDER BY (g, r DESC) +SETTINGS index_granularity = 3, index_granularity_bytes = 0, min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; +INSERT INTO t_nofinal SELECT 1 + intDiv(number, 5), 5 - number % 5 FROM numbers(10); +SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r = 1 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nofinal WHERE g = 2 AND r <= 2; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r <= 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nofinal WHERE g = 2 AND r >= 4; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 2 AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nofinal WHERE g = 1 AND r = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nofinal WHERE g = 1 AND r = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_nofinal; + +SELECT 'middle DESC column not referenced by the filter'; +DROP TABLE IF EXISTS t_skip; +CREATE TABLE t_skip (a UInt16, b UInt16, c UInt16) +ENGINE = MergeTree ORDER BY (a, b DESC, c) SETTINGS index_granularity = 4; +INSERT INTO t_skip SELECT intDiv(number, 100), 9 - intDiv(number % 100, 10), number % 10 FROM numbers(1000); +SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND c = 7 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c = 7; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c = 7) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_skip WHERE a = 5 AND b >= 8 AND c <= 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b >= 8 AND c <= 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c >= 8; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_skip WHERE a = 5 AND b = 3 AND c >= 8) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_skip; + +SELECT 'DESC key column not loaded in the in-memory index'; +DROP TABLE IF EXISTS t_unloaded; +CREATE TABLE t_unloaded (a UInt16, b UInt16, c UInt16) +ENGINE = MergeTree ORDER BY (a, b DESC, c) +SETTINGS index_granularity = 4, + primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns = 0.01; +INSERT INTO t_unloaded SELECT intDiv(number, 100), 9 - intDiv(number % 100, 10), number % 10 FROM numbers(1000); +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b >= 8) OR a >= 100 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3 AND c = 7) OR a >= 100; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b = 3 AND c = 7) OR a >= 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_unloaded WHERE (a = 5 AND b <= 1) OR a >= 100; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_unloaded WHERE (a = 5 AND b <= 1) OR a >= 100) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_unloaded; + +SELECT 'explicit PRIMARY KEY clause inherits ORDER BY directions'; +DROP TABLE IF EXISTS t_explicit_pk; +CREATE TABLE t_explicit_pk (g String, r Int8) +ENGINE = MergeTree ORDER BY (g, r DESC) PRIMARY KEY (g, r); +INSERT INTO t_explicit_pk VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r >= 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_explicit_pk WHERE g = 'novel' AND r >= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_explicit_pk; + +SELECT 'ascending control'; +DROP TABLE IF EXISTS t_asc; +CREATE TABLE t_asc (g String, r Int8) ENGINE = MergeTree ORDER BY (g, r); +INSERT INTO t_asc VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); +SELECT count() FROM t_asc WHERE g = 'novel' AND r = 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_asc WHERE g = 'novel' AND r = 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_asc; + +SELECT 'NaN runs in a float key column form mark boundaries'; +SET allow_suspicious_primary_key = 1; +DROP TABLE IF EXISTS t_nan; +CREATE TABLE t_nan (g UInt32, r Float64) +ENGINE = MergeTree ORDER BY (g, r DESC) SETTINGS index_granularity = 1; +INSERT INTO t_nan VALUES (1, 0/0), (1, 0/0), (1, 5), (1, 3), (2, 0/0), (2, 7); +SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r >= 4 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan WHERE g = 1 AND r = 5; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r = 5) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan WHERE g = 1 AND r <= 3; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 1 AND r <= 3) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan WHERE g = 2; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan WHERE g = 2) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan WHERE g = 1 AND isNaN(r); +DROP TABLE t_nan; + +DROP TABLE IF EXISTS t_nan_asc; +CREATE TABLE t_nan_asc (g UInt32, r Float64) +ENGINE = MergeTree ORDER BY (g, r) SETTINGS index_granularity = 1; +INSERT INTO t_nan_asc VALUES (1, 3), (1, 5), (1, 0/0), (1, 0/0), (2, 7), (2, 0/0); +SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r = 5 SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan_asc WHERE g = 1 AND r >= 4; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 1 AND r >= 4) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan_asc WHERE g = 2 AND r = 7; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_nan_asc WHERE g = 2 AND r = 7) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_nan_asc WHERE g = 1 AND isNaN(r); +DROP TABLE t_nan_asc; + +SELECT 'realistic ranges across merged parts'; +-- Several inserts merged with FINAL exercise the merge path writing reverse-sorted keys, and the +-- per-group time ranges are nested (the middle group is denser and wider), so range conditions cut +-- granule boundaries at different positions in every group. The ground truth countIf scans without +-- using the primary key, so its value must literally equal the indexed counts below it. +DROP TABLE IF EXISTS t_ranges; +CREATE TABLE t_ranges (org String, dt DateTime('UTC'), id UInt64) +ENGINE = MergeTree ORDER BY (org, dt DESC, id) SETTINGS index_granularity = 128; +INSERT INTO t_ranges SELECT 'org_a', toDateTime('2026-06-01', 'UTC') + intDiv(number * 2592000, 2000), number FROM numbers(2000); +INSERT INTO t_ranges SELECT 'org_m', toDateTime('2026-06-01', 'UTC') + intDiv(number * 2592000, 8000), number FROM numbers(8000); +INSERT INTO t_ranges SELECT 'org_z', toDateTime('2026-06-10', 'UTC') + intDiv(number * 1814400, 2000), number FROM numbers(2000); +OPTIMIZE TABLE t_ranges FINAL; +SELECT countIf(org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC')) FROM t_ranges; +SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_m' AND dt > toDateTime('2026-06-20', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT countIf(org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC')) FROM t_ranges; +SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') SETTINGS use_lightweight_primary_key_index_analysis = 0) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT countIf(org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC')) FROM t_ranges; +SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC'); +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC')) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT countIf(org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC')) FROM t_ranges; +SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC'); +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT count() FROM t_ranges WHERE org = 'org_z' AND dt < toDateTime('2026-06-10', 'UTC')) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +-- Reading in order with a limit must see the same rows the index analysis selects. +SELECT dt, id FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT dt, id FROM t_ranges WHERE org = 'org_a' AND dt < toDateTime('2026-06-05', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +SELECT dt, id FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1, actions = 0, pretty = 0 SELECT dt, id FROM t_ranges WHERE org = 'org_z' AND dt > toDateTime('2026-06-28', 'UTC') ORDER BY org, dt DESC, id LIMIT 3 SETTINGS optimize_read_in_order = 1) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Granules%' OR explain LIKE '%Search Algorithm%'; +DROP TABLE t_ranges; diff --git a/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.reference b/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.reference new file mode 100644 index 000000000000..26fa49e7aa93 --- /dev/null +++ b/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.reference @@ -0,0 +1,9 @@ +before alter +2 +2 +after alter +2 +2 +2 +after detach and attach +2 diff --git a/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.sql b/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.sql new file mode 100644 index 000000000000..6ea5426d81d9 --- /dev/null +++ b/tests/queries/0_stateless/04613_reverse_key_replicated_metadata.sql @@ -0,0 +1,35 @@ +-- Tags: zookeeper, no-random-merge-tree-settings + +-- A replica applies metadata changes from ZooKeeper through +-- ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata, which rebuilds an explicitly defined +-- primary key from its own PRIMARY KEY clause. The clause cannot express per-column directions, +-- so the rebuilt description must inherit the DESC modifiers from the sorting key; losing them +-- would make primary key pruning analyze the reverse key as ascending and skip granules that +-- contain matching rows. + +DROP TABLE IF EXISTS t_rev_replicated SYNC; +CREATE TABLE t_rev_replicated (g String, r Int8) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/04613_reverse_key/t', 'r1') +ORDER BY (g, r DESC) PRIMARY KEY (g, r); +INSERT INTO t_rev_replicated VALUES ('manual', 2), ('manual', 1), ('novel', 3), ('novel', 3); + +SELECT 'before alter'; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r = 3; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r >= 3; + +-- The ALTER is applied through the replication log, so the replica rebuilds its metadata from the +-- ZooKeeper diff; the queries must keep seeing the rows afterwards. +ALTER TABLE t_rev_replicated ADD COLUMN extra UInt8 DEFAULT 0; + +SELECT 'after alter'; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r = 3; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r >= 3; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r = 3 SETTINGS use_lightweight_primary_key_index_analysis = 0; + +DETACH TABLE t_rev_replicated; +ATTACH TABLE t_rev_replicated; + +SELECT 'after detach and attach'; +SELECT count() FROM t_rev_replicated WHERE g = 'novel' AND r = 3; + +DROP TABLE t_rev_replicated SYNC; From eecfd0eab30eff2abea388d5db7ca5f0a0df0585 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 12:06:33 +0000 Subject: [PATCH 47/86] Backport #111442 to 26.6: Fix checking paths against `user_files` --- src/Databases/SQLite/SQLiteUtils.cpp | 3 +- src/Formats/FormatSchemaInfo.cpp | 6 +-- src/Functions/FunctionFile.cpp | 3 +- src/Interpreters/InterpreterCreateQuery.cpp | 5 ++- src/Storages/StorageFile.cpp | 4 ++ ...ile_function_user_files_boundary.reference | 6 +++ ...04614_file_function_user_files_boundary.sh | 43 +++++++++++++++++++ 7 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/04614_file_function_user_files_boundary.reference create mode 100755 tests/queries/0_stateless/04614_file_function_user_files_boundary.sh diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 848459ecacd9..8418a4b82b37 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -2,6 +2,7 @@ #if USE_SQLITE #include +#include #include #include @@ -33,7 +34,7 @@ static String validateSQLiteDatabasePath(const String & path, const String & use String absolute_user_files_path = fs::absolute(user_files_path).lexically_normal(); - if (need_check && !absolute_path.starts_with(absolute_user_files_path)) + if (need_check && !fileOrSymlinkPathStartsWith(absolute_path, absolute_user_files_path)) { processSQLiteError(fmt::format("SQLite database file path '{}' must be inside 'user_files' directory", path), throw_on_error); return ""; diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index b6cad14b33a8..6a1170c8232f 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -317,9 +317,7 @@ void FormatSchemaInfo::processSchemaFile( } else if ( path.has_parent_path() - && !fs::weakly_canonical(default_schema_directory_path / path) - .string() - .starts_with(fs::weakly_canonical(default_schema_directory_path).string())) + && !pathStartsWith(default_schema_directory_path / path, default_schema_directory_path)) { if (is_server) throw Exception( @@ -385,7 +383,7 @@ MaybeAutogeneratedFormatSchemaInfo::MaybeAutogeneratedFormatSch if (settings.schema.is_server) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Absolute path in the 'output_format_schema' setting is prohibited: {}", path.string()); } - else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string())) + else if (path.has_parent_path() && !pathStartsWith(default_schema_directory_path / path, default_schema_directory_path)) { if (settings.schema.is_server) throw Exception( diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index c9e8a01439df..f782b29d8263 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -144,7 +145,7 @@ class FunctionFile final : public IFunction try { - if (need_check && !file_path.string().starts_with(user_files_absolute_path_string)) + if (need_check && !fileOrSymlinkPathStartsWith(file_path.string(), user_files_absolute_path_string)) throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string()); ReadBufferFromFile in(file_path); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ced702fa3be4..6ba105d7ea64 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1710,7 +1711,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) fs::path data_path = fs::path(create.attach_from_path).lexically_normal(); if (data_path.is_relative()) data_path = (user_files / data_path).lexically_normal(); - if (!startsWith(data_path, user_files)) + if (!fileOrSymlinkPathStartsWith(data_path.string(), user_files.string())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "Data directory {} must be inside {} to attach it", String(data_path), String(user_files)); @@ -1720,7 +1721,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) else { fs::path data_path = (root_path / create.attach_from_path).lexically_normal(); - if (!startsWith(data_path, user_files)) + if (!fileOrSymlinkPathStartsWith(data_path.string(), user_files.string())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "Data directory {} must be inside {} to attach it", String(data_path), String(user_files)); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 643bef2f593e..b6ce1dcd9a4e 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -1351,6 +1351,10 @@ String StorageFileSource::FilesIterator::next() auto task = getContext()->getClusterFunctionReadTaskCallback()(); if (!task || task->isEmpty()) return {}; + + /// The read task may come from a client impersonating an initiator server, so validate the path. + checkCreationIsAllowed(getContext(), getContext()->getUserFilesPath(), task->path, /*can_be_directory=*/ true); + return task->path; } diff --git a/tests/queries/0_stateless/04614_file_function_user_files_boundary.reference b/tests/queries/0_stateless/04614_file_function_user_files_boundary.reference new file mode 100644 index 000000000000..c4d7897bf33d --- /dev/null +++ b/tests/queries/0_stateless/04614_file_function_user_files_boundary.reference @@ -0,0 +1,6 @@ +--- sibling directory (absolute path) --- +is not inside +--- sibling directory (relative path) --- +is not inside +--- nested path inside user_files --- +ok diff --git a/tests/queries/0_stateless/04614_file_function_user_files_boundary.sh b/tests/queries/0_stateless/04614_file_function_user_files_boundary.sh new file mode 100755 index 000000000000..9cdd30d55ddc --- /dev/null +++ b/tests/queries/0_stateless/04614_file_function_user_files_boundary.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Regression test for a directory-boundary bypass in the `file()` function. +# The path check used a plain string-prefix test, so a *sibling* directory whose +# name merely begins with `user_files` (e.g. `.../user_files_evil`) was treated as +# being inside `user_files_path` and its contents could be read. A real boundary +# check must reject it. + +# A sibling of user_files_path whose name starts with the same prefix. The unique +# suffix keeps concurrent runs of this test from colliding on the same directory. +EVIL_SUFFIX="_evil_${CLICKHOUSE_TEST_UNIQUE_NAME}" +EVIL_DIR="${USER_FILES_PATH}${EVIL_SUFFIX}" + +cleanup() { + rm -rf "${EVIL_DIR}" + rm -rf "${USER_FILES_PATH:?}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +} +trap cleanup EXIT + +mkdir -p "${EVIL_DIR}" +echo -n "LEAKED" > "${EVIL_DIR}/secret.txt" + +# A legitimate file nested inside user_files_path, to prove the boundary check +# does not over-reject paths that are genuinely inside the directory. +mkdir -p "${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +echo -n "ok" > "${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}/inside.txt" + +# The sibling directory must be rejected: the content of secret.txt must not leak. +# We only distinguish the access-denied rejection ("is not inside") from a leak. +echo "--- sibling directory (absolute path) ---" +${CLICKHOUSE_CLIENT} --query "SELECT file('${EVIL_DIR}/secret.txt')" 2>&1 | grep -o -m1 "is not inside\|LEAKED" || echo "UNEXPECTED" + +# Same escape expressed as a relative path from user_files_path. +echo "--- sibling directory (relative path) ---" +${CLICKHOUSE_CLIENT} --query "SELECT file('../user_files${EVIL_SUFFIX}/secret.txt')" 2>&1 | grep -o -m1 "is not inside\|LEAKED" || echo "UNEXPECTED" + +# A genuinely nested path is still allowed. +echo "--- nested path inside user_files ---" +${CLICKHOUSE_CLIENT} --query "SELECT file('${CLICKHOUSE_TEST_UNIQUE_NAME}/inside.txt')" From 24fe5a96c9ba606125e5c981d3c75d06fd344670 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 12:44:52 +0000 Subject: [PATCH 48/86] Backport #111606 to 26.6: Fix OOB when deserializing bad aggregate function states --- src/Columns/ColumnString.cpp | 3 +++ tests/queries/0_stateless/02477_invalid_reads.sql | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 597121b7da99..5a833bc74f90 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -363,6 +363,9 @@ void ColumnString::deserializeAndInsertFromArena(ReadBuffer & in, const IColumn: readBinaryLittleEndian(string_size, in); bool serialize_string_with_zero_byte = settings && settings->serialize_string_with_zero_byte; + if (string_size < serialize_string_with_zero_byte) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed serialized string in aggregation state: size {} is smaller than the zero-byte terminator", string_size); const size_t old_size = chars.size(); const size_t new_size = old_size + string_size - serialize_string_with_zero_byte; chars.resize(new_size); diff --git a/tests/queries/0_stateless/02477_invalid_reads.sql b/tests/queries/0_stateless/02477_invalid_reads.sql index 1e362fc75753..9cddff7ba716 100644 --- a/tests/queries/0_stateless/02477_invalid_reads.sql +++ b/tests/queries/0_stateless/02477_invalid_reads.sql @@ -59,3 +59,8 @@ SELECT finalizeAggregation(CAST(unhex('0F0000000000000000'), 'AggregateFunction(quantileExact, UInt64)')); -- { serverError CANNOT_READ_ALL_DATA } SELECT finalizeAggregation(CAST(unhex('0F000000000000803F'), 'AggregateFunction(quantileTDigest, UInt64)')); -- { serverError CANNOT_READ_ALL_DATA } + +-- groupUniqArray over a composite type deserializes each stored key through the ColumnString arena path; +-- a zero string_size must be rejected instead of underflowing to a huge/OOB allocation. +SELECT finalizeAggregation(CAST(unhex('01080000000000000000'), + 'AggregateFunction(groupUniqArray, Tuple(String))')); -- { serverError INCORRECT_DATA } From b4d197e2955e1f043c736c66949e1ef27d930f38 Mon Sep 17 00:00:00 2001 From: "Nihal Z. Miaji" <81457724+nihalzp@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:53:16 +0600 Subject: [PATCH 49/86] Update 04612_reverse_key_index_analysis.sql --- .../0_stateless/04612_reverse_key_index_analysis.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql b/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql index 369c0a0ca608..3612f4acf17e 100644 --- a/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql +++ b/tests/queries/0_stateless/04612_reverse_key_index_analysis.sql @@ -1,14 +1,6 @@ -- Tags: no-random-settings, no-random-merge-tree-settings -- no-random-settings, no-random-merge-tree-settings: EXPLAIN output may differ with random settings. --- Primary key pruning on tables with reverse-sorted key columns must never drop granules that --- contain matching rows. The shapes below are the tricky ones: a condition that bounds a DESC --- column from below while another key column also participates, mark-range boundaries where an --- earlier key column changes between marks, NULLs (stored physically first on a DESC column), --- parts without a final mark (non-adaptive granularity), key columns not loaded in the in-memory --- index, and key columns skipped by the sparse analysis. The core queries run through both index --- analysis paths (use_lightweight_primary_key_index_analysis 1 and 0), and every query has an --- EXPLAIN companion pinning the pruning decision (Parts/Granules), not only the result. -- { echo } From f658c3853dad8eafeaa8873f5c2e1747196c8682 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 28 Jul 2026 14:05:45 +0000 Subject: [PATCH 50/86] Drop `explain_query_plan_default` from the test: it does not exist on `26.6` The backported test carried `SET explain_query_plan_default = 'legacy'`, a setting added after `26.6`, so every run failed with `UNKNOWN_SETTING` (115). On `26.6` the legacy `EXPLAIN` output is already the only output, so the pin is unnecessary here. Verified with the release binaries: the test reproduces `Block structure mismatch` (352) on stock `26.6` and matches the reference with this backport applied. Co-Authored-By: Claude Opus 5 (1M context) --- ...04650_join_use_nulls_repeated_on_condition_in_where.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql index 45569615715d..12f1111d3a53 100644 --- a/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql +++ b/tests/queries/0_stateless/04650_join_use_nulls_repeated_on_condition_in_where.sql @@ -1,8 +1,7 @@ --- The plan assertions below match analyzer-generated column identifiers (`__table2.`) in the legacy --- `EXPLAIN` output, so both are pinned for the whole file. The old analyzer does not build the plan --- shape that triggers this bug, so nothing is lost by pinning. +-- The plan assertions below match analyzer-generated column identifiers (`__table2.`) in the +-- `EXPLAIN` output, so the analyzer is pinned for the whole file. The old analyzer does not build the +-- plan shape that triggers this bug, so nothing is lost by pinning. SET enable_analyzer = 1; -SET explain_query_plan_default = 'legacy'; DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; From d4df0906e8a0d8ca65dd8aad4bd24ca20d038be8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 17:35:28 +0000 Subject: [PATCH 51/86] Backport #109768 to 26.6: Mask session_token and Google ADC secrets in the explicit-url S3 form --- .../FunctionSecretArgumentsFinderTreeNode.cpp | 70 + .../FunctionSecretArgumentsFinderTreeNode.h | 25 + src/Analyzer/Resolve/resolveFunction.cpp | 44 +- src/Databases/DatabaseS3.cpp | 51 +- src/Databases/DatabaseS3.h | 3 - src/Interpreters/ActionsDAG.cpp | 3 +- src/Interpreters/ActionsDAG.h | 6 +- src/Interpreters/InterpreterExplainQuery.cpp | 93 +- src/Parsers/ASTFunction.cpp | 102 +- src/Parsers/FunctionSecretArgumentsFinder.cpp | 1241 +++++++++++++++++ src/Parsers/FunctionSecretArgumentsFinder.h | 1000 ++----------- .../FunctionSecretArgumentsFinderAST.h | 10 + src/Planner/PlannerActionsVisitor.cpp | 39 +- src/Processors/QueryPlan/QueryPlanFormat.cpp | 16 + .../0_stateless/02968_url_args.reference | 2 +- ...at_inference_create_query_s3_url.reference | 6 +- ...gs_finder_mixed_named_positional.reference | 6 +- ...3_explicit_url_named_secret_mask.reference | 220 +++ ...4510_s3_explicit_url_named_secret_mask.sql | 332 +++++ ...cret_args_expression_derived_key.reference | 34 + ...628_secret_args_expression_derived_key.sql | 40 + .../04648_url_secret_masking_forms.reference | 131 ++ .../04648_url_secret_masking_forms.sql | 41 + tmp/check_files.sh | 36 + 24 files changed, 2553 insertions(+), 998 deletions(-) create mode 100644 src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp create mode 100644 src/Parsers/FunctionSecretArgumentsFinder.cpp create mode 100644 tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference create mode 100644 tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql create mode 100644 tests/queries/0_stateless/04628_secret_args_expression_derived_key.reference create mode 100644 tests/queries/0_stateless/04628_secret_args_expression_derived_key.sql create mode 100644 tests/queries/0_stateless/04648_url_secret_masking_forms.reference create mode 100644 tests/queries/0_stateless/04648_url_secret_masking_forms.sql create mode 100644 tmp/check_files.sh diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp new file mode 100644 index 000000000000..8c61d13923ca --- /dev/null +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp @@ -0,0 +1,70 @@ +#include + +#include + +namespace DB +{ + +namespace +{ + /// The secret value of a `key = value` argument is its second child; anything else carries the + /// secret in the node itself. + QueryTreeNodePtr & secretValueSlot(QueryTreeNodePtr & node) + { + if (auto * function_node = node->as(); + function_node && function_node->getFunctionName() == "equals" && function_node->getArguments().getNodes().size() == 2) + return function_node->getArguments().getNodes()[1]; + return node; + } + + /// Whether a nested secret map child is a `key = value` node whose value stays visible when the + /// map is masked (the non-secret identifiers of `extra_credentials`; `headers` values are all hidden). + bool isNonSecretMapChild(const String & map_name, const QueryTreeNodePtr & node) + { + if (map_name != "extra_credentials") + return false; + const auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "equals" || function_node->getArguments().getNodes().size() != 2) + return false; + /// Keep the value visible only when it is a plain literal or identifier; a non-literal value + /// (e.g. `role_arn = headers('Authorization' = '...')`) can hide a nested secret, so fail closed. + const auto & value_node = function_node->getArguments().getNodes()[1]; + if (!value_node->as() && !value_node->as()) + return false; + const auto & key_node = function_node->getArguments().getNodes()[0]; + if (const auto * key_constant = key_node->as()) + return key_constant->getValue().getType() == Field::Types::String + && FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(key_constant->getValue().safeGet()); + if (const auto * key_identifier = key_node->as()) + return FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(key_identifier->getIdentifier().getFullName()); + return false; + } +} + +void forEachSecretArgumentNode( + QueryTreeNodes & arguments, + const FunctionSecretArgumentsFinder::Result & secret_arguments, + const std::function & on_secret) +{ + for (size_t n = 0; n < arguments.size(); ++n) + { + if (auto * function_node = arguments[n]->as(); + function_node + && std::find(secret_arguments.nested_maps.begin(), secret_arguments.nested_maps.end(), function_node->getFunctionName()) + != secret_arguments.nested_maps.end()) + { + for (auto & inner : function_node->getArguments().getNodes()) + { + if (!isNonSecretMapChild(function_node->getFunctionName(), inner)) + on_secret(n, secretValueSlot(inner)); + } + continue; + } + + const bool in_span = secret_arguments.start <= n && n < secret_arguments.start + secret_arguments.count; + if (in_span || secret_arguments.masked_arguments.contains(n) || secret_arguments.replaced_arguments.contains(n)) + on_secret(n, secretValueSlot(arguments[n])); + } +} + +} diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h index 8bcb6e147420..6afdd5d3a952 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h @@ -1,5 +1,8 @@ #pragma once +#include + +#include #include #include #include @@ -63,6 +66,15 @@ class FunctionTreeNodeImpl : public AbstractFunction return false; } + bool tryGetLiteralText(String * res) const override + { + const auto * literal = argument->as(); + if (!literal) + return false; + if (res) + *res = applyVisitor(FieldVisitorToString(), literal->getValue()); + return true; + } private: const IQueryTreeNode * argument = nullptr; }; @@ -109,4 +121,17 @@ class FunctionSecretArgumentsFinderTreeNodeImpl : public FunctionSecretArguments using FunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeNodeImpl; using TableFunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeNodeImpl; +/// Visits the secret value slots selected by a finder result in a resolved argument list, for the +/// query-tree surfaces (`EXPLAIN QUERY TREE`, projection names): the span members and the arguments +/// with a partial replacement (for a `key = value` argument, its value; a tree dump cannot represent +/// partial masking, so the whole value is masked: fail closed), and the values of the nested secret +/// maps (`headers(..)` / `extra_credentials(..)`; a malformed child is visited whole). The callback +/// receives the top-level argument index and a mutable reference to each secret value node, so it can +/// mask a constant in place or replace a non-constant node entirely (the parsers accept identifiers +/// and expressions as values, which have no display mask of their own). +void forEachSecretArgumentNode( + QueryTreeNodes & arguments, + const FunctionSecretArgumentsFinder::Result & secret_arguments, + const std::function & on_secret); + } diff --git a/src/Analyzer/Resolve/resolveFunction.cpp b/src/Analyzer/Resolve/resolveFunction.cpp index 306a23c8c9b8..0d038372238f 100644 --- a/src/Analyzer/Resolve/resolveFunction.cpp +++ b/src/Analyzer/Resolve/resolveFunction.cpp @@ -1089,19 +1089,49 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi /// Mask arguments if needed if (!scope.context->getSettingsRef()[Setting::format_display_secrets_in_show_and_select]) { - if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.count) + if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.hasSecrets()) { auto & argument_nodes = function_node_ptr->getArgumentsNode()->as().getNodes(); - for (size_t n = secret_arguments.start; n < secret_arguments.start + secret_arguments.count; ++n) + /// This tree is used for execution, so the value itself cannot be rewritten; only the + /// display mask of its constants can be set. `setMaskId` is a display flag, so it hides + /// the literal in projection names, `EXPLAIN QUERY TREE` and the `EXPLAIN actions = 1` + /// ActionsDAG (see PlannerActionsVisitor) without changing what is executed. + auto assign_mask = [&](ConstantNode & constant) { - if (auto * constant = argument_nodes[n]->as()) + auto mask = scope.projection_mask_map->insert({constant.getTreeHash(), scope.projection_mask_map->size() + 1}).first->second; + constant.setMaskId(mask); + return mask; + }; + /// A secret value can be an expression, not a bare literal (e.g. an `encrypt` key built as + /// `leftPad('...', 16, '*')`, including one inlined from a SQL UDF body). Hide every + /// constant inside it so no fragment of the secret leaks; returns whether any literal was + /// hidden. A slot that carries no literal (a plaintext like `toString(number)` or a key + /// held in a column) exposes nothing in the query text, so it is left as is. + std::function mask_secret_constants = [&](const QueryTreeNodePtr & subtree) + { + if (auto * constant = subtree->as()) { - auto mask = scope.projection_mask_map->insert({constant->getTreeHash(), scope.projection_mask_map->size() + 1}).first->second; - constant->setMaskId(mask); - arguments_projection_names[n] = "[HIDDEN id: " + std::to_string(mask) + "]"; + assign_mask(*constant); + return true; } - } + bool masked_any = false; + for (const auto & child : subtree->getChildren()) + if (child) + masked_any |= mask_secret_constants(child); + return masked_any; + }; + + forEachSecretArgumentNode( + argument_nodes, + secret_arguments, + [&](size_t n, QueryTreeNodePtr & secret_node) + { + if (auto * constant = secret_node->as()) + arguments_projection_names[n] = "[HIDDEN id: " + std::to_string(assign_mask(*constant)) + "]"; + else if (mask_secret_constants(secret_node)) + arguments_projection_names[n] = "[HIDDEN]"; + }); } } diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp index 2471c2f20f8c..5fb176ec58f3 100644 --- a/src/Databases/DatabaseS3.cpp +++ b/src/Databases/DatabaseS3.cpp @@ -43,7 +43,6 @@ static const std::unordered_set optional_configuration_keys = namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int UNKNOWN_TABLE; extern const int BAD_ARGUMENTS; extern const int FILE_DOESNT_EXIST; @@ -61,17 +60,6 @@ DatabaseS3::DatabaseS3(const String & name_, const Configuration& config_, Conte { } -void DatabaseS3::addTable(const std::string & table_name, StoragePtr table_storage) const -{ - std::lock_guard lock(mutex); - auto [_, inserted] = loaded_tables.emplace(table_name, table_storage); - if (!inserted) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Table with name `{}` already exists in database `{}` (engine {})", - table_name, getDatabaseName(), getEngineName()); -} - std::string DatabaseS3::getFullUrl(const std::string & name) const { if (!config.url_prefix.empty()) @@ -98,23 +86,14 @@ bool DatabaseS3::checkUrl(const std::string & url, ContextPtr context_, bool thr bool DatabaseS3::isTableExist(const String & name, ContextPtr context_) const { - std::lock_guard lock(mutex); - if (loaded_tables.contains(name)) - return true; - return checkUrl(getFullUrl(name), context_, false); } StoragePtr DatabaseS3::getTableImpl(const String & name, ContextPtr context_) const { - /// Check if the table exists in the loaded tables map. - { - std::lock_guard lock(mutex); - auto it = loaded_tables.find(name); - if (it != loaded_tables.end()) - return it->second; - } - + /// Not cached across sessions: the built S3 client depends on the per-session credential + /// restriction, so reuse could let an allowed session prime a client for a restricted one. + /// Rebuild every time. auto url = getFullUrl(name); checkUrl(url, context_, /* throw_on_error */true); @@ -139,11 +118,8 @@ StoragePtr DatabaseS3::getTableImpl(const String & name, ContextPtr context_) co return nullptr; /// TableFunctionS3 throws exceptions, if table cannot be created. - auto table_storage = table_function->execute(function, context_, name, /*cached_columns_=*/{}, /*use_global_context=*/false, /*is_insert_query=*/true); - if (table_storage) - addTable(name, table_storage); - - return table_storage; + /// Intentionally not cached -- see the note above. + return table_function->execute(function, context_, name, /*cached_columns_=*/{}, /*use_global_context=*/false, /*is_insert_query=*/true); } StoragePtr DatabaseS3::getTable(const String & name, ContextPtr context_) const @@ -182,8 +158,7 @@ StoragePtr DatabaseS3::tryGetTable(const String & name, ContextPtr context_) con bool DatabaseS3::empty() const { - std::lock_guard lock(mutex); - return loaded_tables.empty(); + return true; } ASTPtr DatabaseS3::getCreateDatabaseQueryImpl() const @@ -213,20 +188,6 @@ ASTPtr DatabaseS3::getCreateDatabaseQueryImpl() const void DatabaseS3::shutdown() { - Tables tables_snapshot; - { - std::lock_guard lock(mutex); - tables_snapshot = loaded_tables; - } - - for (const auto & kv : tables_snapshot) - { - auto table_id = kv.second->getStorageID(); - kv.second->flushAndShutdown(); - } - - std::lock_guard lock(mutex); - loaded_tables.clear(); } DatabaseS3::Configuration DatabaseS3::parseArguments(ASTs engine_args, ContextPtr context_) diff --git a/src/Databases/DatabaseS3.h b/src/Databases/DatabaseS3.h index 65737a226fbe..ebc4c8c2f16c 100644 --- a/src/Databases/DatabaseS3.h +++ b/src/Databases/DatabaseS3.h @@ -62,8 +62,6 @@ class DatabaseS3 : public IDatabase, protected WithContext ASTPtr getCreateDatabaseQueryImpl() const override TSA_REQUIRES(mutex); StoragePtr getTableImpl(const String & name, ContextPtr context) const; - void addTable(const std::string & table_name, StoragePtr table_storage) const; - bool checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const; std::string getFullUrl(const std::string & name) const; @@ -71,7 +69,6 @@ class DatabaseS3 : public IDatabase, protected WithContext private: const Configuration config; - mutable Tables loaded_tables TSA_GUARDED_BY(mutex); LoggerPtr log; }; diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index eae99d8e942c..c866f76ad3f8 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -327,7 +327,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column) return addNode(std::move(node)); } -const ActionsDAG::Node & ActionsDAG::addColumn(ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic_constant) +const ActionsDAG::Node & ActionsDAG::addColumn(ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic_constant, bool is_masked_secret) { if (!column) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add column {} because it is nullptr", name); @@ -343,6 +343,7 @@ const ActionsDAG::Node & ActionsDAG::addColumn(ColumnConstPtr column, DataTypePt node.result_name = std::move(name); node.column = std::move(column); node.is_deterministic_constant = is_deterministic_constant; + node.is_masked_secret = is_masked_secret; return addNode(std::move(node)); } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 50cfc296252a..f6924a37a53d 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -107,6 +107,10 @@ class ActionsDAG /// It is a constant calculated from deterministic functions (See IFunction::isDeterministic). /// This property is kept after constant folding of non-deterministic functions like 'now', 'today'. bool is_deterministic_constant = true; + /// Display-only: this constant holds a secret (e.g. an `encrypt` key). The value stays in + /// `column` so the query still executes, but plan dumps must render `[HIDDEN]` instead of it. + /// Not part of the node identity, so it is intentionally excluded from `updateHash`. + bool is_masked_secret = false; /// For COLUMN node and propagated constants. Always ColumnConst of size 0. ColumnConstPtr column; @@ -164,7 +168,7 @@ class ActionsDAG const Node & addInput(std::string name, DataTypePtr type); const Node & addInput(ColumnWithTypeAndName column); - const Node & addColumn(ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic_constant = true); + const Node & addColumn(ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic_constant = true, bool is_masked_secret = false); const Node & addAlias(const Node & child, std::string alias); const Node & addArrayJoin(const Node & child, std::string result_name); const Node & addFunction( diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 4d7ae3a0858d..9abda5bbeaf6 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -229,43 +229,69 @@ namespace using ExplainAnalyzedSyntaxVisitor = InDepthNodeVisitor; - class TableFunctionSecretsVisitor : public InDepthQueryTreeVisitor + /// Recursively hide every constant inside a secret argument, preserving the expression structure + /// (e.g. an `encrypt` key built as `leftPad('...', 16, '*')`). Constants already masked by + /// `resolveFunction` are left untouched so their mask ids survive; the rest are hidden here for the + /// dump-only path where the analysis passes did not run (`run_passes = 0`). + void maskConstantsInSubtree(QueryTreeNodePtr & node) + { + if (auto * constant = node->as()) + { + if (!constant->isMasked()) + constant->setMaskId(); + return; + } + for (auto & child : node->getChildren()) + if (child) + maskConstantsInSubtree(child); + } + + class SecretArgumentsDumpVisitor : public InDepthQueryTreeVisitor { friend class InDepthQueryTreeVisitor; - bool needChildVisit(VisitQueryTreeNodeType & parent [[maybe_unused]], VisitQueryTreeNodeType & child [[maybe_unused]]) + static bool needChildVisit(VisitQueryTreeNodeType &, VisitQueryTreeNodeType &) { - QueryTreeNodeType type = parent->getNodeType(); - return type == QueryTreeNodeType::QUERY || type == QueryTreeNodeType::JOIN || type == QueryTreeNodeType::TABLE_FUNCTION; + /// A secret-bearing function can hide under any carrier (a `UNION`, a scalar subquery, an + /// expression list), so descend everywhere; `visitImpl` selects the ones to mask. + return true; } void visitImpl(VisitQueryTreeNodeType & query_tree_node) { - auto * table_function_node_ptr = query_tree_node->as(); - if (!table_function_node_ptr) - return; - - if (FunctionSecretArgumentsFinder::Result secret_arguments = TableFunctionSecretArgumentsFinderTreeNode(*table_function_node_ptr).getResult(); secret_arguments.count) + if (auto * table_function_node = query_tree_node->as()) { - auto & argument_nodes = table_function_node_ptr->getArguments().getNodes(); - - for (size_t n = secret_arguments.start; n < secret_arguments.start + secret_arguments.count; ++n) - { - ConstantNode * constant_node = nullptr; - if (secret_arguments.are_named) - { - auto * function_node = argument_nodes[n]->as(); - if (function_node && function_node->getArguments().getNodes().size() >= 2) - constant_node = function_node->getArguments().getNodes().at(1)->as(); - } + auto secret_arguments = TableFunctionSecretArgumentsFinderTreeNode(*table_function_node).getResult(); + if (!secret_arguments.hasSecrets()) + return; - if (!constant_node) + /// A table-function secret value that is not a constant (an identifier or a constant + /// expression, e.g. a computed url) is hidden whole: the whole argument is the + /// credential carrier, and a tree dump cannot represent partial masking. Fail closed. + forEachSecretArgumentNode( + table_function_node->getArguments().getNodes(), + secret_arguments, + [](size_t, QueryTreeNodePtr & node) { - constant_node = argument_nodes[n]->as(); - } + if (auto * constant = node->as()) + constant->setMaskId(); + else + node = std::make_shared(Field("[HIDDEN]")); + }); + } + else if (auto * function_node = query_tree_node->as()) + { + auto secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node).getResult(); + if (!secret_arguments.hasSecrets()) + return; - if (constant_node) - constant_node->setMaskId(); - } + /// An ordinary secret function (`encrypt`/`decrypt`/`HMAC`, ...) is not masked by + /// `resolveFunction` when the dump runs with the analysis passes disabled. Its secret + /// is carried in constants (a literal key or one built by an expression), so hide every + /// constant inside the secret argument, keeping the structure visible. + forEachSecretArgumentNode( + function_node->getArguments().getNodes(), + secret_arguments, + [](size_t, QueryTreeNodePtr & node) { maskConstantsInSubtree(node); }); } } }; @@ -566,12 +592,6 @@ bool explainQueryTree( auto query_tree = buildQueryTree(explained_query, query_context); bool need_newline = false; - if (!query_context->getSettingsRef()[Setting::format_display_secrets_in_show_and_select]) - { - TableFunctionSecretsVisitor visitor; - visitor.visit(query_tree); - } - if (settings.run_passes) { auto query_tree_pass_manager = QueryTreePassManager(query_context); @@ -588,6 +608,15 @@ bool explainQueryTree( query_tree_pass_manager.run(query_tree, pass_index); } + /// Mask secrets only after the passes: the masked tree is used solely for the dump below, so + /// redaction (which may replace a non-constant secret value with a hidden constant) can never + /// change how the query is analyzed. With run_passes = 0 the tree is dumped without analysis. + if (!query_context->getSettingsRef()[Setting::format_display_secrets_in_show_and_select]) + { + SecretArgumentsDumpVisitor visitor; + visitor.visit(query_tree); + } + if (settings.dump_tree) { if (need_newline) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 6b9802aaf6bc..ef636913326a 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -240,6 +240,30 @@ ASTSelectWithUnionQuery * ASTFunction::tryGetQueryArgument() const } +/// Whether a nested secret map child is a `key = value` argument whose value stays visible when the +/// map is masked (the non-secret identifiers of `extra_credentials`; `headers` values are all hidden). +static bool isNonSecretMapChild(const String & map_name, const IAST * arg) +{ + if (map_name != "extra_credentials") + return false; + const auto * equals_func = arg->as(); + if (!equals_func || equals_func->name != "equals" || !equals_func->arguments || equals_func->arguments->children.size() != 2) + return false; + /// Keep the value visible only when it is a plain literal or identifier; a non-literal value (e.g. + /// `role_arn = headers('Authorization' = '...')`) can hide a nested secret and is formatted verbatim + /// before the parser rejects it, so fail closed. + const auto & value_ast = equals_func->arguments->children[1]; + if (!value_ast->as() && !value_ast->as()) + return false; + const auto & key_ast = equals_func->arguments->children[0]; + if (const auto * key_literal = key_ast->as()) + return key_literal->value.getType() == Field::Types::String + && FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(key_literal->value.safeGet()); + if (const auto * key_identifier = key_ast->as()) + return FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(key_identifier->name()); + return false; +} + static bool formatNamedArgWithHiddenValue(IAST * arg, WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) { const auto * equals_func = arg->as(); @@ -809,15 +833,68 @@ void ASTFunction::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSetting if (!settings.show_secrets) { + /// An argument with a partially masked replacement (e.g. a presigned S3 URL whose + /// credential parameters are hidden but whose host and path are kept). + if (auto replaced = secret_arguments.replaced_arguments.find(i); replaced != secret_arguments.replaced_arguments.end()) + { + ostr << replaced->second; + continue; + } + + /// A nested secret map like `headers(..)` / `extra_credentials(..)` has its values + /// hidden but its keys kept. Checked before the secret-span branch below because such a + /// map can itself fall inside a named span, where it must not be formatted as `key = ...`. + const ASTFunction * function = argument->as(); + if (function && function->arguments && std::count(secret_arguments.nested_maps.begin(), secret_arguments.nested_maps.end(), function->name) != 0) + { + /// headers('foo' = '[HIDDEN]', 'bar' = '[HIDDEN]') + ostr << function->name << "("; + for (size_t j = 0; j < function->arguments->children.size(); ++j) + { + if (j != 0) + ostr << ", "; + auto inner_arg = function->arguments->children[j]; + /// Known non-secret identifiers keep their values; a child that is not + /// `key = value` cannot be split into a visible key and a hidden value and may + /// be the secret itself, so it fails closed and is hidden whole. + if (isNonSecretMapChild(function->name, inner_arg.get())) + inner_arg->format(ostr, settings, state, nested_dont_need_parens); + else if (!formatNamedArgWithHiddenValue(inner_arg.get(), ostr, settings, state, nested_dont_need_parens)) + ostr << "'[HIDDEN]'"; + } + ostr << ")"; + continue; + } + + /// An individually masked argument: for the named `key = value` form the key stays + /// visible; anything else (a positional secret, or a malformed argument swept in by + /// a fail-closed rule) is hidden whole. + if (auto masked = secret_arguments.masked_arguments.find(i); masked != secret_arguments.masked_arguments.end()) + { + const auto * func_ast = typeid_cast(argument.get()); + if (masked->second && func_ast && func_ast->name == "equals" && func_ast->arguments && func_ast->arguments->children.size() == 2) + { + func_ast->arguments->children[0]->format(ostr, settings, state, nested_dont_need_parens); + ostr << " = "; + } + ostr << "'[HIDDEN]'"; + continue; + } + if (secret_arguments.start <= i && i < secret_arguments.start + secret_arguments.count) { if (secret_arguments.are_named) { - if (const auto * func_ast = typeid_cast(argument.get())) + /// Print `key = ` only for a well-formed `key = value` argument. Anything else + /// swept into the named span (e.g. a positional literal between two named + /// secrets) may itself be the secret, so fail closed: emit only the hidden + /// marker below without echoing the argument. + const auto * func_ast = typeid_cast(argument.get()); + if (func_ast && func_ast->name == "equals" && func_ast->arguments && func_ast->arguments->children.size() == 2) + { func_ast->arguments->children[0]->format(ostr, settings, state, nested_dont_need_parens); - else - argument->format(ostr, settings, state, nested_dont_need_parens); - ostr << " = "; + ostr << " = "; + } } if (!secret_arguments.replacement.empty()) { @@ -838,23 +915,6 @@ void ASTFunction::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSetting break; /// All other arguments should also be hidden. continue; } - - const ASTFunction * function = argument->as(); - if (function && function->arguments && std::count(secret_arguments.nested_maps.begin(), secret_arguments.nested_maps.end(), function->name) != 0) - { - /// headers('foo' = '[HIDDEN]', 'bar' = '[HIDDEN]') - ostr << function->name << "("; - for (size_t j = 0; j < function->arguments->children.size(); ++j) - { - if (j != 0) - ostr << ", "; - auto inner_arg = function->arguments->children[j]; - if (!formatNamedArgWithHiddenValue(inner_arg.get(), ostr, settings, state, nested_dont_need_parens)) - inner_arg->format(ostr, settings, state, nested_dont_need_parens); - } - ostr << ")"; - continue; - } } nested_dont_need_parens.list_element_index = i; diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp new file mode 100644 index 000000000000..296ae4e80271 --- /dev/null +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -0,0 +1,1241 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace +{ + /// Masks credential material embedded in an S3 URL itself: the userinfo part and the values of + /// presigned-URL query parameters. The parameter set mirrors `BackupInfo::removeCredentialsFromS3URL` + /// (which strips the same fields from persisted backup metadata). Returns true if anything was masked. + bool maskS3URICredentials(String & url) + { + bool changed = false; + /// Greedy up to the last at-sign before the path, so a userinfo whose password itself + /// contains an at-sign is masked whole, not just up to the first one. + static re2::RE2 userinfo_pattern = "^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@"; + if (RE2::Replace(&url, userinfo_pattern, "\\1[HIDDEN]@")) + changed = true; + static re2::RE2 presign_pattern + = "([?&](?:AWSAccessKeyId|Signature|Expires|GoogleAccessId|X-Amz-[A-Za-z0-9\\-]*|X-Goog-[A-Za-z0-9\\-]*)=)[^&#]*"; + if (RE2::GlobalReplace(&url, presign_pattern, "\\1[HIDDEN]")) + changed = true; + return changed; + } +} + +void FunctionSecretArgumentsFinder::markSecretArgument(size_t index, bool argument_is_named) +{ + if (index >= function->arguments->size()) + return; + chassert(result.replacement.empty()); /// We shouldn't use replacement with masking other arguments + /// Each argument is masked individually: valid S3 syntax can interleave secrets with non-secret + /// arguments, which a contiguous span cannot represent without hiding the arguments in between. + /// A malformed query can mark the same index as both named and positional; the positional form + /// wins, hiding the argument whole (fail closed). + auto [it, inserted] = result.masked_arguments.emplace(index, argument_is_named); + if (!inserted) + it->second &= argument_is_named; +} + +void FunctionSecretArgumentsFinder::maskNestedSecretMaps() +{ + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + { + const auto f = function->arguments->at(i)->getFunction(); + if (!f) + continue; + const auto name = f->name(); + if ((name == "headers" || name == "extra_credentials") + && std::find(result.nested_maps.begin(), result.nested_maps.end(), name) == result.nested_maps.end()) + result.nested_maps.push_back(name); + } +} + +std::vector FunctionSecretArgumentsFinder::classifyS3Arguments(size_t start, bool positionals_allowed_after_named) +{ + maskNestedSecretMaps(); + + std::vector positional; + bool seen_named = false; + for (size_t i = start; i < function->arguments->size(); ++i) + { + if (const auto f = function->arguments->at(i)->getFunction()) + { + const auto name = f->name(); + if (name == "headers" || name == "extra_credentials") + continue; + if (name == "equals" && f->hasArguments() && f->arguments->size() == 2) + { + seen_named = true; + String key; + if (f->arguments->at(0)->tryGetString(&key, /* allow_identifier= */ true)) + { + if (std::find(std::begin(s3_secret_keys), std::end(s3_secret_keys), key) != std::end(s3_secret_keys)) + { + markSecretArgument(i, /* argument_is_named= */ true); + } + else if (key == "url") + { + /// A `url` override can itself carry credentials (userinfo, presign parameters). + String url; + if (f->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) + { + if (maskS3URICredentials(url)) + result.replaced_arguments[i] = "url = " + quoteString(url); + } + else + { + /// A url built from an expression can embed credentials in its pieces; + /// we cannot evaluate it here, so fail closed and hide the value. + markSecretArgument(i, /* argument_is_named= */ true); + } + } + else if (!f->arguments->at(1)->tryGetString(nullptr, /* allow_identifier= */ true) + && !f->arguments->at(1)->tryGetLiteralText(nullptr)) + { + /// A visible non-secret override (`format`, `structure`, `role_arn`, ...) whose + /// value is not a plain literal or identifier can be a nested secret carrier, + /// e.g. `format = headers('Authorization' = '...')`, formatted verbatim before + /// the parser rejects the non-literal value. Fail closed and hide the value. + markSecretArgument(i, /* argument_is_named= */ true); + } + } + else + { + /// The parsers evaluate the key as a constant expression, so it can name any secret + /// key. We cannot evaluate it here, so fail closed and hide the value (the key + /// expression itself stays visible; keys are not secrets). + markSecretArgument(i, /* argument_is_named= */ true); + } + continue; + } + } + if (seen_named && !positionals_allowed_after_named) + { + /// The parsers reject positional arguments after the first `key = value` argument, but the + /// query is logged before validation and the intended slot is unknowable; fail closed. + markSecretArgument(i); + continue; + } + positional.push_back(i); + } + return positional; +} + +void FunctionSecretArgumentsFinder::maskS3PositionalSecrets( + const std::vector & positional, size_t url_slot, bool with_structure) +{ + /// The parser (`S3StorageParsedArguments::fromAST`) selects the signature from the positional + /// `count` (the number of arguments from `url` on) and `with_structure`, disambiguating only + /// NOSIGN and format-vs-secret by looking at an argument's value. Across every signature the only + /// credential positionals are `secret_access_key` at slot 2 and `session_token` at slot 3, so we + /// reproduce the parser's per-count decision for just those two slots. + /// + /// Value tests fail closed: an unevaluable expression is not recognized as NOSIGN or a format, so + /// the slot that would then be a credential is masked. A query built from a computed format thus + /// loses that non-secret argument in the AST dump, which is safe. The query-tree path resolves such + /// expressions to constants first, so it classifies them exactly. + if (url_slot >= positional.size()) + return; + const size_t count = positional.size() - url_slot; + + auto value_is = [&](size_t slot, auto && predicate) -> bool + { + String value; + return url_slot + slot < positional.size() + && tryGetStringFromArgument(positional[url_slot + slot], &value) && predicate(value); + }; + auto is_nosign = [&](size_t slot) { return value_is(slot, [](const String & v) { return boost::iequals(v, "NOSIGN"); }); }; + auto is_format = [&](size_t slot) { return value_is(slot, [](const String & v) { return v == "auto" || KnownFormatNames::instance().exists(v); }); }; + + bool secret_access_key = false; /// slot 2 + bool session_token = false; /// slot 3 + switch (count) + { + case 0: case 1: case 2: /// url only, or url + format/NOSIGN + break; + case 3: + secret_access_key = !is_nosign(1) && !is_format(1); + break; + case 4: + secret_access_key = !is_nosign(1) && !(with_structure && is_format(1)); + session_token = secret_access_key && !is_format(3); + break; + case 5: + secret_access_key = !with_structure || !is_nosign(1); + session_token = secret_access_key && !is_format(3); + break; + case 6: + secret_access_key = true; + session_token = !with_structure || !is_format(3); + break; + default: /// count >= 7: access-key form only, both credential slots always present + secret_access_key = true; + session_token = true; + break; + } + + if (secret_access_key) + markSecretArgument(positional[url_slot + 2]); + if (session_token) + markSecretArgument(positional[url_slot + 3]); +} + +void FunctionSecretArgumentsFinder::maskS3PositionalsFrom(const std::vector & positional, size_t first_slot) +{ + for (size_t slot = first_slot; slot < positional.size(); ++slot) + markSecretArgument(positional[slot]); +} + +void FunctionSecretArgumentsFinder::maskS3UrlArgument(const std::vector & positional, size_t url_slot) +{ + if (url_slot >= positional.size()) + return; + String url; + if (!tryGetStringFromArgument(positional[url_slot], &url, /* allow_identifier= */ false)) + { + /// The parsers evaluate a constant-expression url before signature parsing, so a url built + /// from an expression can embed credentials in its pieces; we cannot evaluate it here, so + /// fail closed and hide it whole. + markSecretArgument(positional[url_slot]); + return; + } + if (maskS3URICredentials(url)) + result.replaced_arguments[positional[url_slot]] = quoteString(url); +} + +void FunctionSecretArgumentsFinder::findOrdinaryFunctionSecretArguments() +{ + if ((function->name() == "mysql") || (function->name() == "postgresql")) + { + /// mysql('host:port', 'database', 'table', 'user', 'password', ...) + /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) + /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) + findMySQLFunctionSecretArguments(); + } + else if (function->name() == "mongodb") + { + findMongoDBSecretArguments(); + } + else if ((function->name() == "s3") || (function->name() == "cosn") || (function->name() == "oss") || + (function->name() == "deltaLake") || (function->name() == "deltaLakeS3") || (function->name() == "hudi") || + (function->name() == "iceberg") || (function->name() == "gcs") || (function->name() == "icebergS3") || + (function->name() == "paimon") || (function->name() == "paimonS3")) + { + /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) + findS3FunctionSecretArguments(/* is_cluster_function= */ false); + } + else if ((function->name() == "s3Cluster") || (function ->name() == "hudiCluster") || + (function ->name() == "deltaLakeCluster") || (function ->name() == "deltaLakeS3Cluster") || + (function ->name() == "icebergS3Cluster") || (function ->name() == "icebergCluster") || + (function ->name() == "paimonCluster") || (function ->name() == "paimonS3Cluster")) + { + /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) + findS3FunctionSecretArguments(/* is_cluster_function= */ true); + } + else if ((function->name() == "azureBlobStorage") || (function->name() == "deltaLakeAzure") || + (function->name() == "icebergAzure") || (function->name() == "paimonAzure")) + { + /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) + findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); + } + else if ((function->name() == "azureBlobStorageCluster") || (function->name() == "icebergAzureCluster") || + (function->name() == "deltaLakeAzureCluster") || (function->name() == "paimonAzureCluster")) + { + /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) + findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); + } + else if ((function->name() == "remote") || (function->name() == "remoteSecure")) + { + /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) + findRemoteFunctionSecretArguments(); + } + else if ((function->name() == "encrypt") || (function->name() == "decrypt") || + (function->name() == "aes_encrypt_mysql") || (function->name() == "aes_decrypt_mysql") || + (function->name() == "tryDecrypt")) + { + /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) + findEncryptionFunctionSecretArguments(); + } + else if (boost::iequals(function->name(), "HMAC")) + { + /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') + findHMACSecretArguments(); + } + else if (function->name() == "url" || function->name() == "urlCluster") + { + /// url('url', ...) keeps the url at slot 0; urlCluster('cluster', 'url', ...) at slot 1. + findURLSecretArguments(function->name() == "urlCluster" ? 1 : 0); + } + else if (function->name() == "redis") + { + findRedisFunctionSecretArguments(); + } + else if (function->name() == "ytsaurus") + { + findYTsaurusStorageTableEngineSecretArguments(); + } + else if ((function->name() == "arrowFlight") || (function->name() == "arrowflight")) + { + findArrowFlightSecretArguments(); + } + else if ((function->name() == "jdbc") || (function->name() == "odbc")) + { + /// jdbc('DSN', schema, table) or jdbc('DSN', table) + /// odbc('DSN', schema, table) or odbc('DSN', table) + /// The DSN (connection string) may contain credentials. + findXDBCSecretArguments(); + } +} + +void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// mysql(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + } + else + { + /// mysql('host:port', 'database', 'table', 'user', 'password', ...) + markSecretArgument(4); + } +} + +void FunctionSecretArgumentsFinder::findMongoDBSecretArguments() +{ + String uri; + + if (isNamedCollectionName(0)) + { + /// MongoDB(named_collection, ..., password = 'password', ...) + if (findSecretNamedArgument("password", 1)) + return; + + /// MongoDB(named_collection, ..., uri = 'mongodb://username:password@127.0.0.1:27017', ...) + if (findNamedArgument(&uri, "uri", 1) == -1) + return; + + result.are_named = true; + result.start = 1; + } + else if (function->arguments->size() == 2) + { + tryGetStringFromArgument(0, &uri); + result.are_named = false; + result.start = 0; + } + else + { + // MongoDB('127.0.0.1:27017', 'database', 'collection', 'user, 'password'...) + markSecretArgument(4, false); + return; + } + + chassert(result.count == 0); + maskURIPassword(&uri); + result.count = 1; + result.replacement = std::move(uri); +} + +void FunctionSecretArgumentsFinder::findRedisTableEngineSecretArguments() +{ + /// Redis does not have URL/address argument, + /// only 'host:port' and separate "password" argument. + + if (isNamedCollectionName(0)) + { + if (findSecretNamedArgument("password", 1)) + return; + } + else + { + // Redis('host:port', 'db_index', 'password', 'pool_size') + markSecretArgument(2, false); + return; + } +} + +void FunctionSecretArgumentsFinder::findArrowFlightSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// ArrowFlight(named_collection, ..., password = 'password') + findSecretNamedArgument("password", 1); + } + else + { + /// ArrowFlight('host:port', 'dataset', 'username', 'password') + markSecretArgument(3); + } +} + +void FunctionSecretArgumentsFinder::findXDBCSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// jdbc(named_collection, ..., datasource = 'DSN', ...) + /// odbc(named_collection, ..., connection_settings = 'DSN', ...) + /// `datasource` and `connection_settings` are mutually exclusive aliases. + /// If the value is a URI, mask only the password; otherwise hide the whole value. + /// If somehow both are present (invalid query), hide all named arguments. + ssize_t ds_idx = findNamedArgument(nullptr, "datasource", 1); + ssize_t cs_idx = findNamedArgument(nullptr, "connection_settings", 1); + + if (ds_idx >= 0 && cs_idx >= 0) + { + /// Both present — hide all named arguments starting from index 1. + result.start = 1; + result.count = function->arguments->size() - 1; + result.are_named = true; + } + else if (ds_idx >= 0) + maskXDBCSecretNamedArgument("datasource", 1); + else if (cs_idx >= 0) + maskXDBCSecretNamedArgument("connection_settings", 1); + } + else + { + /// jdbc('DSN', schema, table) / jdbc('DSN', table) + /// odbc('DSN', schema, table) / odbc('DSN', table) + /// JDBC('DSN', database, table) / ODBC('DSN', database, table) + /// The connection string may be a URI with credentials embedded, + /// e.g. scheme://username:password@host:port/dbname + /// If so, mask only the password part; otherwise hide the whole argument. + String uri; + if (tryGetStringFromArgument(0, &uri)) + { + if (maskURIPassword(&uri)) + { + chassert(result.count == 0); + result.start = 0; + result.count = 1; + result.replacement = std::move(uri); + return; + } + } + markSecretArgument(0, false); + } +} + +void FunctionSecretArgumentsFinder::maskXDBCSecretNamedArgument(std::string_view key, size_t start) +{ + String value; + ssize_t arg_idx = findNamedArgument(&value, key, start); + if (arg_idx < 0) + return; + + if (!value.empty() && maskURIPassword(&value)) + { + result.are_named = true; + result.start = arg_idx; + result.count = 1; + result.replacement = std::move(value); + } + else + { + markSecretArgument(arg_idx, /* argument_is_named= */ true); + } +} + +void FunctionSecretArgumentsFinder::findS3FunctionSecretArguments(bool is_cluster_function) +{ + /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. + size_t url_slot = is_cluster_function ? 1 : 0; + + if (isNamedCollectionName(url_slot)) + { + /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...) + /// s3Cluster('cluster_name', named_collection, ..., secret_access_key = 'secret_access_key', ...) + findS3NamedCollectionSecretArguments(url_slot + 1); + return; + } + + const auto positional = classifyS3Arguments(); + maskS3UrlArgument(positional, url_slot); + + /// The table function accepts a positional `structure`, unless a `structure = ...` named override + /// is given (the parser then turns `with_structure` off). The parser evaluates key expressions, so + /// an unevaluable key might resolve to `structure`; treat any unreadable key as disabling it too. + /// This fails closed: `with_structure = false` only ever masks the same slots or more. + bool with_structure = true; + for (size_t i = 0; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (!equals_func || equals_func->name() != "equals" || !equals_func->hasArguments() || equals_func->arguments->size() != 2) + continue; + String key; + if (!equals_func->arguments->at(0)->tryGetString(&key, /* allow_identifier= */ true) || key == "structure") + { + with_structure = false; + break; + } + } + maskS3PositionalSecrets(positional, url_slot, with_structure); +} + +void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function) +{ + /// azureBlobStorageCluster('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. + size_t url_arg_idx = is_cluster_function ? 1 : 0; + + if (!is_cluster_function && isNamedCollectionName(0)) + { + /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + if (maskAzureConnectionString(-1, true, 1)) + return; + findSecretNamedArgument("account_key", 1); + return; + } + if (is_cluster_function && isNamedCollectionName(1)) + { + /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) + if (maskAzureConnectionString(-1, true, 2)) + return; + findSecretNamedArgument("account_key", 2); + return; + } + + if (maskAzureConnectionString(url_arg_idx)) + return; + + /// We should check other arguments first because we don't need to do any replacement in case of + /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument + /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument + size_t count = function->arguments->size(); + if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) + { + String fourth_arg; + if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) + { + if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) + return; + } + } + + /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature + if (url_arg_idx + 4 < count) + markSecretArgument(url_arg_idx + 4); +} + +bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named, size_t start) +{ + String url_arg; + if (argument_is_named) + { + url_arg_idx = findNamedArgument(&url_arg, "connection_string", start); + if (url_arg_idx == -1 || url_arg.empty()) + url_arg_idx = findNamedArgument(&url_arg, "storage_account_url", start); + if (url_arg_idx == -1 || url_arg.empty()) + return false; + } + else + { + if (!tryGetStringFromArgument(url_arg_idx, &url_arg)) + return false; + } + + if (!url_arg.starts_with("http")) + { + static re2::RE2 account_key_pattern = "AccountKey=.*?(;|$)"; + if (RE2::Replace(&url_arg, account_key_pattern, "AccountKey=[HIDDEN]\\1")) + { + chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments + result.start = url_arg_idx; + result.are_named = argument_is_named; + result.count = 1; + result.replacement = url_arg; + return true; + } + + static re2::RE2 sas_signature_pattern = "SharedAccessSignature=.*?(;|$)"; + if (RE2::Replace(&url_arg, sas_signature_pattern, "SharedAccessSignature=[HIDDEN]\\1")) + { + chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments + result.start = url_arg_idx; + result.are_named = argument_is_named; + result.count = 1; + result.replacement = url_arg; + return true; + } + } + + return false; +} + +void FunctionSecretArgumentsFinder::findURLSecretArguments(size_t url_offset) +{ + /// `headers(...)` can appear at any position in every url form (function, cluster function, engine, + /// and the named-collection variant); mask its values regardless of the url offset or a leading + /// collection/cluster argument. + maskNestedSecretMaps(); + + if (isNamedCollectionName(url_offset)) + { + /// url(named_collection, url = 'https://user:password@host/...', headers(...), ...): mask the + /// userinfo password of a `url` override. The parser evaluates constant-expression keys and + /// values, so fail closed on anything we cannot read as a plain literal (a nested `headers(...)` + /// map or other expression could carry a secret): an unevaluable key can name `url`, and any + /// non-literal value of a visible override can hide a nested secret. The headers are handled + /// above; a `key = value` override is the only other shape here. + for (size_t i = url_offset + 1; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (!equals_func || equals_func->name() != "equals" || !equals_func->hasArguments() + || equals_func->arguments->size() != 2) + continue; + + String key; + if (!equals_func->arguments->at(0)->tryGetString(&key, /* allow_identifier= */ true)) + { + markSecretArgument(i, /* argument_is_named= */ true); + } + else if (key == "url") + { + String url; + if (equals_func->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) + { + if (maskURIPassword(&url)) + result.replaced_arguments[i] = "url = " + quoteString(url); + } + else + markSecretArgument(i, /* argument_is_named= */ true); + } + else if (!equals_func->arguments->at(1)->tryGetString(nullptr, /* allow_identifier= */ true) + && !equals_func->arguments->at(1)->tryGetLiteralText(nullptr)) + { + markSecretArgument(i, /* argument_is_named= */ true); + } + } + return; + } + + String uri; + if (tryGetStringFromArgument(url_offset, &uri, /* allow_identifier= */ false)) + { + /// A readable url literal: mask only its userinfo password, keeping the host and path visible. + if (maskURIPassword(&uri)) + result.replaced_arguments[url_offset] = quoteString(uri); + } + else + { + /// A url built from a constant expression can embed credentials in its pieces, which we cannot + /// evaluate here; hide it whole rather than leak (fail closed). + markSecretArgument(url_offset); + } +} + +bool FunctionSecretArgumentsFinder::tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier) const +{ + if (arg_idx >= function->arguments->size()) + return false; + + return tryGetStringFromArgument(*function->arguments->at(arg_idx), res, allow_identifier); +} + +bool FunctionSecretArgumentsFinder::tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier) +{ + return argument.tryGetString(res, allow_identifier); +} + +void FunctionSecretArgumentsFinder::findRemoteFunctionSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// remote(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + return; + } + + /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: + /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) + /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key]) + /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key]) + + /// But we should check the number of arguments first because we don't need to do any replacements in case of + /// remote('addresses_expr', db.table) + if (function->arguments->size() < 3) + return; + + size_t arg_num = 1; + + /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'. + auto table_function = function->arguments->at(arg_num)->getFunction(); + if (table_function && KnownTableFunctionNames::instance().exists(table_function->name())) + { + ++arg_num; + } + else + { + std::optional database; + std::optional qualified_table_name; + if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name)) + { + /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'. + /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user' + /// before the argument 'password'. So it's safer to wipe two arguments just in case. + /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string + /// before wiping it (because the `password` argument is always a literal string). + if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false)) + { + /// Wipe either `password` or `user`. + markSecretArgument(arg_num + 2); + } + if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false)) + { + /// Wipe either `password` or `sharding_key`. + markSecretArgument(arg_num + 3); + } + return; + } + + /// Skip the current argument (which is either a database name or a qualified table name). + ++arg_num; + if (database) + { + /// Skip the 'table' argument if the previous argument was a database name. + ++arg_num; + } + } + + /// Skip username. + ++arg_num; + + /// Do our replacement: + /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...) + /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string + /// before wiping it (because the `password` argument is always a literal string). + bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false); + if (can_be_password) + markSecretArgument(arg_num); +} + +bool FunctionSecretArgumentsFinder::tryGetDatabaseNameOrQualifiedTableName( + size_t arg_idx, + std::optional & res_database, + std::optional & res_qualified_table_name) const +{ + res_database.reset(); + res_qualified_table_name.reset(); + + String str; + if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true)) + return false; + + if (str.empty()) + { + res_database = ""; + return true; + } + + auto qualified_table_name = QualifiedTableName::tryParseFromString(str); + if (!qualified_table_name) + return false; + + if (qualified_table_name->database.empty()) + res_database = std::move(qualified_table_name->table); + else + res_qualified_table_name = std::move(qualified_table_name); + return true; +} + +void FunctionSecretArgumentsFinder::findEncryptionFunctionSecretArguments() +{ + if (function->arguments->size() == 0) + return; + + /// We replace all arguments after 'mode' with '[HIDDEN]': + /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]') + result.start = 1; + result.count = function->arguments->size() - 1; +} + +void FunctionSecretArgumentsFinder::findHMACSecretArguments() +{ + if (function->arguments->size() < 3) + return; + + /// We hide the key argument and any following for the case of mistyping or using extra arguments by mistake: + /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') + /// HMAC('sha256', toString(toFixedString('b', 3), 3), '(', 'this_should_be_secret') -> HMAC('sha256', toString(toFixedString('b', 3), 3), '[HIDDEN]', '[HIDDEN]') + result.start = 2; + result.count = function->arguments->size() - 2; +} + +void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() +{ + const String & engine_name = function->name(); + if (engine_name == "ExternalDistributed") + { + /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') + findExternalDistributedTableEngineSecretArguments(); + } + else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") || (engine_name == "MaterializedPostgreSQL")) + { + /// MySQL('host:port', 'database', 'table', 'user', 'password', ...) + /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) + /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) + /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) + findMySQLFunctionSecretArguments(); + } + else if (engine_name == "MongoDB") + { + findMongoDBSecretArguments(); + } + else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || (engine_name == "GCS") + || (engine_name == "DeltaLake") || (engine_name == "DeltaLakeS3") || (engine_name == "Hudi") + || (engine_name == "Iceberg") || (engine_name == "IcebergS3") + || (engine_name == "Paimon") || (engine_name == "PaimonS3") + || (engine_name == "S3Queue")) + { + /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) + findS3TableEngineSecretArguments(); + } + else if (engine_name == "URL") + { + findURLSecretArguments(); + } + else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") + { + findAzureBlobStorageTableEngineSecretArguments(); + } + else if (engine_name == "Redis") + { + findRedisTableEngineSecretArguments(); + } + else if (engine_name == "YTsaurus") + { + findYTsaurusStorageTableEngineSecretArguments(); + } + else if (engine_name == "ArrowFlight") + { + findArrowFlightSecretArguments(); + } + else if ((engine_name == "Remote") || (engine_name == "RemoteSecure")) + { + /// Remote('addresses_expr', db, table, 'user', 'password', ...) + /// RemoteSecure(...) - same as Remote(...) + /// The arguments are identical to the `remote`/`remoteSecure` table functions, so reuse + /// the same finder (it also handles the named-collection form `Remote(named_collection, ...)`). + findRemoteFunctionSecretArguments(); + } + else if ((engine_name == "JDBC") || (engine_name == "ODBC")) + { + /// JDBC('DSN', database, table) + /// ODBC('DSN', database, table) + /// The DSN (connection string) may contain credentials. + findXDBCSecretArguments(); + } +} + +void FunctionSecretArgumentsFinder::findExternalDistributedTableEngineSecretArguments() +{ + if (isNamedCollectionName(1)) + { + /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 2); + } + else + { + /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') + markSecretArgument(5); + } +} + +void FunctionSecretArgumentsFinder::findS3TableEngineSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// S3(named_collection, ..., secret_access_key = 'secret_access_key') + findS3NamedCollectionSecretArguments(1); + return; + } + + const auto positional = classifyS3Arguments(); + maskS3UrlArgument(positional, 0); + + /// The table engine takes its structure from the column list, never as an argument. + maskS3PositionalSecrets(positional, 0, /* with_structure= */ false); +} + +void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArguments() +{ + /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format, [account_name, account_key, ...]) + size_t url_arg_idx = 0; + + if (isNamedCollectionName(url_arg_idx)) + { + /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + if (maskAzureConnectionString(-1, true, 1)) + return; + findSecretNamedArgument("account_key", 1); + return; + } + + if (maskAzureConnectionString(url_arg_idx)) + return; + + /// We should check other arguments first because we don't need to do any replacement in case of + /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument + size_t count = function->arguments->size(); + if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) + { + String fourth_arg; + if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) + { + if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) + return; + } + } + + /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature + if (url_arg_idx + 4 < count) + markSecretArgument(url_arg_idx + 4); +} + +void FunctionSecretArgumentsFinder::findRedisFunctionSecretArguments() +{ + // redis(host:port, key, structure, db_index, password, pool_size) + markSecretArgument(4); +} + +void FunctionSecretArgumentsFinder::findYTsaurusStorageTableEngineSecretArguments() +{ + // YTsaurus('base_uri', 'yt_path', 'auth_token') + markSecretArgument(2); +} + +void FunctionSecretArgumentsFinder::findDatabaseEngineSecretArguments() +{ + const String & engine_name = function->name(); + if (engine_name == "MySQL" || + engine_name == "PostgreSQL" || + engine_name == "MaterializedPostgreSQL") + { + /// MySQL('host:port', 'database', 'user', 'password') + /// PostgreSQL('host:port', 'database', 'user', 'password') + findMySQLDatabaseSecretArguments(); + } + else if (engine_name == "S3") + { + /// S3('url', 'access_key_id', 'secret_access_key') + findS3DatabaseSecretArguments(); + } + else if (engine_name == "DataLakeCatalog") + { + findDataLakeCatalogSecretArguments(); + } + else if (engine_name == "Backup") + { + findBackupDatabaseSecretArguments(); + } +} + +void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// MySQL(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + } + else + { + /// MySQL('host:port', 'database', 'user', 'password') + markSecretArgument(3); + } +} + +void FunctionSecretArgumentsFinder::findS3DatabaseSecretArguments() +{ + if (isNamedCollectionName(0)) + { + /// S3(named_collection, ..., secret_access_key = 'password', ...) + findS3NamedCollectionSecretArguments(1); + } + else + { + /// S3('url', 'access_key_id', 'secret_access_key' [, session_token = ..., google_adc_* = ...]): + /// the engine accepts no positional argument beyond secret_access_key, so fail closed from + /// slot 2 on. Non-secret named overrides (e.g. `use_environment_credentials = 1`) stay visible. + const auto positional = classifyS3Arguments(); + maskS3UrlArgument(positional, 0); + maskS3PositionalsFrom(positional, 2); + } +} + +void FunctionSecretArgumentsFinder::findDataLakeCatalogSecretArguments() +{ + /// datalake catalog should support different storage types, + /// we need a function to check if the url is S3 or Azure. + /// right now we assume it's a S3 url + findS3DatabaseSecretArguments(); +} + +void FunctionSecretArgumentsFinder::findBackupDatabaseSecretArguments() +{ + if (function->arguments->size() < 2) + return; + + auto storage_arg = function->arguments->at(1); + auto storage_function = storage_arg->getFunction(); + + /// The nested S3 destination is not recognized as an S3 engine when the formatter recurses into it, + /// so its secrets must be masked here. Handle both forms: + /// Backup('', S3('url', 'access_key_id', 'secret_access_key' [, ...])) + /// Backup('', S3(named_collection, ..., secret_access_key = '...', session_token = '...', ...)) + /// by reconstructing the nested `S3(...)` with the secret arguments replaced by `[HIDDEN]`. + if (!storage_function || storage_function->name() != "S3" || !storage_function->hasArguments()) + return; + + const auto & nested_args = *storage_function->arguments; + const bool is_named_collection = nested_args.size() >= 1 && nested_args.at(0)->isIdentifier(); + + /// Count the positional arguments first (everything that is not `key = value` or a nested map): + /// the visibility rule below depends on the total, mirroring `BackupInfo::fromAST`, which collects + /// positionals independently of named overrides. + size_t total_positionals = 0; + for (size_t i = 0; i < nested_args.size(); ++i) + { + const auto f = nested_args.at(i)->getFunction(); + if (f && (f->name() == "extra_credentials" + || (f->name() == "equals" && f->hasArguments() && f->arguments->size() == 2))) + continue; + ++total_positionals; + } + + /// Named-collection locator: slot 0 is the collection and slot 1 the non-secret filename. + /// Explicit-url locator: valid signatures have one positional (the url) or three (url, + /// access_key_id, secret_access_key) with the secret at slot 2; any other count is invalid and + /// the intended slots are unknowable, so everything after the url is hidden (fail closed). + const size_t first_hidden_slot = (is_named_collection || total_positionals == 3) ? 2 : 1; + + std::string replacement = "S3("; + bool has_secret = false; + size_t positional_slot = 0; + for (size_t i = 0; i < nested_args.size(); ++i) + { + if (i > 0) + replacement += ", "; + + auto arg = nested_args.at(i); + + /// Named argument `key = value`. + if (auto key_value = arg->getFunction(); + key_value && key_value->name() == "equals" && key_value->hasArguments() && key_value->arguments->size() == 2) + { + String key; + if (key_value->arguments->at(0)->tryGetString(&key, /* allow_identifier= */ true)) + { + const bool is_secret = std::find(std::begin(s3_secret_keys), std::end(s3_secret_keys), key) != std::end(s3_secret_keys); + replacement += key; + replacement += " = "; + String value; + if (is_secret) + { + replacement += "'[HIDDEN]'"; + has_secret = true; + } + else if (key_value->arguments->at(1)->tryGetString(&value, /* allow_identifier= */ true)) + { + /// A `url` override can itself carry credentials (userinfo, presign parameters). + has_secret |= maskS3URICredentials(value); + replacement += quoteString(value); + } + else if (String literal_text; key_value->arguments->at(1)->tryGetLiteralText(&literal_text)) + { + /// A non-string scalar override, e.g. `use_environment_credentials = 1`. + replacement += literal_text; + } + else + { + /// Any remaining value is an expression, not a plain literal or identifier: a `url` + /// built from pieces, or a nested `headers(...)` / `extra_credentials(...)` map or + /// other function whose formatted text would carry its secrets verbatim (the parser + /// evaluates it as a constant, so it is not masked as a nested map here). We cannot + /// evaluate it, so hide it rather than leak. This counts as a secret: otherwise a + /// replacement whose only hidden part is this value would be discarded below and the + /// original expression formatted verbatim. + replacement += "'[HIDDEN]'"; + has_secret = true; + } + } + else + { + /// The key is a constant expression the parser would evaluate, so it can name any + /// secret key; fail closed and hide the whole argument. + replacement += "'[HIDDEN]'"; + has_secret = true; + } + continue; + } + + /// Nested `extra_credentials(k = v, ...)` map: reconstruct with every value hidden. Build into + /// a temporary; if any inner key is not a plain literal (e.g. a constant expression the parser + /// still accepts), fail closed by hiding the whole map rather than emitting it verbatim. + if (auto extra_credentials_func = arg->getFunction(); + extra_credentials_func && extra_credentials_func->name() == "extra_credentials" && extra_credentials_func->hasArguments()) + { + std::string masked_map = "extra_credentials("; + bool reconstructed = true; + const auto & cred_args = *extra_credentials_func->arguments; + for (size_t j = 0; j < cred_args.size(); ++j) + { + String cred_key; + auto cred_kv = cred_args.at(j)->getFunction(); + if (cred_kv && cred_kv->name() == "equals" && cred_kv->hasArguments() && cred_kv->arguments->size() == 2 + && cred_kv->arguments->at(0)->tryGetString(&cred_key, /* allow_identifier= */ true)) + { + if (j > 0) + masked_map += ", "; + String cred_value; + if (isNonSecretExtraCredentialsKey(cred_key) + && cred_kv->arguments->at(1)->tryGetString(&cred_value, /* allow_identifier= */ true)) + masked_map += cred_key + " = " + quoteString(cred_value); + else + masked_map += cred_key + " = '[HIDDEN]'"; + } + else + { + reconstructed = false; + break; + } + } + masked_map += ")"; + replacement += reconstructed ? masked_map : "'[HIDDEN]'"; + has_secret = true; + continue; + } + + /// Positional argument: the slot is counted over positionals only, and its visibility follows + /// the signature rule computed above. + const size_t slot = positional_slot++; + if (slot >= first_hidden_slot) + { + replacement += "'[HIDDEN]'"; + has_secret = true; + continue; + } + + String arg_value; + if (arg->isIdentifier() && arg->tryGetString(&arg_value, /* allow_identifier= */ true)) + replacement += arg_value; /// e.g. the named collection name, kept unquoted. + else if (arg->tryGetString(&arg_value, /* allow_identifier= */ true)) + { + /// The url positional can itself carry credentials (userinfo, presign parameters). + has_secret |= maskS3URICredentials(arg_value); + replacement += quoteString(arg_value); + } + else + { + /// Fail closed: an argument we cannot reconstruct safely (e.g. an unsupported tail like + /// `headers(..)`, or a non-literal expression) must not be emitted verbatim. Hide it. + replacement += "'[HIDDEN]'"; + has_secret = true; + } + } + replacement += ")"; + + if (!has_secret) + return; + + result.start = 1; + result.count = 1; + result.replacement = std::move(replacement); + result.quote_replacement = false; +} + +void FunctionSecretArgumentsFinder::findBackupNameSecretArguments() +{ + const String & engine_name = function->name(); + if (engine_name == "S3") + { + if (isNamedCollectionName(0)) + { + /// BACKUP ... TO S3(named_collection[, 'filename'], ..., secret_access_key = '...', ...): + /// unlike the other named-collection S3 forms, the backup locator accepts one positional + /// (the non-secret filename), in any position relative to the named overrides; anything + /// positional beyond it is invalid, so fail closed there. + maskS3PositionalsFrom(classifyS3Arguments(1, /* positionals_allowed_after_named= */ true), 1); + return; + } + /// BACKUP ... TO S3(url [, aws_access_key_id, aws_secret_access_key] [, session_token = ..., ...]): + /// the locator accepts exactly one or three positionals; the valid triple keeps the url and + /// access_key_id visible and hides the secret at slot 2. Any other positional count is invalid + /// but logged before validation, and the intended slots are unknowable, so fail closed on + /// everything after the url. + const auto positional = classifyS3Arguments(0, /* positionals_allowed_after_named= */ true); + maskS3UrlArgument(positional, 0); + maskS3PositionalsFrom(positional, positional.size() == 3 ? 2 : 1); + } + else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") + { + findAzureBlobStorageTableEngineSecretArguments(); + } +} + +bool FunctionSecretArgumentsFinder::isNamedCollectionName(size_t arg_idx) const +{ + if (function->arguments->size() <= arg_idx) + return false; + + return function->arguments->at(arg_idx)->isIdentifier(); +} + +ssize_t FunctionSecretArgumentsFinder::findNamedArgument(String * res, std::string_view key, size_t start) +{ + for (size_t i = start; i < function->arguments->size(); ++i) + { + const auto & argument = function->arguments->at(i); + const auto equals_func = argument->getFunction(); + if (!equals_func || (equals_func->name() != "equals")) + continue; + + if (!equals_func->arguments || equals_func->arguments->size() != 2) + continue; + + String found_key; + if (!tryGetStringFromArgument(*equals_func->arguments->at(0), &found_key)) + continue; + + if (found_key == key) + { + tryGetStringFromArgument(*equals_func->arguments->at(1), res); + return i; + } + } + + return -1; +} + +bool FunctionSecretArgumentsFinder::findSecretNamedArgument(std::string_view key, size_t start) +{ + bool found = false; + for (ssize_t arg_idx = findNamedArgument(nullptr, key, start); arg_idx >= 0; + arg_idx = findNamedArgument(nullptr, key, static_cast(arg_idx) + 1)) + { + markSecretArgument(arg_idx, /* argument_is_named= */ true); + found = true; + } + return found; +} + +void FunctionSecretArgumentsFinder::findS3NamedCollectionSecretArguments(size_t start) +{ + /// After the collection name every argument must be a named `option = value` override or a nested + /// map; a positional argument is invalid but logged before validation rejects it, so fail closed + /// and hide every positional the classification returns. + maskS3PositionalsFrom(classifyS3Arguments(start), 0); +} + +} diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index d5c490cd426f..4be5e2a8c44a 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -1,17 +1,19 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include -#include +#include namespace DB { +struct QualifiedTableName; + class AbstractFunction { friend class FunctionSecretArgumentsFinder; @@ -23,6 +25,9 @@ class AbstractFunction virtual std::unique_ptr getFunction() const = 0; virtual bool isIdentifier() const = 0; virtual bool tryGetString(String * res, bool allow_identifier) const = 0; + /// The exact literal text of any scalar literal (`1`, `true`, `1.5`), with strings quoted. + /// Lets a reconstructor keep non-string values like `use_environment_credentials = 1` visible. + virtual bool tryGetLiteralText(String * res) const = 0; }; class Arguments { @@ -59,10 +64,21 @@ class FunctionSecretArgumentsFinder std::string replacement; /// Whether to wrap a result using full argument replacement in quotes. bool quote_replacement = true; + /// Per-argument replacements by raw argument index; the text is emitted verbatim (it must carry + /// its own quoting). Used when only a part of an argument is secret, e.g. a presigned S3 URL + /// keeps its host and path while the credential query parameters are hidden. Unlike + /// `replacement`, this composes with the other masking (span, nested maps). + std::map replaced_arguments; + /// Individually masked arguments by raw argument index; the value tells whether the argument + /// is a named `key = value` (the key stays visible and only the value is hidden). Valid S3 + /// syntax can interleave secrets with non-secret arguments (e.g. a named `session_token` + /// after `format`), which a single contiguous span cannot represent without hiding the + /// non-secret arguments in between. + std::map masked_arguments; bool hasSecrets() const { - return count != 0 || !nested_maps.empty(); + return count != 0 || !nested_maps.empty() || !replaced_arguments.empty() || !masked_arguments.empty(); } }; @@ -70,513 +86,88 @@ class FunctionSecretArgumentsFinder FunctionSecretArgumentsFinder::Result getResult() const { return result; } -protected: - const std::unique_ptr function; - Result result; - - void markSecretArgument(size_t index, bool argument_is_named = false) - { - if (index >= function->arguments->size()) - return; - if (!result.count) - { - result.start = index; - result.are_named = argument_is_named; - } - chassert(result.replacement.empty()); /// We shouldn't use replacement with masking other arguments - /// Widen the masked range to cover `index`. Arguments are normally marked consecutively in - /// increasing order, but a malformed query can mix the named secret form (`key = ...`) with the - /// positional form and ask to mask an earlier index after a later one. Masking is best-effort - /// over arbitrary user input, so it must widen the range rather than assert on the order. - size_t end = std::max(result.start + result.count, index + 1); - result.start = std::min(result.start, index); - result.count = end - result.start; - if (!argument_is_named) - result.are_named = false; - } - - void findOrdinaryFunctionSecretArguments() - { - if ((function->name() == "mysql") || (function->name() == "postgresql")) - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) - /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); - } - else if (function->name() == "mongodb") - { - findMongoDBSecretArguments(); - } - else if ((function->name() == "s3") || (function->name() == "cosn") || (function->name() == "oss") || - (function->name() == "deltaLake") || (function->name() == "deltaLakeS3") || (function->name() == "hudi") || - (function->name() == "iceberg") || (function->name() == "gcs") || (function->name() == "icebergS3") || - (function->name() == "paimon") || (function->name() == "paimonS3")) - { - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ false); - } - else if ((function->name() == "s3Cluster") || (function ->name() == "hudiCluster") || - (function ->name() == "deltaLakeCluster") || (function ->name() == "deltaLakeS3Cluster") || - (function ->name() == "icebergS3Cluster") || (function ->name() == "icebergCluster") || - (function ->name() == "paimonCluster") || (function ->name() == "paimonS3Cluster")) - { - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ true); - } - else if ((function->name() == "azureBlobStorage") || (function->name() == "deltaLakeAzure") || - (function->name() == "icebergAzure") || (function->name() == "paimonAzure")) - { - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); - } - else if ((function->name() == "azureBlobStorageCluster") || (function->name() == "icebergAzureCluster") || - (function->name() == "deltaLakeAzureCluster") || (function->name() == "paimonAzureCluster")) - { - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); - } - else if ((function->name() == "remote") || (function->name() == "remoteSecure")) - { - /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) - findRemoteFunctionSecretArguments(); - } - else if ((function->name() == "encrypt") || (function->name() == "decrypt") || - (function->name() == "aes_encrypt_mysql") || (function->name() == "aes_decrypt_mysql") || - (function->name() == "tryDecrypt")) - { - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) - findEncryptionFunctionSecretArguments(); - } - else if (boost::iequals(function->name(), "HMAC")) - { - /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') - findHMACSecretArguments(); - } - else if (function->name() == "url") - { - findURLSecretArguments(); - } - else if (function->name() == "redis") - { - findRedisFunctionSecretArguments(); - } - else if (function->name() == "ytsaurus") - { - findYTsaurusStorageTableEngineSecretArguments(); - } - else if ((function->name() == "arrowFlight") || (function->name() == "arrowflight")) - { - findArrowFlightSecretArguments(); - } - else if ((function->name() == "jdbc") || (function->name() == "odbc")) - { - /// jdbc('DSN', schema, table) or jdbc('DSN', table) - /// odbc('DSN', schema, table) or odbc('DSN', table) - /// The DSN (connection string) may contain credentials. - findXDBCSecretArguments(); - } - } - - void findMySQLFunctionSecretArguments() + /// Whether a key of the `extra_credentials(..)` nested map carries a non-secret identifier whose + /// value stays visible when the map is masked (`role_arn` and `role_session_name`; the map's + /// secret is `external_id`). Any other key - unknown, malformed or an expression - fails closed. + static bool isNonSecretExtraCredentialsKey(std::string_view key) { - if (isNamedCollectionName(0)) - { - /// mysql(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - } - else - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - markSecretArgument(4); - } + return key == "role_arn" || key == "role_session_name"; } - void findMongoDBSecretArguments() - { - String uri; - - if (isNamedCollectionName(0)) - { - /// MongoDB(named_collection, ..., password = 'password', ...) - if (findSecretNamedArgument("password", 1)) - return; - - /// MongoDB(named_collection, ..., uri = 'mongodb://username:password@127.0.0.1:27017', ...) - if (findNamedArgument(&uri, "uri", 1) == -1) - return; - - result.are_named = true; - result.start = 1; - } - else if (function->arguments->size() == 2) - { - tryGetStringFromArgument(0, &uri); - result.are_named = false; - result.start = 0; - } - else - { - // MongoDB('127.0.0.1:27017', 'database', 'collection', 'user, 'password'...) - markSecretArgument(4, false); - return; - } - - chassert(result.count == 0); - maskURIPassword(&uri); - result.count = 1; - result.replacement = std::move(uri); - } - - void findRedisTableEngineSecretArguments() - { - /// Redis does not have URL/address argument, - /// only 'host:port' and separate "password" argument. - - if (isNamedCollectionName(0)) - { - if (findSecretNamedArgument("password", 1)) - return; - } - else - { - // Redis('host:port', 'db_index', 'password', 'pool_size') - markSecretArgument(2, false); - return; - } - } - - void findArrowFlightSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// ArrowFlight(named_collection, ..., password = 'password') - findSecretNamedArgument("password", 1); - } - else - { - /// ArrowFlight('host:port', 'dataset', 'username', 'password') - markSecretArgument(3); - } - } +protected: + const std::unique_ptr function; + Result result; - void findXDBCSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// jdbc(named_collection, ..., datasource = 'DSN', ...) - /// odbc(named_collection, ..., connection_settings = 'DSN', ...) - /// `datasource` and `connection_settings` are mutually exclusive aliases. - /// If the value is a URI, mask only the password; otherwise hide the whole value. - /// If somehow both are present (invalid query), hide all named arguments. - ssize_t ds_idx = findNamedArgument(nullptr, "datasource", 1); - ssize_t cs_idx = findNamedArgument(nullptr, "connection_settings", 1); - - if (ds_idx >= 0 && cs_idx >= 0) - { - /// Both present — hide all named arguments starting from index 1. - result.start = 1; - result.count = function->arguments->size() - 1; - result.are_named = true; - } - else if (ds_idx >= 0) - maskXDBCSecretNamedArgument("datasource", 1); - else if (cs_idx >= 0) - maskXDBCSecretNamedArgument("connection_settings", 1); - } - else - { - /// jdbc('DSN', schema, table) / jdbc('DSN', table) - /// odbc('DSN', schema, table) / odbc('DSN', table) - /// JDBC('DSN', database, table) / ODBC('DSN', database, table) - /// The connection string may be a URI with credentials embedded, - /// e.g. scheme://username:password@host:port/dbname - /// If so, mask only the password part; otherwise hide the whole argument. - String uri; - if (tryGetStringFromArgument(0, &uri)) - { - if (maskURIPassword(&uri)) - { - chassert(result.count == 0); - result.start = 0; - result.count = 1; - result.replacement = std::move(uri); - return; - } - } - markSecretArgument(0, false); - } - } + /// Named arguments carrying S3 secrets, shared by every S3 form (explicit-url and named-collection). + /// `external_id` is the shared secret of the assume-role triple; the other two (`role_arn`, + /// `role_session_name`) are non-secret identifiers passed inside `extra_credentials` and stay + /// visible (see isNonSecretExtraCredentialsKey). + static constexpr std::string_view s3_secret_keys[] + = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id"}; + + void markSecretArgument(size_t index, bool argument_is_named = false); + + /// `headers(..)` and `extra_credentials(..)` are nested maps whose values are secret auth material + /// (`extra_credentials` carries the assume-role secret `external_id`; its non-secret identifiers + /// stay visible, see isNonSecretExtraCredentialsKey). The parsers accept them at any position, not + /// just at the tail. Record them so their values are hidden with the keys kept. + /// Idempotent: each map is recorded at most once. + void maskNestedSecretMaps(); + + /// Single source of truth for reading an S3-style argument list the way the S3 parsers do: + /// `headers(..)` / `extra_credentials(..)` can appear at any position and are stripped before + /// positional slots are assigned; `key = value` arguments are named, and those with a key from + /// `s3_secret_keys` are masked (every occurrence: a duplicated key is logged before validation + /// rejects it); everything else (literals and constant expressions) occupies positional slots + /// in order. Returns the raw AST indices of the positional arguments; all slot arithmetic must + /// use them instead of raw indices. + /// Most parsers reject a positional after the first `key = value` argument, so such positionals + /// are masked (the query is logged before validation and the intended slot is unknowable). The + /// backup locator (`BackupInfo::fromAST`) instead collects positionals independently of named + /// overrides; it passes `positionals_allowed_after_named` to collect them in order. + std::vector classifyS3Arguments(size_t start = 0, bool positionals_allowed_after_named = false); + + /// Masks the positional secrets (`secret_access_key`, `session_token`) of the explicit-url S3 + /// form, selecting the signature by argument count and `with_structure` exactly like the parser + /// (`S3StorageParsedArguments::fromAST`). `positional` is the positional-only argument list, `url` + /// at `url_slot` (1 for `s3Cluster`, 0 otherwise). Value-based disambiguations (NOSIGN, format) + /// fail closed on an unevaluable expression: the potential credential slot is masked. + void maskS3PositionalSecrets(const std::vector & positional, size_t url_slot, bool with_structure); + + /// For S3 locators that accept nothing positional beyond `secret_access_key` (the S3 database + /// engine and the backup S3 destination): mask every positional from `first_slot` on, failing + /// closed on invalid extra positionals, which are logged before validation rejects them. + void maskS3PositionalsFrom(const std::vector & positional, size_t first_slot); + + /// The S3 URL itself can carry credentials: a userinfo part and presigned-URL query parameters. + /// If the url positional does, replace it with a partially masked copy that keeps the host and + /// path visible. The field set mirrors `BackupInfo::removeCredentialsFromS3URL`. + void maskS3UrlArgument(const std::vector & positional, size_t url_slot); + + void findOrdinaryFunctionSecretArguments(); + void findMySQLFunctionSecretArguments(); + void findMongoDBSecretArguments(); + void findRedisTableEngineSecretArguments(); + void findArrowFlightSecretArguments(); + void findXDBCSecretArguments(); /// Similar to `findSecretNamedArgument`, but if the value is a URI with credentials, /// masks only the password part instead of hiding the entire value. - void maskXDBCSecretNamedArgument(std::string_view key, size_t start) - { - String value; - ssize_t arg_idx = findNamedArgument(&value, key, start); - if (arg_idx < 0) - return; - - if (!value.empty() && maskURIPassword(&value)) - { - result.are_named = true; - result.start = arg_idx; - result.count = 1; - result.replacement = std::move(value); - } - else - { - markSecretArgument(arg_idx, /* argument_is_named= */ true); - } - } - - /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should - /// always be at the end). Marks "headers" as secret, if found. - size_t excludeS3OrURLNestedMaps() - { - size_t count = function->arguments->size(); - while (count > 0) - { - const auto f = function->arguments->at(count - 1)->getFunction(); - if (!f) - break; - if (f->name() == "headers") - result.nested_maps.push_back(f->name()); - else if (f->name() != "extra_credentials" && f->name() != "equals") - break; - count -= 1; - } - return count; - } - - void findS3FunctionSecretArguments(bool is_cluster_function) - { - /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. - size_t url_arg_idx = is_cluster_function ? 1 : 0; - - if (!is_cluster_function && isNamedCollectionName(0)) - { - /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...) - findSecretNamedArgument("secret_access_key", 1); - return; - } - - findSecretNamedArgument("secret_access_key", url_arg_idx); - - /// We should check other arguments first because we don't need to do any replacement in case of - /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - size_t count = excludeS3OrURLNestedMaps(); - if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4)) - { - String second_arg; - if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg)) - { - if (boost::iequals(second_arg, "NOSIGN")) - return; /// The argument after 'url' is "NOSIGN". - - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: s3('url', 'format', ...) - } - } - - /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (url_arg_idx + 2 < count) - markSecretArgument(url_arg_idx + 2); - } - - void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function) - { - /// azureBlobStorageCluster('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. - size_t url_arg_idx = is_cluster_function ? 1 : 0; - - if (!is_cluster_function && isNamedCollectionName(0)) - { - /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) - if (maskAzureConnectionString(-1, true, 1)) - return; - findSecretNamedArgument("account_key", 1); - return; - } - if (is_cluster_function && isNamedCollectionName(1)) - { - /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) - if (maskAzureConnectionString(-1, true, 2)) - return; - findSecretNamedArgument("account_key", 2); - return; - } - - if (maskAzureConnectionString(url_arg_idx)) - return; - - /// We should check other arguments first because we don't need to do any replacement in case of - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) - { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; - } - } + void maskXDBCSecretNamedArgument(std::string_view key, size_t start); - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); - } + void findS3FunctionSecretArguments(bool is_cluster_function); + void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); + bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); + /// Masks the secrets of every URL form (`url`/`urlCluster` table functions, the `URL` table + /// engine, and their named-collection variants): the userinfo password of the url positional or a + /// named `url = ...` override, and the `headers(...)` values at any position. `url` is at + /// `url_offset` (1 for `urlCluster`, which puts the cluster name first; 0 otherwise). + void findURLSecretArguments(size_t url_offset = 0); - bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0) - { - String url_arg; - if (argument_is_named) - { - url_arg_idx = findNamedArgument(&url_arg, "connection_string", start); - if (url_arg_idx == -1 || url_arg.empty()) - url_arg_idx = findNamedArgument(&url_arg, "storage_account_url", start); - if (url_arg_idx == -1 || url_arg.empty()) - return false; - } - else - { - if (!tryGetStringFromArgument(url_arg_idx, &url_arg)) - return false; - } + bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const; + static bool tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier = true); - if (!url_arg.starts_with("http")) - { - static re2::RE2 account_key_pattern = "AccountKey=.*?(;|$)"; - if (RE2::Replace(&url_arg, account_key_pattern, "AccountKey=[HIDDEN]\\1")) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; - } - - static re2::RE2 sas_signature_pattern = "SharedAccessSignature=.*?(;|$)"; - if (RE2::Replace(&url_arg, sas_signature_pattern, "SharedAccessSignature=[HIDDEN]\\1")) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; - } - } - - return false; - } - - void findURLSecretArguments() - { - if (isNamedCollectionName(0)) - return; - - excludeS3OrURLNestedMaps(); - - String uri; - if (tryGetStringFromArgument(0, &uri) && maskURIPassword(&uri)) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = 0; - result.count = 1; - result.replacement = std::move(uri); - } - } - - bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const - { - if (arg_idx >= function->arguments->size()) - return false; - - return tryGetStringFromArgument(*function->arguments->at(arg_idx), res, allow_identifier); - } - - static bool tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier = true) - { - return argument.tryGetString(res, allow_identifier); - } - - void findRemoteFunctionSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// remote(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - return; - } - - /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: - /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key]) - - /// But we should check the number of arguments first because we don't need to do any replacements in case of - /// remote('addresses_expr', db.table) - if (function->arguments->size() < 3) - return; - - size_t arg_num = 1; - - /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'. - auto table_function = function->arguments->at(arg_num)->getFunction(); - if (table_function && KnownTableFunctionNames::instance().exists(table_function->name())) - { - ++arg_num; - } - else - { - std::optional database; - std::optional qualified_table_name; - if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name)) - { - /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'. - /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user' - /// before the argument 'password'. So it's safer to wipe two arguments just in case. - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `user`. - markSecretArgument(arg_num + 2); - } - if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `sharding_key`. - markSecretArgument(arg_num + 3); - } - return; - } - - /// Skip the current argument (which is either a database name or a qualified table name). - ++arg_num; - if (database) - { - /// Skip the 'table' argument if the previous argument was a database name. - ++arg_num; - } - } - - /// Skip username. - ++arg_num; - - /// Do our replacement: - /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...) - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false); - if (can_be_password) - markSecretArgument(arg_num); - } + void findRemoteFunctionSecretArguments(); /// Tries to get either a database name or a qualified table name from an argument. /// Empty string is also allowed (it means the default database). @@ -584,386 +175,39 @@ class FunctionSecretArgumentsFinder bool tryGetDatabaseNameOrQualifiedTableName( size_t arg_idx, std::optional & res_database, - std::optional & res_qualified_table_name) const - { - res_database.reset(); - res_qualified_table_name.reset(); - - String str; - if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true)) - return false; - - if (str.empty()) - { - res_database = ""; - return true; - } - - auto qualified_table_name = QualifiedTableName::tryParseFromString(str); - if (!qualified_table_name) - return false; - - if (qualified_table_name->database.empty()) - res_database = std::move(qualified_table_name->table); - else - res_qualified_table_name = std::move(qualified_table_name); - return true; - } - - void findEncryptionFunctionSecretArguments() - { - if (function->arguments->size() == 0) - return; - - /// We replace all arguments after 'mode' with '[HIDDEN]': - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]') - result.start = 1; - result.count = function->arguments->size() - 1; - } - - void findHMACSecretArguments() - { - if (function->arguments->size() < 3) - return; - - /// We hide the key argument and any following for the case of mistyping or using extra arguments by mistake: - /// HMAC('mode', 'message', 'key') -> HMAC('mode', 'message', '[HIDDEN]') - /// HMAC('sha256', toString(toFixedString('b', 3), 3), '(', 'this_should_be_secret') -> HMAC('sha256', toString(toFixedString('b', 3), 3), '[HIDDEN]', '[HIDDEN]') - result.start = 2; - result.count = function->arguments->size() - 2; - } - - void findTableEngineSecretArguments() - { - const String & engine_name = function->name(); - if (engine_name == "ExternalDistributed") - { - /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - findExternalDistributedTableEngineSecretArguments(); - } - else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") || (engine_name == "MaterializedPostgreSQL")) - { - /// MySQL('host:port', 'database', 'table', 'user', 'password', ...) - /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) - /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) - /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); - } - else if (engine_name == "MongoDB") - { - findMongoDBSecretArguments(); - } - else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") - || (engine_name == "DeltaLake") || (engine_name == "Hudi") - || (engine_name == "Iceberg") || (engine_name == "IcebergS3") - || (engine_name == "S3Queue")) - { - /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) - findS3TableEngineSecretArguments(); - } - else if (engine_name == "URL") - { - findURLSecretArguments(); - } - else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") - { - findAzureBlobStorageTableEngineSecretArguments(); - } - else if (engine_name == "Redis") - { - findRedisTableEngineSecretArguments(); - } - else if (engine_name == "YTsaurus") - { - findYTsaurusStorageTableEngineSecretArguments(); - } - else if (engine_name == "ArrowFlight") - { - findArrowFlightSecretArguments(); - } - else if ((engine_name == "JDBC") || (engine_name == "ODBC")) - { - /// JDBC('DSN', database, table) - /// ODBC('DSN', database, table) - /// The DSN (connection string) may contain credentials. - findXDBCSecretArguments(); - } - } - - void findExternalDistributedTableEngineSecretArguments() - { - if (isNamedCollectionName(1)) - { - /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 2); - } - else - { - /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - markSecretArgument(5); - } - } - - void findS3TableEngineSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// S3(named_collection, ..., secret_access_key = 'secret_access_key') - findSecretNamedArgument("secret_access_key", 1); - return; - } - - findSecretNamedArgument("secret_access_key", 0); - - /// We should check other arguments first because we don't need to do any replacement in case of - /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)]) - size_t count = excludeS3OrURLNestedMaps(); - if ((3 <= count) && (count <= 4)) - { - String second_arg; - if (tryGetStringFromArgument(1, &second_arg)) - { - if (boost::iequals(second_arg, "NOSIGN")) - return; /// The argument after 'url' is "NOSIGN". - - if (count == 3) - { - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: S3('url', 'format', ...) - } - } - } - - /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (2 < count) - markSecretArgument(2); - } - - void findAzureBlobStorageTableEngineSecretArguments() - { - /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format, [account_name, account_key, ...]) - size_t url_arg_idx = 0; - - if (isNamedCollectionName(url_arg_idx)) - { - /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) - if (maskAzureConnectionString(-1, true, 1)) - return; - findSecretNamedArgument("account_key", 1); - return; - } - - if (maskAzureConnectionString(url_arg_idx)) - return; - - /// We should check other arguments first because we don't need to do any replacement in case of - /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) - { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; - } - } - - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); - } - - void findRedisFunctionSecretArguments() - { - // redis(host:port, key, structure, db_index, password, pool_size) - markSecretArgument(4); - } - - void findYTsaurusStorageTableEngineSecretArguments() - { - // YTsaurus('base_uri', 'yt_path', 'auth_token') - markSecretArgument(2); - } - - void findDatabaseEngineSecretArguments() - { - const String & engine_name = function->name(); - if (engine_name == "MySQL" || - engine_name == "PostgreSQL" || - engine_name == "MaterializedPostgreSQL") - { - /// MySQL('host:port', 'database', 'user', 'password') - /// PostgreSQL('host:port', 'database', 'user', 'password') - findMySQLDatabaseSecretArguments(); - } - else if (engine_name == "S3") - { - /// S3('url', 'access_key_id', 'secret_access_key') - findS3DatabaseSecretArguments(); - } - else if (engine_name == "DataLakeCatalog") - { - findDataLakeCatalogSecretArguments(); - } - else if (engine_name == "Backup") - { - findBackupDatabaseSecretArguments(); - } - } - - void findMySQLDatabaseSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// MySQL(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - } - else - { - /// MySQL('host:port', 'database', 'user', 'password') - markSecretArgument(3); - } - } - - void findS3DatabaseSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// S3(named_collection, ..., secret_access_key = 'password', ...) - findSecretNamedArgument("secret_access_key", 1); - } - else - { - /// S3('url', 'access_key_id', 'secret_access_key') - markSecretArgument(2); - } - } - - void findDataLakeCatalogSecretArguments() - { - /// datalake catalog should support different storage types, - /// we need a function to check if the url is S3 or Azure. - /// right now we assume it's a S3 url - findS3DatabaseSecretArguments(); - } - - void findBackupDatabaseSecretArguments() - { - if (function->arguments->size() < 2) - return; - - auto storage_arg = function->arguments->at(1); - auto storage_function = storage_arg->getFunction(); - - /// Backup('', S3('url', 'access_key_id', 'secret_access_key')) - if (storage_function && storage_function->name() == "S3" && storage_function->arguments->size() >= 3) - { - std::string replacement = "S3("; - - for (size_t i = 0; i < storage_function->arguments->size(); ++i) - { - if (i > 0) - { - replacement += ", "; - } - - if (i == 2) // Secret key position - { - replacement += "'[HIDDEN]'"; - } - else - { - String arg_value; - if (!storage_function->arguments->at(i)->tryGetString(&arg_value, true)) - { - return; - } - replacement += "'" + arg_value + "'"; - } - } - replacement += ")"; - - result.start = 1; - result.count = 1; - result.replacement = std::move(replacement); - result.quote_replacement = false; - } - } - - void findBackupNameSecretArguments() - { - const String & engine_name = function->name(); - if (engine_name == "S3") - { - if (isNamedCollectionName(0)) - { - /// BACKUP ... TO S3(named_collection, ..., secret_access_key = 'secret_access_key', ...) - findSecretNamedArgument("secret_access_key", 1); - return; - } - /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key]) - markSecretArgument(2); - } - else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") - { - findAzureBlobStorageTableEngineSecretArguments(); - } - } + std::optional & res_qualified_table_name) const; + + void findEncryptionFunctionSecretArguments(); + void findHMACSecretArguments(); + void findTableEngineSecretArguments(); + void findExternalDistributedTableEngineSecretArguments(); + void findS3TableEngineSecretArguments(); + void findAzureBlobStorageTableEngineSecretArguments(); + void findRedisFunctionSecretArguments(); + void findYTsaurusStorageTableEngineSecretArguments(); + void findDatabaseEngineSecretArguments(); + void findMySQLDatabaseSecretArguments(); + void findS3DatabaseSecretArguments(); + void findDataLakeCatalogSecretArguments(); + void findBackupDatabaseSecretArguments(); + void findBackupNameSecretArguments(); /// Whether a specified argument can be the name of a named collection? - bool isNamedCollectionName(size_t arg_idx) const - { - if (function->arguments->size() <= arg_idx) - return false; - - return function->arguments->at(arg_idx)->isIdentifier(); - } + bool isNamedCollectionName(size_t arg_idx) const; /// Looks for an argument with a specified name. This function looks for arguments in format `key=value` where the key is specified. /// Returns -1 if no argument was found. - ssize_t findNamedArgument(String * res, std::string_view key, size_t start = 0) - { - for (size_t i = start; i < function->arguments->size(); ++i) - { - const auto & argument = function->arguments->at(i); - const auto equals_func = argument->getFunction(); - if (!equals_func || (equals_func->name() != "equals")) - continue; - - if (!equals_func->arguments || equals_func->arguments->size() != 2) - continue; - - String found_key; - if (!tryGetStringFromArgument(*equals_func->arguments->at(0), &found_key)) - continue; - - if (found_key == key) - { - tryGetStringFromArgument(*equals_func->arguments->at(1), res); - return i; - } - } + ssize_t findNamedArgument(String * res, std::string_view key, size_t start = 0); - return -1; - } + /// Looks for secret arguments with a specified name in format `key=value` and marks them secret. + /// Marks *every* occurrence, not just the first: a malformed query is formatted for logging before + /// duplicate-key validation runs, so `session_token = 'a', session_token = 'b'` must hide both. + bool findSecretNamedArgument(std::string_view key, size_t start = 0); - /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified. - /// If the argument is found, it is marked as a secret. - bool findSecretNamedArgument(std::string_view key, size_t start = 0) - { - ssize_t arg_idx = findNamedArgument(nullptr, key, start); - if (arg_idx >= 0) - { - markSecretArgument(arg_idx, /* argument_is_named= */ true); - return true; - } - return false; - } + /// Masks the secrets of an S3 named-collection form: the secret named overrides (every occurrence, + /// in any order; the span covering them may hide a non-secret argument in between, which is safe) + /// and the `headers(...)` / `extra_credentials(...)` map overrides. + void findS3NamedCollectionSecretArguments(size_t start = 0); }; } diff --git a/src/Parsers/FunctionSecretArgumentsFinderAST.h b/src/Parsers/FunctionSecretArgumentsFinderAST.h index 86211b3a299c..754411535c8d 100644 --- a/src/Parsers/FunctionSecretArgumentsFinderAST.h +++ b/src/Parsers/FunctionSecretArgumentsFinderAST.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -46,6 +47,15 @@ class FunctionAST : public AbstractFunction return false; } + bool tryGetLiteralText(String * res) const override + { + const auto * literal = argument->as(); + if (!literal) + return false; + if (res) + *res = applyVisitor(FieldVisitorToString(), literal->value); + return true; + } private: const IAST * argument = nullptr; }; diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 867dc7aac998..1c2fc4c0818e 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ namespace Setting extern const SettingsBool enable_named_columns_in_function_tuple; extern const SettingsBool transform_null_in; extern const SettingsInt64 optimize_const_name_size; + extern const SettingsBool format_display_secrets_in_show_and_select; } namespace ErrorCodes @@ -622,7 +624,7 @@ class ActionsScopeNode } const ActionsDAG::Node * addConstantIfNecessary( - const std::string & node_name, ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic) + const std::string & node_name, ColumnConstPtr column, DataTypePtr type, std::string name, bool is_deterministic, bool is_masked_secret = false) { auto it = node_name_to_node.find(node_name); if (it != node_name_to_node.end()) @@ -637,7 +639,7 @@ class ActionsScopeNode return it->second; } - const auto * node = &actions_dag.addColumn(std::move(column), std::move(type), std::move(name), is_deterministic); + const auto * node = &actions_dag.addColumn(std::move(column), std::move(type), std::move(name), is_deterministic, is_masked_secret); node_name_to_node[node->result_name] = node; return node; @@ -985,7 +987,8 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi }(); actions_stack[0].addConstantIfNecessary( - constant_node_name, constant_node.getColumn(), constant_type, constant_node_name, constant_node.isDeterministic()); + constant_node_name, constant_node.getColumn(), constant_type, constant_node_name, constant_node.isDeterministic(), + /* is_masked_secret= */ constant_node.isMasked()); size_t actions_stack_size = actions_stack.size(); if (actions_stack_size > 1) @@ -1200,6 +1203,33 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi return { function_node_name, Levels(exists_function_level) }; } +/// A secret function argument can be a constant that the planner folds in from a column or subquery +/// after the query-tree masking ran, so it is not flagged as a secret in the tree (e.g. the key of +/// `encrypt(..., k)` where `k` is `'secret' AS k` in a subquery). Flag such constant argument nodes so +/// plan dumps render them as `[HIDDEN]`. The finder runs only when secrets are hidden (the caller +/// gates on the setting). +void markFoldedSecretConstants(const FunctionNode & function_node, const ActionsDAG::NodeRawConstPtrs & children) +{ + auto secret_arguments = FunctionSecretArgumentsFinderTreeNode(function_node).getResult(); + if (!secret_arguments.hasSecrets()) + return; + + auto mark = [&](size_t index) + { + /// Any node carrying a constant column is a folded secret value, whether it is a plain COLUMN + /// node or a FUNCTION node folded to a constant (e.g. `concat(k1, k2)`); flag either. + if (index < children.size() && children[index]->column && !children[index]->is_masked_secret) + const_cast(children[index])->is_masked_secret = true; + }; + + for (size_t i = secret_arguments.start; i < secret_arguments.start + secret_arguments.count; ++i) + mark(i); + for (const auto & [index, _] : secret_arguments.masked_arguments) + mark(index); + for (const auto & [index, _] : secret_arguments.replaced_arguments) + mark(index); +} + PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node) { const auto & function_node = node->as(); @@ -1284,6 +1314,9 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi for (auto & function_argument_node_name : function_arguments_node_names) children.push_back(actions_stack[level].getNodeOrThrow(function_argument_node_name)); + if (!planner_context->getQueryContext()->getSettingsRef()[Setting::format_display_secrets_in_show_and_select]) + markFoldedSecretConstants(function_node, children); + if (function_node.getFunctionName() == "arrayJoin") { if (level != 0) diff --git a/src/Processors/QueryPlan/QueryPlanFormat.cpp b/src/Processors/QueryPlan/QueryPlanFormat.cpp index 1fd6c4f584fe..5738dc15bb75 100644 --- a/src/Processors/QueryPlan/QueryPlanFormat.cpp +++ b/src/Processors/QueryPlan/QueryPlanFormat.cpp @@ -201,6 +201,16 @@ namespace QueryPlanFormat String formatConstant(const ActionsDAG::Node * node) { + /// A masked secret constant must render as `[HIDDEN]`, never as the value held in its + /// column (kept only so the query can still execute). `is_masked_secret` is the reliable + /// signal; the name check is a fallback for a masked constant whose name is its `[HIDDEN...]` + /// placeholder but which was reached without the flag (e.g. an aliased column keeps its own + /// name, so the flag is what catches it there). + if (node->is_masked_secret) + return "[HIDDEN]"; + if (node->result_name.contains("[HIDDEN")) + return node->result_name; + if (!node->column) return node->result_name; @@ -312,6 +322,12 @@ namespace QueryPlanFormat { using ActionType = ActionsDAG::ActionType; + /// A masked secret carrier (a folded constant, which may be a FUNCTION node with a constant + /// column, not only a COLUMN node) must render as `[HIDDEN]` regardless of its node type, + /// before we dispatch into formatting its value or its child expression. + if (node->is_masked_secret) + return "[HIDDEN]"; + switch (node->type) { case ActionType::INPUT: diff --git a/tests/queries/0_stateless/02968_url_args.reference b/tests/queries/0_stateless/02968_url_args.reference index 3c51a5edf814..7c3d6233df48 100644 --- a/tests/queries/0_stateless/02968_url_args.reference +++ b/tests/queries/0_stateless/02968_url_args.reference @@ -1,7 +1,7 @@ CREATE TABLE default.a\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE TABLE default.b\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers()) CREATE TABLE default.c\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\')) -CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'bar\'), \'CSV\') +CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'[HIDDEN]\'), \'CSV\') CREATE VIEW default.e\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE VIEW default.f\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers()) CREATE VIEW default.g\n(\n `x` Int64\n)\nAS SELECT count()\nFROM s3(\'https://example.s3.amazonaws.com/a.csv\', CSV, headers(\'foo\' = \'[HIDDEN]\')) diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference index 72696cef3429..38df73fa4d5f 100644 --- a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference @@ -5,10 +5,10 @@ CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://l CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\', \'none\') -CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') -CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'[HIDDEN]\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'[HIDDEN]\', \'JSON\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\', \'none\') -CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'[HIDDEN]\', \'JSON\', \'none\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\', \'none\') diff --git a/tests/queries/0_stateless/04343_secret_args_finder_mixed_named_positional.reference b/tests/queries/0_stateless/04343_secret_args_finder_mixed_named_positional.reference index a053b0444fc6..290704830ca1 100644 --- a/tests/queries/0_stateless/04343_secret_args_finder_mixed_named_positional.reference +++ b/tests/queries/0_stateless/04343_secret_args_finder_mixed_named_positional.reference @@ -1,4 +1,4 @@ -SELECT * FROM gcs(\'url\', \'a\', \'[HIDDEN]\') -SELECT * FROM s3(\'http://localhost:11111/foo\', equals(secret_access_key, materialize(257), \'a\'), \'[HIDDEN]\') -SELECT * FROM s3(\'url\', \'a\', \'[HIDDEN]\') +SELECT * FROM gcs(\'url\', \'a\', \'[HIDDEN]\', secret_access_key = \'[HIDDEN]\') +SELECT * FROM s3(\'http://localhost:11111/foo\', equals(secret_access_key, materialize(257), \'a\'), \'[HIDDEN]\', secret_access_key = \'[HIDDEN]\') +SELECT * FROM s3(\'url\', \'a\', \'[HIDDEN]\', secret_access_key = \'[HIDDEN]\') 1 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference new file mode 100644 index 000000000000..8a939435ccdc --- /dev/null +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -0,0 +1,220 @@ +CREATE TABLE default.t_04510\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510\', \'ak\', \'[HIDDEN]\', \'TSV\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', format = \'TSV\', extra_credentials(\'role_arn\' = \'visible_role_arn\', \'external_id\' = \'[HIDDEN]\')) +CREATE TABLE default.t_04510_pos\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'TSV\') +CREATE TABLE default.t_04510_mid\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510mid\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'TSV\', headers(\'Authorization\' = \'[HIDDEN]\')) +CREATE TABLE default.t_04510_exprfmt\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510exprfmt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'none\') +CREATE TABLE default.t_04510_nosign5\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510nosign5\', \'NOSIGN\', \'[HIDDEN]\', \'CSV\', \'none\') +CREATE TABLE default.t_04510_parqtok\n(\n `x` UInt8\n)\nENGINE = S3(\'http://localhost:11111/test/04510parqtok\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'CSV\', \'none\') +CREATE TABLE default.t_04510_gcs\n(\n `x` UInt8\n)\nENGINE = GCS(\'http://localhost:11111/test/04510gcs\', \'ak\', \'[HIDDEN]\', \'TSV\') +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 5 + CONSTANT id: 5, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 6, constant_value: \'ak\', constant_value_type: String + CONSTANT id: 7, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 8, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 9, constant_value: \'x UInt8\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 5 + CONSTANT id: 5, constant_value: \'http://localhost:11111/test/04510qt\', constant_value_type: String + IDENTIFIER id: 6, identifier: NOSIGN + CONSTANT id: 7, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 8, constant_value: \'x UInt8\', constant_value_type: String + FUNCTION id: 9, function_name: headers, function_type: ordinary + ARGUMENTS + LIST id: 10, nodes: 1 + FUNCTION id: 11, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 12, nodes: 2 + CONSTANT id: 13, constant_value: \'Authorization\', constant_value_type: String + CONSTANT id: 14, constant_value: [HIDDEN], constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 6 + CONSTANT id: 5, constant_value: \'http://localhost:11111/test/04510qt\', constant_value_type: String + CONSTANT id: 6, constant_value: \'ak\', constant_value_type: String + CONSTANT id: 7, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 8, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 9, constant_value: \'x UInt8\', constant_value_type: String + FUNCTION id: 10, function_name: extra_credentials, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 1 + FUNCTION id: 12, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 13, nodes: 2 + IDENTIFIER id: 14, identifier: external_id + CONSTANT id: 15, constant_value: [HIDDEN], constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 6 + CONSTANT id: 5, constant_value: \'http://localhost:11111/test/04510qt\', constant_value_type: String + CONSTANT id: 6, constant_value: \'ak\', constant_value_type: String + CONSTANT id: 7, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 8, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 9, constant_value: \'x UInt8\', constant_value_type: String + FUNCTION id: 10, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 2 + IDENTIFIER id: 12, identifier: session_token + CONSTANT id: 13, constant_value: \'[HIDDEN]\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 5 + CONSTANT id: 5, constant_value: \'http://localhost:11111/test/04510qt\', constant_value_type: String + IDENTIFIER id: 6, identifier: NOSIGN + CONSTANT id: 7, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 8, constant_value: \'x UInt8\', constant_value_type: String + FUNCTION id: 9, function_name: headers, function_type: ordinary + ARGUMENTS + LIST id: 10, nodes: 1 + FUNCTION id: 11, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 12, nodes: 2 + CONSTANT id: 13, constant_value: \'Authorization\', constant_value_type: String + CONSTANT id: 14, constant_value: \'[HIDDEN]\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: s3 + ARGUMENTS + LIST id: 4, nodes: 3 + IDENTIFIER id: 5, identifier: nc_04510_missing + FUNCTION id: 6, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 7, nodes: 2 + IDENTIFIER id: 8, identifier: url + CONSTANT id: 9, constant_value: [HIDDEN], constant_value_type: String + FUNCTION id: 10, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 2 + IDENTIFIER id: 12, identifier: structure + CONSTANT id: 13, constant_value: \'x UInt8\', constant_value_type: String +UNION id: 0, union_mode: UNION_ALL + QUERIES + LIST id: 1, nodes: 2 + QUERY id: 2 + PROJECTION + LIST id: 3, nodes: 1 + MATCHER id: 4, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 5, table_function_name: s3 + ARGUMENTS + LIST id: 6, nodes: 5 + CONSTANT id: 7, constant_value: \'http://localhost:11111/test/04510qt\', constant_value_type: String + CONSTANT id: 8, constant_value: \'ak\', constant_value_type: String + CONSTANT id: 9, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 10, constant_value: \'TSV\', constant_value_type: String + CONSTANT id: 11, constant_value: \'x UInt8\', constant_value_type: String + QUERY id: 12 + PROJECTION + LIST id: 13, nodes: 1 + CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8 + JOIN TREE + IDENTIFIER id: 15, identifier: system.one +-- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers\n-- (role_arn, role_session_name) stay visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; +CREATE TABLE t_04510 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', external_id = \'[HIDDEN]\'), format = \'TSV\') +SHOW CREATE TABLE t_04510 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510; +-- Engine form with a positional session_token (4th positional argument) must be hidden too.\nDROP TABLE IF EXISTS t_04510_pos; +CREATE TABLE t_04510_pos (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'TSV\') +SHOW CREATE TABLE t_04510_pos SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_pos; +-- The parser strips nested maps from any position before assigning positional slots, so a map\n-- placed before the positional session_token must not shift the token out of the masked slot.\nDROP TABLE IF EXISTS t_04510_mid; +CREATE TABLE t_04510_mid (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510mid\', \'ak\', \'[HIDDEN]\', headers(\'Authorization\' = \'[HIDDEN]\'), \'[HIDDEN]\', \'TSV\') +SHOW CREATE TABLE t_04510_mid SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_mid; +-- A constant-expression format at the session_token slot is valid (the parser evaluates it). The\n-- storage stores the evaluated literal, so SHOW CREATE keeps the format visible. In the logged text\n-- of the original query the unevaluated expression is indistinguishable from a session token (which\n-- would show its pieces verbatim), so there it is hidden: fail closed.\nDROP TABLE IF EXISTS t_04510_exprfmt; +CREATE TABLE t_04510_exprfmt (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510exprfmt\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'none\') +SHOW CREATE TABLE t_04510_exprfmt SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_exprfmt; +-- Five positional arguments in the engine form (no positional structure) is the access-key signature,\n-- not a NOSIGN one: NOSIGN sits in the access_key_id slot and the following argument is the\n-- secret_access_key, which must be hidden even though the leading token reads as NOSIGN.\nDROP TABLE IF EXISTS t_04510_nosign5; +CREATE TABLE t_04510_nosign5 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510nosign5\', NOSIGN, \'[HIDDEN]\', \'CSV\', \'none\') +SHOW CREATE TABLE t_04510_nosign5 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_nosign5; +-- Six positional arguments in the engine form fix the session_token at the 4th slot regardless of its\n-- value, so a session_token that happens to spell a registered format name must still be hidden.\nDROP TABLE IF EXISTS t_04510_parqtok; +CREATE TABLE t_04510_parqtok (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510parqtok\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'CSV\', \'none\') +SHOW CREATE TABLE t_04510_parqtok SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_parqtok; +-- Every S3-backed table engine shares the S3 credential signature, so SHOW CREATE must hide the\n-- secret_access_key for the whole family (GCS, the data-lake engines, ...), not only for `S3`.\nDROP TABLE IF EXISTS t_04510_gcs; +CREATE TABLE t_04510_gcs (`x` UInt8) ENGINE = GCS(\'http://localhost:11111/test/04510gcs\', \'ak\', \'[HIDDEN]\', \'TSV\') +SHOW CREATE TABLE t_04510_gcs SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_gcs; +SELECT * FROM s3(\'url_basic\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_interleaved\', secret_access_key = \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'), session_token = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_postoken\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') +SELECT * FROM s3(\'url_noncontig\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\', session_token = \'[HIDDEN]\') +-- The five-positional NOSIGN form carries no credentials; nothing must be masked.\nSELECT * FROM s3(\'url_nosign5\', NOSIGN, \'TSV\', \'x UInt8\', \'none\'); +SELECT * FROM s3(\'url_midtok\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'), \'[HIDDEN]\', \'TSV\', \'x UInt8\') +SELECT * FROM s3(\'url_dup\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', format = \'TSV\', session_token = \'[HIDDEN]\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_badmap\', \'ak\', \'[HIDDEN]\', extra_credentials(\'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_posafter\', access_key_id = \'ak\', \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_exprkey\', \'ak\', \'[HIDDEN]\', concat(\'session_\', \'token\') = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_fmthdr\', format = \'[HIDDEN]\', structure = \'x UInt8\') +SELECT * FROM s3(\'url_rolehdr\', \'ak\', \'[HIDDEN]\', extra_credentials(role_arn = \'[HIDDEN]\')) +SELECT * FROM s3(\'url_exprstruct\', \'CSV\', \'[HIDDEN]\', \'TSV\', concat(\'struc\', \'ture\') = \'[HIDDEN]\') +SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/x?X-Amz-Signature=[HIDDEN]&partNumber=7\', \'TSV\', \'x UInt8\') +SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/x?X-Amz-Signature=[HIDDEN]\', \'TSV\', \'x UInt8\') +SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/x/o\\\'clock?X-Amz-Signature=[HIDDEN]\', \'TSV\', \'x UInt8\') +SELECT * FROM s3(\'[HIDDEN]\', \'TSV\', \'x UInt8\') +BACKUP TABLE nonexistent_04510 TO S3(\'https://[HIDDEN]@localhost:11111/x?X-Amz-Signature=[HIDDEN]\', \'ak\', \'[HIDDEN]\') +CREATE DATABASE db_04510_authurl ENGINE = Backup(\'\', S3(\'https://[HIDDEN]@localhost:11111/x?X-Amz-Signature=[HIDDEN]\', \'ak\', \'[HIDDEN]\')) +SELECT * FROM s3(nc_04510_missing, extra_credentials(external_id = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_authurl_missing, url = \'https://[HIDDEN]@localhost:11111/x?X-Amz-Signature=[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_headers_missing, headers(\'Authorization\' = \'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_badhdr_missing, headers(\'[HIDDEN]\'), format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_span_missing, secret_access_key = \'[HIDDEN]\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_exprkey_missing, concat(\'secret_\', \'access_key\') = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +SELECT * FROM s3(nc_prepos_missing, \'[HIDDEN]\', secret_access_key = \'[HIDDEN]\', format = \'TSV\', structure = \'x UInt8\') +BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_named\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\')) +BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_pos\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, \'visible_bkp_dir\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, secret_access_key = \'[HIDDEN]\', \'visible_bkp_dir2\') +BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_mixed\', equals(access_key_id, \'ak\'), \'[HIDDEN]\') +CREATE DATABASE db_04510_ec ENGINE = Backup(\'\', S3(\'url_dbec\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) +CREATE DATABASE db_04510_postok ENGINE = Backup(\'\', S3(\'url_dbpostok\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_ncpos ENGINE = Backup(\'\', S3(nc_dbnc_missing, \'visible_dbnc_dir\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_ncorder ENGINE = Backup(\'\', S3(nc_dbord_missing, secret_access_key = \'[HIDDEN]\', \'visible_dbnc_dir2\')) +CREATE DATABASE db_04510_ncenv ENGINE = Backup(\'\', S3(nc_dbenv_missing, secret_access_key = \'[HIDDEN]\', use_environment_credentials = 1)) +CREATE DATABASE db_04510_ncexpr ENGINE = Backup(\'\', S3(nc_dbexpr_missing, secret_access_key = \'[HIDDEN]\', filename = \'[HIDDEN]\')) +CREATE DATABASE db_04510_mixed ENGINE = Backup(\'\', S3(\'url_dbmixed\', access_key_id = \'ak\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = \'[HIDDEN]\')) +CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') +DROP DATABASE IF EXISTS default_1 +CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) +DROP DATABASE default_1 +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\', extra_credentials(external_id = \'[HIDDEN]\')) +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\', session_token = \'[HIDDEN]\') +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = \'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', structure = \'x UInt8\') +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') UNION ALL SELECT 1 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql new file mode 100644 index 000000000000..7d1c627caeac --- /dev/null +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -0,0 +1,332 @@ +-- Tags: no-fasttest +-- no-fasttest: the S3 table engine is not available in the fast test build. + +-- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and +-- the extra_credentials assume-role material (external_id) passed to the explicit-url or +-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged +-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE +-- and logged query text. + +-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers +-- (role_arn, role_session_name) stay visible while external_id is hidden. +DROP TABLE IF EXISTS t_04510; +CREATE TABLE t_04510 (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510', 'ak', 'SEKRIT_SAK', + session_token = 'SEKRIT_ST', + google_adc_client_secret = 'SEKRIT_ADCCS', + google_adc_refresh_token = 'SEKRIT_ADCRT', + extra_credentials(role_arn = 'visible_role_arn', external_id = 'SEKRIT_EID'), + format = 'TSV'); +SHOW CREATE TABLE t_04510 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510; + +-- Engine form with a positional session_token (4th positional argument) must be hidden too. +DROP TABLE IF EXISTS t_04510_pos; +CREATE TABLE t_04510_pos (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510pos', 'ak', 'SEKRIT_SAK', 'SEKRIT_POSTOK', 'TSV'); +SHOW CREATE TABLE t_04510_pos SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_pos; + +-- The parser strips nested maps from any position before assigning positional slots, so a map +-- placed before the positional session_token must not shift the token out of the masked slot. +DROP TABLE IF EXISTS t_04510_mid; +CREATE TABLE t_04510_mid (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510mid', 'ak', 'SEKRIT_SAK', + headers('Authorization' = 'SEKRIT_HDR'), 'SEKRIT_MIDTOK', 'TSV'); +SHOW CREATE TABLE t_04510_mid SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_mid; + +-- A constant-expression format at the session_token slot is valid (the parser evaluates it). The +-- storage stores the evaluated literal, so SHOW CREATE keeps the format visible. In the logged text +-- of the original query the unevaluated expression is indistinguishable from a session token (which +-- would show its pieces verbatim), so there it is hidden: fail closed. +DROP TABLE IF EXISTS t_04510_exprfmt; +CREATE TABLE t_04510_exprfmt (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510exprfmt', 'ak', 'SEKRIT_SAK', concat('TS', 'V'), 'none'); +SHOW CREATE TABLE t_04510_exprfmt SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_exprfmt; + +-- Five positional arguments in the engine form (no positional structure) is the access-key signature, +-- not a NOSIGN one: NOSIGN sits in the access_key_id slot and the following argument is the +-- secret_access_key, which must be hidden even though the leading token reads as NOSIGN. +DROP TABLE IF EXISTS t_04510_nosign5; +CREATE TABLE t_04510_nosign5 (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510nosign5', NOSIGN, 'SEKRIT_NOSIGNSAK', 'CSV', 'none'); +SHOW CREATE TABLE t_04510_nosign5 SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_nosign5; + +-- Six positional arguments in the engine form fix the session_token at the 4th slot regardless of its +-- value, so a session_token that happens to spell a registered format name must still be hidden. +DROP TABLE IF EXISTS t_04510_parqtok; +CREATE TABLE t_04510_parqtok (x UInt8) +ENGINE = S3('http://localhost:11111/test/04510parqtok', 'ak', 'SEKRIT_PARQSAK', 'Parquet', 'CSV', 'none'); +SHOW CREATE TABLE t_04510_parqtok SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_parqtok; + +-- Every S3-backed table engine shares the S3 credential signature, so SHOW CREATE must hide the +-- secret_access_key for the whole family (GCS, the data-lake engines, ...), not only for `S3`. +DROP TABLE IF EXISTS t_04510_gcs; +CREATE TABLE t_04510_gcs (x UInt8) +ENGINE = GCS('http://localhost:11111/test/04510gcs', 'ak', 'SEKRIT_GCSSAK', 'TSV'); +SHOW CREATE TABLE t_04510_gcs SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04510_gcs; + +-- The forms below all fail at analysis (empty host / missing collection) before any network access, +-- and are logged with secrets replaced. Each carries a unique marker checked by the final assertion. + +-- Explicit-url function form. +SELECT * FROM s3('url_basic', 'ak', 'SEKRIT_SAK', + session_token = 'SEKRIT_ST', + google_adc_client_secret = 'SEKRIT_ADCCS', + google_adc_refresh_token = 'SEKRIT_ADCRT', + extra_credentials(external_id = 'SEKRIT_EID'), + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- extra_credentials placed between two named secret overrides must be masked as a nested map, +-- not swept into the named secret span (which would leak its first nested value). +SELECT * FROM s3('url_interleaved', secret_access_key = 'SEKRIT_SAK', + extra_credentials(external_id = 'SEKRIT_EID'), + session_token = 'SEKRIT_ST', + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- Explicit-url function form with a positional session_token (4th positional argument). +SELECT * FROM s3('url_postoken', 'ak', 'SEKRIT_SAK', 'SEKRIT_POSTOK', + 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- Secrets can be non-contiguous in valid syntax; the non-secret arguments in between (format, +-- structure) must stay visible. +SELECT * FROM s3('url_noncontig', 'ak', 'SEKRIT_SAK', 'TSV', 'x UInt8', + session_token = 'SEKRIT_TAILTOK'); -- { serverError BAD_ARGUMENTS } + +-- The five-positional NOSIGN form carries no credentials; nothing must be masked. +SELECT * FROM s3('url_nosign5', NOSIGN, 'TSV', 'x UInt8', 'none'); -- { serverError BAD_ARGUMENTS } + +-- A nested map before the positional session_token must not shift it out of the masked slot. +SELECT * FROM s3('url_midtok', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = 'SEKRIT_EID'), + 'SEKRIT_MIDTOK', 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A duplicated secret key is malformed but formatted for logging before validation rejects it, so +-- every occurrence must be masked, not just the first. +SELECT * FROM s3('url_dup', 'ak', 'sk', + session_token = 'SEKRIT_DUP1', + format = 'TSV', + session_token = 'SEKRIT_DUP2', + structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A nested map with a malformed child (not `key = value`) must fail closed in formatting. +-- The analyzer rejects `extra_credentials` as an unknown function; with `enable_analyzer = 0` +-- the S3 URI validation is reached first. +SELECT * FROM s3('url_badmap', 'ak', 'SEKRIT_SAK', + extra_credentials('SEKRIT_RAWCRED'), + format = 'TSV', structure = 'x UInt8'); -- { serverError UNKNOWN_FUNCTION, BAD_ARGUMENTS } + +-- The parser rejects a positional after the first key = value argument, but the query is logged +-- first and the intended slot is unknowable, so the positional must be masked. +SELECT * FROM s3('url_posafter', access_key_id = 'ak', 'SEKRIT_SK', + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- The parser evaluates constant-expression keys, so this can be an effective session_token override; +-- the value must be masked without evaluating the key. +SELECT * FROM s3('url_exprkey', 'ak', 'SEKRIT_SAK', + concat('session_', 'token') = 'SEKRIT_EXPRTOK', + format = 'TSV', structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A nested map placed as the value of a visible non-secret key must be hidden, not formatted verbatim: +-- the value is not a plain literal, so it fails closed before the parser rejects the non-literal value. +SELECT * FROM s3('url_fmthdr', format = headers('Authorization' = 'SEKRIT_FMTHDR'), + structure = 'x UInt8'); -- { serverError BAD_ARGUMENTS, UNKNOWN_FUNCTION } + +-- Same inside extra_credentials: role_arn stays visible only for a literal/identifier value; a nested +-- map value carries a secret and must be hidden. +SELECT * FROM s3('url_rolehdr', 'ak', 'SEKRIT_SAK', + extra_credentials(role_arn = headers('Authorization' = 'SEKRIT_ROLEHDR'))); -- { serverError BAD_ARGUMENTS, UNKNOWN_FUNCTION } + +-- A `structure = ...` override given as a constant expression is evaluated by the parser, which then +-- drops the positional structure slot: the leading positional is then a secret_access_key, not a +-- format, and must be hidden even though it reads as a format name. The masker cannot evaluate the +-- key, so it fails closed and drops the positional structure slot for any unevaluable key. +SELECT * FROM s3('url_exprstruct', 'CSV', 'SEKRIT_EXPRSTRUCT', 'TSV', + concat('struc', 'ture') = 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- The URL itself can carry credentials: the userinfo and presigned-URL query parameters must be +-- masked while the host, path and non-credential parameters stay visible. The one-character bucket +-- makes S3 URI validation reject the query before any network access. +SELECT * FROM s3('https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG&partNumber=7', + 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A password may itself contain '@'; the userinfo must be masked up to the last '@' before the host, +-- not just the first, so no fragment of it stays visible. +SELECT * FROM s3('https://user:SEKRIT_PWA@SEKRIT_PWB@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG', + 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A retained URL part can legally contain an apostrophe; the partially-masked url must be re-emitted +-- as a properly escaped SQL literal (not by naive quoting, which would produce a broken statement). +SELECT * FROM s3('https://user:SEKRIT_PW@localhost:11111/x/o''clock?X-Amz-Signature=SEKRIT_SIG', + 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- A url built from a constant expression is evaluated by the parser and can embed credentials in +-- its pieces; the masker cannot evaluate it, so the whole url argument is hidden (fail closed). +SELECT * FROM s3(concat('https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=', 'SEKRIT_SIG'), + 'TSV', 'x UInt8'); -- { serverError BAD_ARGUMENTS } + +-- Same for BACKUP and the Backup database reconstructor. +BACKUP TABLE nonexistent_04510 TO S3('https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG', + 'ak', 'SEKRIT_SAK'); -- { serverError BAD_ARGUMENTS } +CREATE DATABASE db_04510_authurl ENGINE = Backup('', S3('https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG', + 'ak', 'SEKRIT_SAK')); -- { serverError BAD_ARGUMENTS } + +-- Named-collection form: an extra_credentials override alongside a collection must be masked too. +-- The collection need not exist; masking runs on the AST before the collection is resolved. +SELECT * FROM s3(nc_04510_missing, extra_credentials(external_id = 'SEKRIT_EID'), + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- A url override on a named collection can carry credentials too. +SELECT * FROM s3(nc_authurl_missing, url = 'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG', + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- Named-collection form: a headers() override must have its values masked too. +SELECT * FROM s3(nc_headers_missing, headers('Authorization' = 'SEKRIT_HDRVAL'), + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- Named-collection form: a headers() override with a malformed child must fail closed. +SELECT * FROM s3(nc_badhdr_missing, headers('Authorization: SEKRIT_RAWHDR'), + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- A positional argument swept inside a named secret span must not be echoed as a bogus key. +SELECT * FROM s3(nc_span_missing, secret_access_key = 'SEKRIT_SPAN1', 'SEKRIT_MIDPOS', + session_token = 'SEKRIT_SPAN2', + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- A constant-expression key can be an effective secret override for a named collection too. +SELECT * FROM s3(nc_exprkey_missing, concat('secret_', 'access_key') = 'SEKRIT_EXPRVAL', + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- The named-collection form permits no positional argument at all, so one placed before the first +-- named override must also be masked. +SELECT * FROM s3(nc_prepos_missing, 'SEKRIT_PREPOS', + secret_access_key = 'SEKRIT_SK', + format = 'TSV', structure = 'x UInt8'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } + +-- BACKUP ... TO S3 explicit-url form. +BACKUP TABLE nonexistent_04510 TO S3('url_bkp_named', 'ak', 'SEKRIT_SAK', + session_token = 'SEKRIT_ST', + google_adc_client_secret = 'SEKRIT_ADCCS', + google_adc_refresh_token = 'SEKRIT_ADCRT', + extra_credentials(external_id = 'SEKRIT_EID')); -- { serverError BAD_ARGUMENTS } + +-- BACKUP ... TO S3 with an invalid 4th positional argument (a session token) is rejected by the +-- backup engine, but the positional token must still be masked in the logged query text. +BACKUP TABLE nonexistent_04510 TO S3('url_bkp_pos', 'ak', 'SEKRIT_SAK', + 'SEKRIT_BACKUPTOK'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- The backup named-collection locator accepts one positional: the non-secret filename, which must +-- stay visible; any positional beyond it is invalid and must be masked. +BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, 'visible_bkp_dir', + 'SEKRIT_BKPNCPOS'); -- { serverError BAD_ARGUMENTS } + +-- The filename is collected independently of named overrides, so it stays visible after one too. +BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, + secret_access_key = 'SEKRIT_BKPORD', 'visible_bkp_dir2'); -- { serverError BAD_ARGUMENTS } + +-- An explicit-url locator with an invalid positional count (neither 1 nor 3): the intended slots +-- are unknowable, so everything after the url must be masked. +BACKUP TABLE nonexistent_04510 TO S3('url_bkp_mixed', + access_key_id = 'ak', 'SEKRIT_BKPMIX'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- Backup database engine reconstructs the nested S3 destination; extra_credentials must be masked. +CREATE DATABASE db_04510_ec ENGINE = Backup('', S3('url_dbec', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = 'SEKRIT_EID'))); -- { serverError BAD_ARGUMENTS } + +-- The reconstructor must fail closed on an invalid extra positional argument (a session token). +CREATE DATABASE db_04510_postok ENGINE = Backup('', S3('url_dbpostok', 'ak', 'SEKRIT_SAK', + 'SEKRIT_DBTOK')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- Named-collection locator in the reconstructor: the filename stays visible, but a second positional +-- is invalid and must be masked. +CREATE DATABASE db_04510_ncpos ENGINE = Backup('', S3(nc_dbnc_missing, 'visible_dbnc_dir', + 'SEKRIT_DBNCPOS')); -- { serverError BAD_ARGUMENTS } + +-- The reconstructor also keeps the filename visible when it follows a named override. +CREATE DATABASE db_04510_ncorder ENGINE = Backup('', S3(nc_dbord_missing, + secret_access_key = 'SEKRIT_DBORD', 'visible_dbnc_dir2')); -- { serverError BAD_ARGUMENTS } + +-- Non-string scalar overrides are valid and non-secret; the reconstructor keeps them visible. +CREATE DATABASE db_04510_ncenv ENGINE = Backup('', S3(nc_dbenv_missing, + secret_access_key = 'SEKRIT_DBENVKEY', use_environment_credentials = 1)); -- { serverError BAD_ARGUMENTS } + +-- A non-secret named value that is an expression (a computed filename, or a nested headers() / +-- extra_credentials() map) is accepted by the backup named-collection path, which the parser evaluates +-- as a constant. The reconstructor cannot classify it and its formatted text would carry any nested +-- secret verbatim, so it fails closed to [HIDDEN] rather than leak. +CREATE DATABASE db_04510_ncexpr ENGINE = Backup('', S3(nc_dbexpr_missing, + secret_access_key = 'SEKRIT_DBEXPRKEY', + filename = headers('Authorization' = 'SEKRIT_NESTEDHDR'))); -- { serverError BAD_ARGUMENTS } + +-- The reconstructor masks everything after the url on an invalid positional count too. +CREATE DATABASE db_04510_mixed ENGINE = Backup('', S3('url_dbmixed', + access_key_id = 'ak', 'SEKRIT_DBMIX')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A url override built from an expression can embed credentials in its pieces; the reconstructor +-- must hide it even when it is the only secret-bearing argument. +CREATE DATABASE db_04510_ncurl ENGINE = Backup('', S3(nc_dburl_missing, + url = concat('https://user:SEKRIT_PW@', 'localhost/x?X-Amz-Signature=SEKRIT_SIG'))); -- { serverError BAD_ARGUMENTS } + +-- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim. +CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_SAK', + headers('X-Auth' = 'SEKRIT_HDR'))); -- { serverError BAD_ARGUMENTS } + +-- The reconstructor must also fail closed on a constant-expression extra_credentials key. +CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', + extra_credentials(concat('extern', 'al_id') = 'SEKRIT_EXPR'))); -- { serverError BAD_ARGUMENTS } + +-- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must +-- be masked in the logged query text. +CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', + 'SEKRIT_S3DBTOK'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A valid non-secret named override (use_environment_credentials) must stay visible while +-- secret_access_key is hidden. This CREATE succeeds (the S3 database is lazy), so use a unique +-- database name to avoid collisions across parallel runs, and drop it after. +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = S3('url_dbenv', 'ak', 'SEKRIT_SAK', use_environment_credentials = 1); +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; + +-- The query-tree surface (EXPLAIN QUERY TREE) must hide the same carriers as the logged query text: +-- a credential-bearing url (masked whole, since a tree dump cannot represent partial masking), the +-- positional secrets, and the values of headers(...) / extra_credentials(...). +-- run_passes = 0 keeps the table function unresolved: the masking visitor runs before the passes +-- either way, and resolution would touch the storage (URI validation, credential checks), which +-- varies across test configurations. +SET enable_analyzer = 1; +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('https://user:SEKRIT_PW@localhost:11111/test/04510qt?X-Amz-Signature=SEKRIT_SIG', 'ak', 'SEKRIT_SAK', 'TSV', 'x UInt8'); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/04510qt', NOSIGN, 'TSV', 'x UInt8', headers('Authorization' = 'SEKRIT_HDR')); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/04510qt', 'ak', 'SEKRIT_SAK', 'TSV', 'x UInt8', extra_credentials(external_id = 'SEKRIT_EID')); + +-- Identifier-valued secrets (the parsers evaluate identifiers as literals) have no display mask of +-- their own, so the dump-only tree replaces them with a hidden constant. +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/04510qt', 'ak', 'SEKRIT_SAK', 'TSV', 'x UInt8', session_token = SEKRIT_IDTOK); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/04510qt', NOSIGN, 'TSV', 'x UInt8', headers('Authorization' = SEKRIT_BEARER)); + +-- A named url override is an `equals` node in the tree; its credential-bearing value must be hidden. +-- Without passes the collection need not exist. +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = 'https://user:SEKRIT_PW@localhost:11111/test/04510qt?X-Amz-Signature=SEKRIT_SIG', structure = 'x UInt8'); + +-- A table function can sit under a UNION (or any other carrier); the dump masking visitor must descend +-- into every node, not only query/join carriers, or the secret leaks in the tree dump. +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3('http://localhost:11111/test/04510qt', 'ak', 'SEKRIT_UNIONSAK', 'TSV', 'x UInt8') UNION ALL SELECT 1; + +SYSTEM FLUSH LOGS query_log; + +-- The exact logged text of every query above, in execution order: secrets must appear as '[HIDDEN]' +-- while every non-secret part (urls, formats, structures, filenames, non-secret overrides) stays +-- visible verbatim. Each query has exactly one terminal event: QueryFinish for the successful ones, +-- an exception event for the rejected ones. +SELECT query +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND query_kind != 'Set' -- sent by the test harness, not by this test + AND query NOT ILIKE 'SYSTEM FLUSH%' -- its own terminal event races with the flush it performs + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE +ORDER BY event_time_microseconds; diff --git a/tests/queries/0_stateless/04628_secret_args_expression_derived_key.reference b/tests/queries/0_stateless/04628_secret_args_expression_derived_key.reference new file mode 100644 index 000000000000..d8ce64e4af1b --- /dev/null +++ b/tests/queries/0_stateless/04628_secret_args_expression_derived_key.reference @@ -0,0 +1,34 @@ +QUERY id: 0 + PROJECTION COLUMNS + encrypt(\'aes-128-ecb\', [HIDDEN id: 1], [HIDDEN]) String + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: encrypt, function_type: ordinary, result_type: String + ARGUMENTS + LIST id: 3, nodes: 3 + CONSTANT id: 4, constant_value: \'aes-128-ecb\', constant_value_type: String + CONSTANT id: 5, constant_value: [HIDDEN id: 1], constant_value_type: String + FUNCTION id: 6, function_name: leftPad, function_type: ordinary, result_type: String + ARGUMENTS + LIST id: 7, nodes: 3 + CONSTANT id: 8, constant_value: [HIDDEN id: 2], constant_value_type: String + CONSTANT id: 9, constant_value: [HIDDEN id: 3], constant_value_type: UInt8 + CONSTANT id: 10, constant_value: [HIDDEN id: 4], constant_value_type: String + JOIN TREE + TABLE id: 11, alias: __table1, table_name: system.one +encrypt(\'aes-128-ecb\', [HIDDEN id: 1], [HIDDEN]) String +0 +0 +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: encrypt, function_type: ordinary + ARGUMENTS + LIST id: 3, nodes: 3 + CONSTANT id: 4, constant_value: \'aes-128-ecb\', constant_value_type: String + CONSTANT id: 5, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 6, constant_value: [HIDDEN], constant_value_type: String + JOIN TREE + IDENTIFIER id: 7, identifier: system.one +0 +0 diff --git a/tests/queries/0_stateless/04628_secret_args_expression_derived_key.sql b/tests/queries/0_stateless/04628_secret_args_expression_derived_key.sql new file mode 100644 index 000000000000..d03e5cc86e1c --- /dev/null +++ b/tests/queries/0_stateless/04628_secret_args_expression_derived_key.sql @@ -0,0 +1,40 @@ +-- Tags: no-fasttest +-- no-fasttest: encrypt requires the OpenSSL-based functions, absent in the fast test build. + +-- A secret argument of encrypt/decrypt is not always a bare literal: it can be built by an expression +-- (e.g. leftPad('...', 16, '*')). When secrets are not displayed, every constant used to derive the +-- key must be hidden, not just a direct literal, so no fragment of the key leaks. The mask is a +-- display flag, so it hides the value identically in the query-tree dump and the result column name. + +SET enable_analyzer = 1; +SET format_display_secrets_in_show_and_select = 0; + +-- Query-tree dump: the literal inside the key-deriving expression must show as [HIDDEN]. +EXPLAIN QUERY TREE SELECT encrypt('aes-128-ecb', 'plaintext', leftPad('SEKRIT_DERIVEDKEY', 16, '*')); + +-- Result column name (projection name): the derived-key literal must not appear in the header either. +DESCRIBE (SELECT encrypt('aes-128-ecb', 'plaintext', leftPad('SEKRIT_DERIVEDKEY', 16, '*'))); + +-- The ActionsDAG dump of EXPLAIN actions must hide the derived-key literal too. The pretty format +-- (the default) reads the constant value straight from the column, so it has to consult the masked +-- name instead of the raw value; the legacy format already relies on that name. viewExplain lets us +-- assert that no fragment leaks without dumping the config-dependent plan into the reference. +SELECT countIf(explain LIKE '%SEKRIT_DAGKEY%') AS pretty_dag_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', (SELECT encrypt('aes-128-ecb', materialize('plaintext'), leftPad('SEKRIT_DAGKEY', 16, '*')) FROM numbers(1))); +SELECT countIf(explain LIKE '%SEKRIT_DAGKEY%') AS legacy_dag_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 0', (SELECT encrypt('aes-128-ecb', materialize('plaintext'), leftPad('SEKRIT_DAGKEY', 16, '*')) FROM numbers(1))); + +-- EXPLAIN QUERY TREE with the passes disabled runs no analysis, so an ordinary secret function must be +-- masked by the dump itself; both the plaintext and the key literal must show as [HIDDEN]. +EXPLAIN QUERY TREE run_passes = 0 SELECT encrypt('aes-128-ecb', 'SEKRIT_PLAINTEXT', 'SEKRIT_LITERALKEY'); + +-- A secret key that reaches encrypt as a column aliased in a subquery is folded into the plan as a +-- fresh constant after the query-tree masking ran, so the planner must flag that constant; the pretty +-- ActionsDAG dump must not print it. +SELECT countIf(explain LIKE '%SEKRIT_SUBQKEY16%') AS subquery_dag_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', (SELECT encrypt('aes-128-ecb', toString(number), k) FROM (SELECT 'SEKRIT_SUBQKEY16' AS k, number FROM numbers(1)))); + +-- The folded secret key can also be an expression over several constant columns, so it is a FUNCTION +-- node carrying a constant, not a plain constant column; the plan dump must still hide it. +SELECT countIf(explain LIKE '%SEKRIT%') AS concat_dag_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', (SELECT encrypt('aes-128-ecb', toString(number), concat(k1, k2)) FROM (SELECT 'SEKRIT_C' AS k1, 'ONCKEY16' AS k2, number FROM numbers(1)))); diff --git a/tests/queries/0_stateless/04648_url_secret_masking_forms.reference b/tests/queries/0_stateless/04648_url_secret_masking_forms.reference new file mode 100644 index 000000000000..49df17fa19fc --- /dev/null +++ b/tests/queries/0_stateless/04648_url_secret_masking_forms.reference @@ -0,0 +1,131 @@ +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 4 + CONSTANT id: 5, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 6, constant_value: \'CSV\', constant_value_type: String + CONSTANT id: 7, constant_value: \'c UInt8\', constant_value_type: String + FUNCTION id: 8, function_name: headers, function_type: ordinary + ARGUMENTS + LIST id: 9, nodes: 1 + FUNCTION id: 10, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 2 + CONSTANT id: 12, constant_value: \'Authorization\', constant_value_type: String + CONSTANT id: 13, constant_value: [HIDDEN], constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: urlCluster + ARGUMENTS + LIST id: 4, nodes: 5 + CONSTANT id: 5, constant_value: \'c\', constant_value_type: String + CONSTANT id: 6, constant_value: [HIDDEN], constant_value_type: String + CONSTANT id: 7, constant_value: \'CSV\', constant_value_type: String + CONSTANT id: 8, constant_value: \'c UInt8\', constant_value_type: String + FUNCTION id: 9, function_name: headers, function_type: ordinary + ARGUMENTS + LIST id: 10, nodes: 1 + FUNCTION id: 11, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 12, nodes: 2 + CONSTANT id: 13, constant_value: \'Authorization\', constant_value_type: String + CONSTANT id: 14, constant_value: [HIDDEN], constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 3 + IDENTIFIER id: 5, identifier: nc_04648_missing + FUNCTION id: 6, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 7, nodes: 2 + IDENTIFIER id: 8, identifier: url + CONSTANT id: 9, constant_value: [HIDDEN], constant_value_type: String + FUNCTION id: 10, function_name: headers, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 1 + FUNCTION id: 12, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 13, nodes: 2 + CONSTANT id: 14, constant_value: \'Authorization\', constant_value_type: String + CONSTANT id: 15, constant_value: [HIDDEN], constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 3 + CONSTANT id: 5, constant_value: \'[HIDDEN]\', constant_value_type: String + CONSTANT id: 6, constant_value: \'CSV\', constant_value_type: String + CONSTANT id: 7, constant_value: \'c UInt8\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 2 + IDENTIFIER id: 5, identifier: nc_04648_missing + FUNCTION id: 6, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 7, nodes: 2 + IDENTIFIER id: 8, identifier: url + CONSTANT id: 9, constant_value: \'[HIDDEN]\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 2 + IDENTIFIER id: 5, identifier: nc_04648_missing + FUNCTION id: 6, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 7, nodes: 2 + FUNCTION id: 8, function_name: concat, function_type: ordinary + ARGUMENTS + LIST id: 9, nodes: 2 + CONSTANT id: 10, constant_value: \'u\', constant_value_type: String + CONSTANT id: 11, constant_value: \'rl\', constant_value_type: String + CONSTANT id: 12, constant_value: \'[HIDDEN]\', constant_value_type: String +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + TABLE_FUNCTION id: 3, table_function_name: url + ARGUMENTS + LIST id: 4, nodes: 3 + IDENTIFIER id: 5, identifier: nc_04648_missing + FUNCTION id: 6, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 7, nodes: 2 + IDENTIFIER id: 8, identifier: format + CONSTANT id: 9, constant_value: \'[HIDDEN]\', constant_value_type: String + FUNCTION id: 10, function_name: equals, function_type: ordinary + ARGUMENTS + LIST id: 11, nodes: 2 + IDENTIFIER id: 12, identifier: structure + CONSTANT id: 13, constant_value: \'c UInt8\', constant_value_type: String +CREATE TABLE default.t_04648_url\n(\n `x` UInt8\n)\nENGINE = URL(\'https://user:[HIDDEN]@localhost:11111/x\', \'CSV\', headers(\'Authorization\' = \'[HIDDEN]\')) +CREATE VIEW default.v_04648\n(\n `c` UInt8\n)\nAS SELECT *\nFROM url(\'https://user:[HIDDEN]@localhost:11111/x\', \'CSV\', \'c UInt8\') +QUERY id: 0 + PROJECTION + LIST id: 1, nodes: 1 + MATCHER id: 2, matcher_type: ASTERISK + JOIN TREE + IDENTIFIER id: 3, identifier: v_04648 diff --git a/tests/queries/0_stateless/04648_url_secret_masking_forms.sql b/tests/queries/0_stateless/04648_url_secret_masking_forms.sql new file mode 100644 index 000000000000..8ff86e0fe1c7 --- /dev/null +++ b/tests/queries/0_stateless/04648_url_secret_masking_forms.sql @@ -0,0 +1,41 @@ +-- Tags: no-fasttest +-- no-fasttest: the URL table engine and url/urlCluster table functions are not in the fast test build. + +-- Credentials in every URL carrier must be hidden when secrets are not displayed: the userinfo of the +-- url positional or a named `url = ...` override, and the `headers(...)` values, for the `url` and +-- `urlCluster` table functions (including their named-collection forms) and the `URL` table engine. +-- urlCluster puts the cluster name first, so the url is its second argument. + +SET enable_analyzer = 1; +SET format_display_secrets_in_show_and_select = 0; + +-- Function forms via the query-tree dump (run_passes = 0 keeps them unresolved, so no network access). +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url('https://user:SEKRIT_PW@localhost:11111/x', 'CSV', 'c UInt8', headers('Authorization' = 'SEKRIT_HDR')); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM urlCluster('c', 'https://user:SEKRIT_PW@localhost:11111/x', 'CSV', 'c UInt8', headers('Authorization' = 'SEKRIT_HDR')); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url(nc_04648_missing, url = 'https://user:SEKRIT_PW@localhost:11111/x?token=SEKRIT_TOK', headers('Authorization' = 'SEKRIT_HDR')); + +-- A url built from a constant expression is evaluated by the parser but not by the masker, so both a +-- positional and a named `url` override must fail closed (hidden whole) rather than leak the pieces. +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url(concat('https://user:', 'SEKRIT_EXPR@localhost:11111/x'), 'CSV', 'c UInt8'); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url(nc_04648_missing, url = concat('https://user:', 'SEKRIT_NAMEDEXPR@localhost:11111/x')); + +-- A named-collection override key evaluated from a constant expression can name `url`, and a nested +-- map placed as the value of any visible non-url override (format, description, ...) carries a secret; +-- both are formatted before the collection is validated, so both must fail closed. +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url(nc_04648_missing, concat('u', 'rl') = concat('https://user:', 'SEKRIT_EXPRKEY@localhost/x')); +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM url(nc_04648_missing, format = headers('Authorization' = 'SEKRIT_URLFMTHDR'), structure = 'c UInt8'); + +-- The URL table engine: SHOW CREATE must hide the userinfo and the headers values. +DROP TABLE IF EXISTS t_04648_url; +CREATE TABLE t_04648_url (x UInt8) +ENGINE = URL('https://user:SEKRIT_PW@localhost:11111/x', 'CSV', headers('Authorization' = 'SEKRIT_HDR')); +SHOW CREATE TABLE t_04648_url SETTINGS format_display_secrets_in_show_and_select = 0; +DROP TABLE t_04648_url; + +-- A view over a URL source keeps its definition masked in SHOW CREATE, and querying the view logs +-- only the view name (the credential is never in the logged query text). +DROP TABLE IF EXISTS v_04648; +CREATE VIEW v_04648 AS SELECT * FROM url('https://user:SEKRIT_PW@localhost:11111/x', 'CSV', 'c UInt8'); +SHOW CREATE TABLE v_04648 SETTINGS format_display_secrets_in_show_and_select = 0; +EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM v_04648; +DROP TABLE v_04648; diff --git a/tmp/check_files.sh b/tmp/check_files.sh new file mode 100644 index 000000000000..bacbbf4a5b81 --- /dev/null +++ b/tmp/check_files.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e +FILES=( + src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp + src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h + src/Analyzer/Resolve/resolveFunction.cpp + src/Databases/DatabaseS3.cpp + src/Databases/DatabaseS3.h + src/Interpreters/ActionsDAG.cpp + src/Interpreters/ActionsDAG.h + src/Interpreters/InterpreterExplainQuery.cpp + src/Parsers/ASTFunction.cpp + src/Parsers/FunctionSecretArgumentsFinderAST.h + src/Planner/PlannerActionsVisitor.cpp + src/Processors/QueryPlan/QueryPlanFormat.cpp + tests/queries/0_stateless/02968_url_args.reference + tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference + tests/queries/0_stateless/04343_secret_args_finder_mixed_named_positional.reference + tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference + tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql + tests/queries/0_stateless/04628_secret_args_expression_derived_key.reference + tests/queries/0_stateless/04628_secret_args_expression_derived_key.sql + tests/queries/0_stateless/04648_url_secret_masking_forms.reference + tests/queries/0_stateless/04648_url_secret_masking_forms.sql +) +for f in "${FILES[@]}"; do + if git show 1e0acd6a91a0c82d20bad23f0e19785698289de7:"$f" > /tmp/master_ver_check 2>/dev/null; then + if diff -q /tmp/master_ver_check "$f" > /dev/null 2>&1; then + echo "MATCH: $f" + else + echo "DIFFERS: $f" + fi + else + echo "MISSING IN MASTER?: $f" + fi +done From 11227e6463bc0321f33190b3eaafbbe73c4a8589 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 22:52:26 +0000 Subject: [PATCH 52/86] Backport #111806 to 26.6: Expose named collections storage type in server settings --- docs/en/operations/named-collections.md | 10 ++++ .../system-tables/server_settings.md | 2 +- src/Core/ServerSettings.cpp | 9 ++++ .../test_named_collections/test.py | 53 +++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md index 00b7d312f116..54b411ef9194 100644 --- a/docs/en/operations/named-collections.md +++ b/docs/en/operations/named-collections.md @@ -98,6 +98,16 @@ To use ZooKeeper/Keeper we also need to set up a `path` (path in ZooKeeper/Keepe An optional configuration parameter `update_timeout_ms` by default is equal to `5000`. +You can inspect the active storage type through `system.server_settings` and `getServerSetting`: + +```sql +SELECT value, getServerSetting('named_collections_storage_type') +FROM system.server_settings +WHERE name = 'named_collections_storage.type'; +``` + +Changing the storage type requires a server restart; `SYSTEM RELOAD CONFIG` does not change the active backend. + ## Storing named collections in configuration files {#storing-named-collections-in-configuration-files} ### XML example {#xml-example} diff --git a/docs/en/operations/system-tables/server_settings.md b/docs/en/operations/system-tables/server_settings.md index ab30982fa800..955a91d52a2a 100644 --- a/docs/en/operations/system-tables/server_settings.md +++ b/docs/en/operations/system-tables/server_settings.md @@ -14,7 +14,7 @@ import SystemTableCloud from '@site/docs/_snippets/_system_table_cloud.md'; ## Description {#description} Contains information about global settings for the server, which are specified in `config.xml`. -Currently, the table shows only settings from the first layer of `config.xml` and doesn't support nested configs (e.g. [logger](../../operations/server-configuration-parameters/settings.md#logger)). +The table also includes supported nested settings with a fixed structure; dynamic sections such as lists are not included. ## Columns {#columns} diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 2aa39554953e..af0ab749801b 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1662,6 +1662,11 @@ The policy on how to perform a scheduling of CPU slots specified by `concurrent_ /// Settings with a path are server settings with at least one layer of nesting that have a fixed structure (no lists, lists, enumerations, repetitions, ...). #define LIST_OF_SERVER_SETTINGS_WITH_PATH(DECLARE, ALIAS) \ + DECLARE(String, named_collections_storage_type, "local", R"( +The storage type for named collections. Possible values are `local`, `local_encrypted`, `keeper`, +`keeper_encrypted`, `zookeeper`, and `zookeeper_encrypted`. +Configured as `named_collections_storage.type` (`` in XML). +)", 0, "named_collections_storage.type") \ DECLARE(UInt64, query_cache_max_size_in_bytes, 1073741824, R"(The maximum cache size in bytes. 0 means the query cache is disabled.)", 0, "query_cache.max_size_in_bytes") \ DECLARE(UInt64, query_cache_max_entries, 1024, R"(The maximum number of SELECT query results stored in the cache.)", 0, "query_cache.max_entries") \ DECLARE(UInt64, query_cache_max_entry_size_in_bytes, 1048576, R"(The maximum size in bytes SELECT query results may have to be saved in the cache.)", 0, "query_cache.max_entry_size_in_bytes") \ @@ -1850,6 +1855,10 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {"max_server_memory_usage", {std::to_string(total_memory_tracker.getHardLimit()), ChangeableWithoutRestart::Yes}}, {"min_allocation_size_to_throw_on_memory_limit", {std::to_string(CurrentMemoryTracker::getMinAllocationSizeBytesToThrow()), ChangeableWithoutRestart::Yes}}, + /// Named collections metadata storage is initialized once, so use its effective startup type. + {"named_collections_storage_type", + {context->getServerSettingsCopy()[ServerSetting::named_collections_storage_type].toString(), ChangeableWithoutRestart::No}}, + {"max_table_size_to_drop", {std::to_string(context->getMaxTableSizeToDrop()), ChangeableWithoutRestart::Yes}}, {"max_named_collection_num_to_warn", {std::to_string(context->getMaxNamedCollectionNumToWarn()), ChangeableWithoutRestart::Yes}}, {"max_table_num_to_warn", {std::to_string(context->getMaxTableNumToWarn()), ChangeableWithoutRestart::Yes}}, diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 0b6d759a5387..65edc1e581dd 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -508,6 +508,59 @@ def test_config_reload(cluster): ) +def test_storage_type_does_not_change_on_config_reload(cluster): + node = cluster.instances["node"] + + def get_storage_type_state(instance): + return instance.query( + """ + SELECT name, value, default, changed, type, changeable_without_restart, + getServerSetting('named_collections_storage_type') + FROM system.server_settings + WHERE name = 'named_collections_storage.type' + """ + ).strip() + + assert ( + "named_collections_storage.type\tlocal\tlocal\t0\tString\tNo\tlocal" + == get_storage_type_state(node) + ) + assert ( + "named_collections_storage.type\tzookeeper\tlocal\t1\tString\tNo\tzookeeper" + == get_storage_type_state(cluster.instances["node_with_keeper"]) + ) + + config = """ + + + keeper + /named_collections_reload_test + + +""" + + with node.with_replace_config( + "/etc/clickhouse-server/config.d/named_collections.xml", + config, + reload_before=True, + reload_after=True, + ): + assert ( + "named_collections_storage.type\tlocal\tlocal\t1\tString\tNo\tlocal" + == get_storage_type_state(node) + ) + + node.query("CREATE NAMED COLLECTION storage_type_reload_test AS value = 1") + assert "1" == node.query( + """ + SELECT collection['value'] + FROM system.named_collections + WHERE name = 'storage_type_reload_test' + """ + ).strip() + node.query("DROP NAMED COLLECTION storage_type_reload_test") + + @pytest.mark.parametrize("with_keeper", [False, True]) def test_sql_commands(cluster, with_keeper): zk = None From 6ce177208d7332cb69950e9695231f2d46c2a1a5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 28 Jul 2026 22:53:54 +0000 Subject: [PATCH 53/86] Backport #111721 to 26.6: Fix NOT_FOUND_COLUMN_IN_BLOCK on FINAL when the sort key filter is moved to PREWHERE and not projected --- .../QueryPlan/ReadFromMergeTree.cpp | 14 +++- ..._final_sorting_key_not_projected.reference | 9 +++ ...ewhere_final_sorting_key_not_projected.sql | 75 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.reference create mode 100644 tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.sql diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 8602db328b77..d2b8853b2148 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1530,10 +1530,16 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( } /// Returns the list of column names required for the transforms in addMergingFinal -static NameSet getColumnsRequiredForMergingFinal(const SortDescription & sort_description, MergeTreeData::MergingParams merging_params) +static NameSet getColumnsRequiredForMergingFinal( + const SortDescription & sort_description, const StorageMetadataPtr & metadata_snapshot, MergeTreeData::MergingParams merging_params) { NameSet required_columns = sort_description | std::views::transform([](const SortColumnDescription & desc) { return desc.column_name; }) | std::ranges::to(); + /// The merge always orders by the physical sorting key, so those columns must be read even when + /// they are not in the query output (e.g. a sorting-key column moved to PREWHERE and pruned from + /// the output header would otherwise be dropped, leaving the merge without its key column). + for (const auto & column : metadata_snapshot->getColumnsRequiredForFinal()) + required_columns.insert(column); switch (merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: @@ -4678,7 +4684,8 @@ bool ReadFromMergeTree::canRemoveUnusedColumns() const if (query_info.isFinal()) { // Cannot remove columns if FINAL requires them for merging - NameSet required_for_final = getColumnsRequiredForMergingFinal(result_sort_description, data.merging_params); + NameSet required_for_final + = getColumnsRequiredForMergingFinal(result_sort_description, storage_snapshot->metadata, data.merging_params); const auto has_column_that_is_not_required_for_final = std::ranges::any_of(all_column_names, [&](const auto & column_name) { return !required_for_final.contains(column_name); }); @@ -4699,7 +4706,8 @@ ReadFromMergeTree::RemoveUnusedColumnsResult ReadFromMergeTree::removeUnusedColu std::set required_storage_column_positions; if (query_info.isFinal()) { - const auto required_for_final = getColumnsRequiredForMergingFinal(result_sort_description, data.merging_params); + const auto required_for_final + = getColumnsRequiredForMergingFinal(result_sort_description, storage_snapshot->metadata, data.merging_params); for (size_t pos = 0; pos < output_header->columns(); ++pos) { diff --git a/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.reference b/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.reference new file mode 100644 index 000000000000..632ae8ab3c82 --- /dev/null +++ b/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.reference @@ -0,0 +1,9 @@ +300 +1 +300 +300 +1 +1 +3 +7 +300 diff --git a/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.sql b/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.sql new file mode 100644 index 000000000000..9b3bf9d35024 --- /dev/null +++ b/tests/queries/0_stateless/04627_prewhere_final_sorting_key_not_projected.sql @@ -0,0 +1,75 @@ +-- Regression test for issue #111709: a FINAL query filtering on a sorting-key column that is +-- not in the SELECT list must not throw NOT_FOUND_COLUMN_IN_BLOCK when the filter is in PREWHERE. + +SET enable_analyzer = 1; +SET optimize_move_to_prewhere = 1; +SET optimize_move_to_prewhere_if_final = 1; +-- Pin both prewhere-related optimizations (the runner disables each with 5% probability): +-- query_plan_optimize_prewhere keeps the EXPLAIN assertion stable, and query_plan_remove_unused_columns +-- must stay on or the pruning path this fix touches is skipped and the test stops guarding the fix. +SET query_plan_optimize_prewhere = 1; +SET query_plan_remove_unused_columns = 1; + +DROP TABLE IF EXISTS t_04627_summing; +CREATE TABLE t_04627_summing (k UInt32, s Int64) +ENGINE = SummingMergeTree ORDER BY k SETTINGS optimize_on_insert = 0; +SYSTEM STOP MERGES t_04627_summing; +INSERT INTO t_04627_summing SELECT number % 10 + 1, 3 FROM numbers(50); +INSERT INTO t_04627_summing SELECT number % 10 + 1, 3 FROM numbers(50); + +-- The failing query from the issue: k is filtered but not projected. Must return sum(s) = 300. +SELECT sum(s) FROM t_04627_summing FINAL WHERE k GROUP BY s; +-- Must match the result with the optimization disabled (returns 1). +SELECT (SELECT sum(s) FROM t_04627_summing FINAL WHERE k GROUP BY s) + = (SELECT sum(s) FROM t_04627_summing FINAL WHERE k GROUP BY s SETTINGS optimize_move_to_prewhere_if_final = 0); +-- Explicit PREWHERE on the unprojected key hits the same merge-key pruning path. +SELECT sum(s) FROM t_04627_summing FINAL PREWHERE k GROUP BY s; +-- The old planner (enable_analyzer = 0) shares the pruning path too. +SELECT sum(s) FROM t_04627_summing FINAL WHERE k GROUP BY s SETTINGS enable_analyzer = 0; +-- The optimization must still fire: the filter on k is moved to PREWHERE (returns 1). +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT s FROM t_04627_summing FINAL WHERE k GROUP BY s) +WHERE explain ILIKE '%Prewhere filter column:%k%'; + +DROP TABLE t_04627_summing; + +-- Composite sorting key: filter on the second key column (not projected) must keep both a and b +-- readable for the merge. Returns 1 (FINAL result matches the optimization-disabled result). +DROP TABLE IF EXISTS t_04627_composite; +CREATE TABLE t_04627_composite (a UInt32, b UInt32, val Int64) +ENGINE = SummingMergeTree ORDER BY (a, b) SETTINGS optimize_on_insert = 0; +SYSTEM STOP MERGES t_04627_composite; +INSERT INTO t_04627_composite SELECT number % 10, number % 7 + 1, 5 FROM numbers(100); +INSERT INTO t_04627_composite SELECT number % 10, number % 7 + 1, 5 FROM numbers(100); +SELECT (SELECT groupArray(c) FROM (SELECT sum(val) AS c FROM t_04627_composite FINAL WHERE b GROUP BY val ORDER BY val)) + = (SELECT groupArray(c) FROM (SELECT sum(val) AS c FROM t_04627_composite FINAL WHERE b GROUP BY val ORDER BY val SETTINGS optimize_move_to_prewhere_if_final = 0)); +DROP TABLE t_04627_composite; + +-- CoalescingMergeTree shares the same FINAL merge path. +DROP TABLE IF EXISTS t_04627_coalescing; +CREATE TABLE t_04627_coalescing (k UInt32, s Int64) +ENGINE = CoalescingMergeTree ORDER BY k SETTINGS optimize_on_insert = 0; +SYSTEM STOP MERGES t_04627_coalescing; +INSERT INTO t_04627_coalescing SELECT number % 10 + 1, 3 FROM numbers(50); +INSERT INTO t_04627_coalescing SELECT number % 10 + 1, 3 FROM numbers(50); +SELECT DISTINCT s FROM t_04627_coalescing FINAL WHERE k GROUP BY s; +DROP TABLE t_04627_coalescing; + +-- ReplacingMergeTree shares the same FINAL merge path. +DROP TABLE IF EXISTS t_04627_replacing; +CREATE TABLE t_04627_replacing (k UInt32, v UInt32, s Int64) +ENGINE = ReplacingMergeTree(v) ORDER BY k SETTINGS optimize_on_insert = 0; +SYSTEM STOP MERGES t_04627_replacing; +INSERT INTO t_04627_replacing SELECT number % 10 + 1, 1, 3 FROM numbers(50); +INSERT INTO t_04627_replacing SELECT number % 10 + 1, 2, 7 FROM numbers(50); +SELECT DISTINCT s FROM t_04627_replacing FINAL WHERE k GROUP BY s; +DROP TABLE t_04627_replacing; + +-- AggregatingMergeTree (SimpleAggregateFunction) shares the same FINAL merge path. +DROP TABLE IF EXISTS t_04627_aggregating; +CREATE TABLE t_04627_aggregating (k UInt32, s SimpleAggregateFunction(sum, Int64)) +ENGINE = AggregatingMergeTree ORDER BY k SETTINGS optimize_on_insert = 0; +SYSTEM STOP MERGES t_04627_aggregating; +INSERT INTO t_04627_aggregating SELECT number % 10 + 1, 3 FROM numbers(50); +INSERT INTO t_04627_aggregating SELECT number % 10 + 1, 3 FROM numbers(50); +SELECT sum(s) FROM t_04627_aggregating FINAL WHERE k GROUP BY s; +DROP TABLE t_04627_aggregating; From ae02502c97add63d3417a1eb09992162220ac280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 29 Jul 2026 10:31:31 +0000 Subject: [PATCH 54/86] Fix test failure on 26.6 backport of #109768 04510_s3_explicit_url_named_secret_mask exercised use_environment_credentials on the S3 database engine, which this branch's DatabaseS3 does not support; dropped that assertion. https://github.com/ClickHouse/ClickHouse/pull/112323 --- .../04510_s3_explicit_url_named_secret_mask.reference | 3 --- .../04510_s3_explicit_url_named_secret_mask.sql | 7 ------- 2 files changed, 10 deletions(-) diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index 8a939435ccdc..702fa6365824 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -208,9 +208,6 @@ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') -DROP DATABASE IF EXISTS default_1 -CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) -DROP DATABASE default_1 EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\', extra_credentials(external_id = \'[HIDDEN]\')) diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 7d1c627caeac..54a01c7e532a 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -285,13 +285,6 @@ CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', 'SEKRIT_S3DBTOK'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } --- A valid non-secret named override (use_environment_credentials) must stay visible while --- secret_access_key is hidden. This CREATE succeeds (the S3 database is lazy), so use a unique --- database name to avoid collisions across parallel runs, and drop it after. -DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; -CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = S3('url_dbenv', 'ak', 'SEKRIT_SAK', use_environment_credentials = 1); -DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; - -- The query-tree surface (EXPLAIN QUERY TREE) must hide the same carriers as the logged query text: -- a credential-bearing url (masked whole, since a tree dump cannot represent partial masking), the -- positional secrets, and the values of headers(...) / extra_credentials(...). From 9a1e602fba5bba7b69872aa500e025678f97ac5c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 29 Jul 2026 12:47:22 +0000 Subject: [PATCH 55/86] Backport #111991 to 26.6: Fix test 04627_analyzer_compatibility_final_all_joined_tables with old analyzer --- .../04627_analyzer_compatibility_final_all_joined_tables.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql index 9e93ef62edfb..412c2d994eb3 100644 --- a/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql +++ b/tests/queries/0_stateless/04627_analyzer_compatibility_final_all_joined_tables.sql @@ -1,6 +1,11 @@ -- Compatibility setting for the fix of FINAL leaking onto other tables of a JOIN -- (https://github.com/ClickHouse/ClickHouse/pull/108979). +-- The setting `analyzer_compatibility_apply_final_to_all_joined_tables` has an effect only when +-- the analyzer is enabled. The old analyzer has different semantics of FINAL in JOIN (it ignores +-- FINAL on the right table), so pin the analyzer explicitly. +SET enable_analyzer = 1; + DROP TABLE IF EXISTS t_left; DROP TABLE IF EXISTS t_right; From 042c6ba03abc9261c6b86b7225cc730f2b72eab0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 29 Jul 2026 14:34:47 +0000 Subject: [PATCH 56/86] Backport #110584 to 26.6: Fix IS NULL after metadata-only T to Nullable(T) ALTER with optimize_functions_to_subcolumns --- src/Interpreters/getColumnFromBlock.cpp | 23 +++- src/Interpreters/inplaceBlockConversions.cpp | 12 +- src/Interpreters/inplaceBlockConversions.h | 3 +- src/Storages/MergeTree/IMergeTreeReader.cpp | 19 ++- src/Storages/MergeTree/IMergeTreeReader.h | 6 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 6 + .../MergeTree/MergeTreeReadersChain.cpp | 7 +- .../02941_variant_type_alters.reference | 88 ++++++------ .../0_stateless/02941_variant_type_alters.sh | 1 - ...ter_metadata_only_nullable_alter.reference | 36 +++++ ...umn_after_metadata_only_nullable_alter.sql | 126 ++++++++++++++++++ 11 files changed, 267 insertions(+), 60 deletions(-) create mode 100644 tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference create mode 100644 tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql diff --git a/src/Interpreters/getColumnFromBlock.cpp b/src/Interpreters/getColumnFromBlock.cpp index 4f301329b559..4627582b8955 100644 --- a/src/Interpreters/getColumnFromBlock.cpp +++ b/src/Interpreters/getColumnFromBlock.cpp @@ -46,21 +46,32 @@ ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requ return nullptr; auto subcolumn_name = requested_subcolumn.getSubcolumnName(); - /// If requested subcolumn is dynamic, we should first perform cast and then - /// extract the subcolumn, because the data of dynamic subcolumn can change after cast. - if ((elem->type->hasDynamicStructure() || requested_column_type->hasDynamicStructure()) && !elem->type->equals(*requested_column_type)) + bool is_dynamic = elem->type->hasDynamicStructure() || requested_column_type->hasDynamicStructure(); + + /// Cast the parent to the requested type first, then extract, when types differ and either the + /// subcolumn is dynamic (its data can change after cast) or the block's (older) type lacks it + /// (metadata-only `ALTER MODIFY COLUMN T -> Nullable(T)`). Otherwise the subcolumn is readable + /// from the block directly, so extract it below without casting the whole parent. + auto source_column = elem->column->decompress()->convertToFullColumnIfConst(); + + bool block_type_has_subcolumn = elem->type->tryGetSubcolumnType(subcolumn_name) != nullptr; + if (!elem->type->equals(*requested_column_type) && (is_dynamic || !block_type_has_subcolumn)) { - auto cast_column = castColumn({elem->column->decompress(), elem->type, ""}, requested_column_type); + auto cast_column = castColumn({source_column, elem->type, ""}, requested_column_type); auto elem_column = requested_column_type->tryGetSubcolumn(subcolumn_name, cast_column); auto elem_type = requested_column_type->tryGetSubcolumnType(subcolumn_name); if (!elem_type || !elem_column) return nullptr; - return elem_column; + /// Dynamic subcolumn data already matches after the cast; an extra cast could alter it. + if (is_dynamic) + return elem_column; + + return castColumn({elem_column, elem_type, ""}, requested_subcolumn.type); } - auto elem_column = elem->type->tryGetSubcolumn(subcolumn_name, elem->column->decompress()); + auto elem_column = elem->type->tryGetSubcolumn(subcolumn_name, source_column); auto elem_type = elem->type->tryGetSubcolumnType(subcolumn_name); if (!elem_type || !elem_column) diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index fc6751d06621..124b8fe9fa18 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -423,7 +423,8 @@ void fillMissingColumns( const NamesAndTypesList & available_columns, const NameSet & partially_read_columns, StorageSnapshotPtr storage_snapshot, - bool share_nested_offsets) + bool share_nested_offsets, + const NameSet & additional_available_columns) { size_t num_columns = requested_columns.size(); if (num_columns != res_columns.size()) @@ -452,6 +453,15 @@ void fillMissingColumns( if (res_columns[i] || hasDefault(storage_snapshot, *requested_column)) continue; + /// Subcolumn missing from the part's (older) type but whose parent is available (read here + /// or produced by an earlier step): defer to evaluateMissingDefaults instead of default- + /// filling. Needs a storage_snapshot, i.e. a caller that runs that pass (not Memory engine). + if (storage_snapshot + && requested_column->isSubcolumn() + && (available_columns.contains(requested_column->getNameInStorage()) + || additional_available_columns.contains(requested_column->getNameInStorage()))) + continue; + std::vector current_offsets; size_t num_dimensions = 0; diff --git a/src/Interpreters/inplaceBlockConversions.h b/src/Interpreters/inplaceBlockConversions.h index ebb79c70c7f1..1f2d566e5448 100644 --- a/src/Interpreters/inplaceBlockConversions.h +++ b/src/Interpreters/inplaceBlockConversions.h @@ -45,6 +45,7 @@ void fillMissingColumns( const NamesAndTypesList & available_columns, const NameSet & partially_read_columns, StorageSnapshotPtr storage_snapshot, - bool share_nested_offsets = true); + bool share_nested_offsets = true, + const NameSet & additional_available_columns = {}); } diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 3a77a9f13fb6..accc7b5bc419 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -152,7 +153,9 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const } } -void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) const +void IMergeTreeReader::fillMissingColumns( + Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows, + const NameSet & previous_step_columns) const { try { @@ -187,7 +190,8 @@ void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_e : available_columns, partially_read_columns, storage_snapshot, - share_nested); + share_nested, + previous_step_columns); should_evaluate_missing_defaults = std::any_of(res_columns.begin(), res_columns.end(), [](const auto & column) { return column == nullptr; }); @@ -292,12 +296,17 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns } auto name_in_storage = it->getNameInStorage(); - res_columns[pos] = additional_columns.getByName(name_in_storage).column; if (it->isSubcolumn()) { - const auto & type_in_storage = it->getTypeInStorage(); - res_columns[pos] = type_in_storage->getSubcolumn(it->getSubcolumnName(), res_columns[pos]); + /// The parent may still be in its pre-`ALTER MODIFY` type here (an earlier on-fly step + /// is not converted by performRequiredConversions); tryGetSubcolumnFromBlock casts it + /// to the storage type before extracting. + res_columns[pos] = tryGetSubcolumnFromBlock(additional_columns, it->getTypeInStorage(), *it); + } + else + { + res_columns[pos] = additional_columns.getByName(name_in_storage).column; } } } diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index ea86fbac76e8..eb9c8b37882e 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -60,7 +60,11 @@ class IMergeTreeReader : private boost::noncopyable /// Add columns from ordered_names that are not present in the block. /// Missing columns are added in the order specified by ordered_names. /// num_rows is needed in case if all res_columns are nullptr. - void fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) const; + /// `previous_step_columns` names columns produced by earlier reader-chain steps; a subcolumn + /// whose parent is among them is deferred to evaluateMissingDefaults instead of default-filled. + void fillMissingColumns( + Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows, + const NameSet & previous_step_columns = {}) const; /// Evaluate defaulted columns if necessary. void evaluateMissingDefaults(Block additional_columns, Columns & res_columns) const; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 77e1ba52b808..1f44e124d73d 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -123,6 +123,12 @@ bool injectRequiredColumnsRecursively( add_column(column_name); return true; } + + /// Parent is present but the part's (older) type lacks the requested subcolumn (metadata-only + /// `ALTER MODIFY COLUMN T -> Nullable(T)`). Read the parent so it can be converted and the + /// subcolumn extracted from it, instead of being filled from the storage-type default. + add_column(column_in_storage->getNameInStorage()); + return true; } else if (isTextIndexVirtualColumn(column_name_in_part) && hasMaterializedTextIndex(storage_snapshot, data_part_info_for_reader, column_name_in_part)) { diff --git a/src/Storages/MergeTree/MergeTreeReadersChain.cpp b/src/Storages/MergeTree/MergeTreeReadersChain.cpp index 8024c4de198a..c9809a97337e 100644 --- a/src/Storages/MergeTree/MergeTreeReadersChain.cpp +++ b/src/Storages/MergeTree/MergeTreeReadersChain.cpp @@ -378,8 +378,13 @@ void MergeTreeReadersChain::executeActionsBeforePrewhere( /// fillMissingColumns() must be called after reading but before any filterings because /// some columns (e.g. arrays) might be only partially filled and thus not be valid and /// fillMissingColumns() fixes this. + /// Names of columns produced by earlier chain steps (advertised in `previous_header`), so a + /// subcolumn whose parent is among them is deferred to evaluateMissingDefaults, not default-filled. + NameSet previous_step_columns; + for (const auto & col : previous_header) + previous_step_columns.insert(col.name); bool should_evaluate_missing_defaults = false; - merge_tree_reader->fillMissingColumns(read_columns, should_evaluate_missing_defaults, num_read_rows); + merge_tree_reader->fillMissingColumns(read_columns, should_evaluate_missing_defaults, num_read_rows, previous_step_columns); if (result.total_rows_per_granule != num_read_rows) { diff --git a/tests/queries/0_stateless/02941_variant_type_alters.reference b/tests/queries/0_stateless/02941_variant_type_alters.reference index 52c834e455bf..7e402108591a 100644 --- a/tests/queries/0_stateless/02941_variant_type_alters.reference +++ b/tests/queries/0_stateless/02941_variant_type_alters.reference @@ -60,51 +60,51 @@ insert after alter modify column 1 20 20 20 \N 20 \N 21 21 str_21 str_21 \N \N alter modify column 2 -0 0 \N \N \N \N \N \N -1 1 \N \N \N \N \N \N -2 2 \N \N \N \N \N \N -3 3 \N \N 3 \N 3 \N -4 4 \N \N 4 \N 4 \N -5 5 \N \N 5 \N 5 \N -6 6 \N \N str_6 str_6 \N \N -7 7 \N \N str_7 str_7 \N \N -8 8 \N \N str_8 str_8 \N \N -9 9 \N \N \N \N \N \N -10 10 \N \N \N \N \N \N -11 11 \N \N \N \N \N \N -12 12 \N \N 12 \N 12 \N -13 13 \N \N str_13 str_13 \N \N -14 14 \N \N \N \N \N \N -15 15 \N \N 1970-01-16 \N \N 1970-01-16 -16 16 \N \N 1970-01-17 \N \N 1970-01-17 -17 17 \N \N 1970-01-18 \N \N 1970-01-18 -18 18 \N \N 1970-01-19 \N \N 1970-01-19 -19 19 \N \N \N \N \N \N -20 20 \N \N 20 \N 20 \N -21 21 \N \N str_21 str_21 \N \N +0 0 0 \N \N \N \N \N +1 1 1 \N \N \N \N \N +2 2 2 \N \N \N \N \N +3 3 3 \N 3 \N 3 \N +4 4 4 \N 4 \N 4 \N +5 5 5 \N 5 \N 5 \N +6 6 6 \N str_6 str_6 \N \N +7 7 7 \N str_7 str_7 \N \N +8 8 8 \N str_8 str_8 \N \N +9 9 9 \N \N \N \N \N +10 10 10 \N \N \N \N \N +11 11 11 \N \N \N \N \N +12 12 12 \N 12 \N 12 \N +13 13 13 \N str_13 str_13 \N \N +14 14 14 \N \N \N \N \N +15 15 15 \N 1970-01-16 \N \N 1970-01-16 +16 16 16 \N 1970-01-17 \N \N 1970-01-17 +17 17 17 \N 1970-01-18 \N \N 1970-01-18 +18 18 18 \N 1970-01-19 \N \N 1970-01-19 +19 19 19 \N \N \N \N \N +20 20 20 \N 20 \N 20 \N +21 21 21 \N str_21 str_21 \N \N insert after alter modify column 2 -0 0 \N \N \N \N \N \N -1 1 \N \N \N \N \N \N -2 2 \N \N \N \N \N \N -3 3 \N \N 3 \N 3 \N -4 4 \N \N 4 \N 4 \N -5 5 \N \N 5 \N 5 \N -6 6 \N \N str_6 str_6 \N \N -7 7 \N \N str_7 str_7 \N \N -8 8 \N \N str_8 str_8 \N \N -9 9 \N \N \N \N \N \N -10 10 \N \N \N \N \N \N -11 11 \N \N \N \N \N \N -12 12 \N \N 12 \N 12 \N -13 13 \N \N str_13 str_13 \N \N -14 14 \N \N \N \N \N \N -15 15 \N \N 1970-01-16 \N \N 1970-01-16 -16 16 \N \N 1970-01-17 \N \N 1970-01-17 -17 17 \N \N 1970-01-18 \N \N 1970-01-18 -18 18 \N \N 1970-01-19 \N \N 1970-01-19 -19 19 \N \N \N \N \N \N -20 20 \N \N 20 \N 20 \N -21 21 \N \N str_21 str_21 \N \N +0 0 0 \N \N \N \N \N +1 1 1 \N \N \N \N \N +2 2 2 \N \N \N \N \N +3 3 3 \N 3 \N 3 \N +4 4 4 \N 4 \N 4 \N +5 5 5 \N 5 \N 5 \N +6 6 6 \N str_6 str_6 \N \N +7 7 7 \N str_7 str_7 \N \N +8 8 8 \N str_8 str_8 \N \N +9 9 9 \N \N \N \N \N +10 10 10 \N \N \N \N \N +11 11 11 \N \N \N \N \N +12 12 12 \N 12 \N 12 \N +13 13 13 \N str_13 str_13 \N \N +14 14 14 \N \N \N \N \N +15 15 15 \N 1970-01-16 \N \N 1970-01-16 +16 16 16 \N 1970-01-17 \N \N 1970-01-17 +17 17 17 \N 1970-01-18 \N \N 1970-01-18 +18 18 18 \N 1970-01-19 \N \N 1970-01-19 +19 19 19 \N \N \N \N \N +20 20 20 \N 20 \N 20 \N +21 21 21 \N str_21 str_21 \N \N 22 str_22 \N str_22 \N \N \N \N 23 \N \N \N \N \N \N \N 24 24 24 \N \N \N \N \N diff --git a/tests/queries/0_stateless/02941_variant_type_alters.sh b/tests/queries/0_stateless/02941_variant_type_alters.sh index d93e7d82a4b9..54735b66b8f2 100755 --- a/tests/queries/0_stateless/02941_variant_type_alters.sh +++ b/tests/queries/0_stateless/02941_variant_type_alters.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash # Tags: long, memory-engine -# memory-engine: minor data inconsistency after alter CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference new file mode 100644 index 000000000000..9ec7614951e0 --- /dev/null +++ b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference @@ -0,0 +1,36 @@ +compact, DEFAULT NULL +0 +0 +42 0 +compact, no default +1 +1 +1 +1 +1 +1 +1 42 0 +2 \N 1 +wide +0 +0 +42 0 +string +1 +1 +1 a 0 +2 \N 1 +memory +1 +1 +1 +1 +1 42 0 +2 \N 1 +apply_mutations_on_fly +1 +1 +1 0 0 +2 \N 1 +tuple element +7 diff --git a/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql new file mode 100644 index 000000000000..5dd8b9c667ef --- /dev/null +++ b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql @@ -0,0 +1,126 @@ +DROP TABLE IF EXISTS t_null_sub_evolved; + +-- A part written before a metadata-only `T` -> `Nullable(T)` ALTER stores non-nullable +-- data and has no `.null` substream. Reading the `.null` subcolumn (as done by +-- optimize_functions_to_subcolumns for isNull/isNotNull/count) must derive it from the +-- present parent column, giving null-map = 0 for the existing rows, not the storage-type +-- default (NULL). + +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); + +ALTER TABLE t_null_sub_evolved + MODIFY COLUMN x Nullable(UInt256) DEFAULT NULL +SETTINGS mutations_sync = 2; + +SELECT 'compact, DEFAULT NULL'; +SELECT count() FROM t_null_sub_evolved WHERE id = 1 AND x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE id = 1 AND x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- No DEFAULT, add a genuine NULL in a new part; check isNull/isNotNull/count. +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(UInt256) SETTINGS mutations_sync = 2; +INSERT INTO t_null_sub_evolved VALUES (2, NULL); + +SELECT 'compact, no default'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_null_sub_evolved WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count(x) FROM t_null_sub_evolved SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count(x) FROM t_null_sub_evolved SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- Wide part. +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(UInt256) SETTINGS mutations_sync = 2; + +SELECT 'wide'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- String -> Nullable(String) evolution. +CREATE TABLE t_null_sub_evolved (id UInt8, x String) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 'a'); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(String) SETTINGS mutations_sync = 2; +INSERT INTO t_null_sub_evolved VALUES (2, NULL); + +SELECT 'string'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- Memory engine: MODIFY COLUMN is metadata-only (in-RAM blocks are not rewritten), so a block +-- inserted before the ALTER holds non-nullable data with no `.null` substream. Reading `.null` +-- must derive it from the converted parent (null-map = 0), not crash and not return NULL. +DROP TABLE IF EXISTS t_null_sub_mem; +CREATE TABLE t_null_sub_mem (id UInt8, x UInt256) ENGINE = Memory; + +INSERT INTO t_null_sub_mem VALUES (1, 42); +ALTER TABLE t_null_sub_mem MODIFY COLUMN x Nullable(UInt256); +INSERT INTO t_null_sub_mem VALUES (2, NULL); + +SELECT 'memory'; +SELECT count() FROM t_null_sub_mem WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_mem WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_null_sub_mem WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count(x) FROM t_null_sub_mem SETTINGS optimize_functions_to_subcolumns = 1; +SELECT id, x, x.null FROM t_null_sub_mem ORDER BY id; + +DROP TABLE t_null_sub_mem; + +-- apply_mutations_on_fly: an on-fly UPDATE produces the full `x` in an earlier pipeline step, +-- then a metadata-only MODIFY COLUMN makes it Nullable. The later step reads only `x.null`; it +-- must be derived from the parent produced by the on-fly step (null-map = 0), not default-filled +-- to all-NULL (issue #110555 corner case). +DROP TABLE IF EXISTS t_null_sub_amof; +CREATE TABLE t_null_sub_amof (id UInt8, x UInt8) +ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_amof VALUES (1, 5); +SYSTEM STOP MERGES t_null_sub_amof; +ALTER TABLE t_null_sub_amof UPDATE x = 0 WHERE 1 SETTINGS mutations_sync = 0; +ALTER TABLE t_null_sub_amof MODIFY COLUMN x Nullable(UInt8) SETTINGS mutations_sync = 0, alter_sync = 0; +INSERT INTO t_null_sub_amof VALUES (2, NULL); + +SELECT 'apply_mutations_on_fly'; +SELECT count() FROM t_null_sub_amof WHERE x IS NULL SETTINGS apply_mutations_on_fly = 1, optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_amof WHERE x IS NULL SETTINGS apply_mutations_on_fly = 1, optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_amof ORDER BY id SETTINGS apply_mutations_on_fly = 1; + +DROP TABLE t_null_sub_amof; + +-- Tuple subcolumn that STILL exists in the old block type after a sibling element's +-- metadata-only type change. Reading `t.a` must extract it directly and must NOT cast the whole +-- tuple (which would throw on the non-convertible old `b` value). Memory engine keeps the +-- pre-ALTER block, so this exercises the old-type-still-has-subcolumn path. +DROP TABLE IF EXISTS t_tuple_elem_mem; +CREATE TABLE t_tuple_elem_mem (id UInt8, t Tuple(a UInt8, b String)) ENGINE = Memory; + +INSERT INTO t_tuple_elem_mem VALUES (1, (7, 'x')); +ALTER TABLE t_tuple_elem_mem MODIFY COLUMN t Tuple(a UInt8, b UInt64); + +SELECT 'tuple element'; +SELECT t.a FROM t_tuple_elem_mem; + +DROP TABLE t_tuple_elem_mem; From eff1307ecc5ca00f2257a828cf31f0207fc81aba Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 29 Jul 2026 15:31:37 +0000 Subject: [PATCH 57/86] Backport #109615 to 26.6: Fix reading Parquet Tuple column as Nullable(Tuple) --- .../Formats/Impl/Parquet/SchemaConverter.cpp | 20 ++++++++++++ .../0_stateless/00900_long_parquet_load_2.sh | 4 +++ ..._list_map_wrapper_nullable_tuple.reference | 7 +++++ ...ptional_list_map_wrapper_nullable_tuple.sh | 29 ++++++++++++++++++ ...nside_nullable_parquet_roundtrip.reference | 23 ++++++++++++-- ...uple_inside_nullable_parquet_roundtrip.sql | 24 +++++++++++++-- ...onal_list_wrapper_required_element.parquet | Bin 0 -> 660 bytes ...ptional_map_wrapper_required_value.parquet | Bin 0 -> 961 bytes .../04065_optional_struct_under_list.parquet | Bin 0 -> 727 bytes 9 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference create mode 100755 tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh create mode 100644 tests/queries/0_stateless/data_parquet/04065_optional_list_wrapper_required_element.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04065_optional_map_wrapper_required_value.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04065_optional_struct_under_list.parquet diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index e29f7bd4daf5..2c6d294d0ab3 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -613,6 +613,26 @@ void SchemaConverter::processSubtreeTuple(TraversalNode & node) /// `name2` /// ... + /// The requested type may wrap the tuple in Nullable (e.g. `Nullable(Tuple(...))` is a legal + /// type). Unwrap it, match elements against the inner Tuple, and let the outer wrapper be + /// restored via outer_type_hint (needs_cast) in processSubtree. + /// Only unwrap when the tuple is always defined (REQUIRED group, no optional struct-group + /// ancestor), so the restored outer Nullable is always-non-null and lossless. Only Nullable + /// levels nested below the innermost array count: a Nullable level at or before it is the + /// optional wrapper of a LIST/MAP, whose nulls are normalized to empty collections by + /// processRepDefLevelsForArray and never reach the inner tuple null-map. + size_t innermost_array_idx = 0; + for (size_t i = 0; i < levels.size(); ++i) + if (levels[i].is_array) + innermost_array_idx = i; + bool has_optional_ancestor = false; + for (size_t i = innermost_array_idx + 1; i < levels.size(); ++i) + has_optional_ancestor |= !levels[i].is_array; + if (node.type_hint && node.type_hint->isNullable() + && node.element->repetition_type == parq::FieldRepetitionType::REQUIRED + && !has_optional_ancestor) + node.type_hint = assert_cast(*node.type_hint).getNestedType(); + const DataTypeTuple * tuple_type_hint = typeid_cast(node.type_hint.get()); if (node.type_hint && !tuple_type_hint && !typeid_cast(node.type_hint.get())) throw Exception(ErrorCodes::TYPE_MISMATCH, "Requested type of column {} doesn't match parquet schema: parquet type is Tuple, requested type is {}", node.getNameForLogging(), node.type_hint->getName()); diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 1b5a2e3513d6..5031de6a4dfc 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -43,6 +43,10 @@ EXCLUDE=( 04045_delta_sample_93093.parquet # Hand-crafted file for testing dictionary memory estimation with sparse nullables. 04099_dict_nullable_string_memory.parquet + # Schema fixtures for 04065 Nullable(Tuple) wrapper test (LIST/MAP optional wrappers). + 04065_optional_list_wrapper_required_element.parquet + 04065_optional_map_wrapper_required_value.parquet + 04065_optional_struct_under_list.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference new file mode 100644 index 000000000000..9c2f0a06b963 --- /dev/null +++ b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference @@ -0,0 +1,7 @@ +-- optional LIST wrapper, REQUIRED element group: Array(Nullable(Tuple)) accepted (always-defined) +[(1),(2)] Array(Nullable(Tuple(x UInt32))) +[(3)] Array(Nullable(Tuple(x UInt32))) +-- optional MAP wrapper, REQUIRED value group: Map(String, Nullable(Tuple)) accepted +{'k1':(1),'k2':(2)} Map(String, Nullable(Tuple(x UInt32))) +-- optional element group under a list: genuine struct-level nulls, still rejected +TYPE_MISMATCH diff --git a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh new file mode 100755 index 000000000000..b5909b9d2fc9 --- /dev/null +++ b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: Parquet format is not available in fasttest builds + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Parquet fixtures whose LIST/MAP wrapper is OPTIONAL but the inner element/value group is +# REQUIRED. ClickHouse's own writer only emits REQUIRED list wrappers, so these come from an +# external writer (pyarrow) and are checked in under data_parquet/. +# +# The optional wrapper's nulls are normalized to empty collections by the reader and never reach +# the inner tuple null-map, so an always-defined REQUIRED element/value read as Nullable(Tuple) +# is lossless and must be accepted (issue #109605 follow-up). A genuinely OPTIONAL inner group +# carries real struct-level nulls and must still be rejected. + +DATA="$CURDIR/data_parquet" + +opts="--enable_nullable_tuple_type=1 --allow_experimental_nullable_tuple_type=1" + +echo "-- optional LIST wrapper, REQUIRED element group: Array(Nullable(Tuple)) accepted (always-defined)" +$CLICKHOUSE_LOCAL $opts -q "SELECT a, toTypeName(a) FROM file('$DATA/04065_optional_list_wrapper_required_element.parquet', 'Parquet', 'a Array(Nullable(Tuple(x UInt32)))')" + +echo "-- optional MAP wrapper, REQUIRED value group: Map(String, Nullable(Tuple)) accepted" +$CLICKHOUSE_LOCAL $opts -q "SELECT m, toTypeName(m) FROM file('$DATA/04065_optional_map_wrapper_required_value.parquet', 'Parquet', 'm Map(String, Nullable(Tuple(x UInt32)))')" + +echo "-- optional element group under a list: genuine struct-level nulls, still rejected" +$CLICKHOUSE_LOCAL $opts -q "SELECT a FROM file('$DATA/04065_optional_struct_under_list.parquet', 'Parquet', 'a Array(Nullable(Tuple(inner Tuple(x UInt32))))')" 2>&1 | grep -o "TYPE_MISMATCH" | head -1 diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference index 8f1bf22310f9..a5758ea71f26 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference @@ -94,13 +94,21 @@ INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Pa -- Parquet V3 native reader multi col (not yet supported) SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_multi; --- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) +-- Type hint mismatch: file has a REQUIRED Tuple group, read as Nullable(Tuple(...)) (issue #109605). +-- A REQUIRED group is always defined, so the outer Nullable is always-non-null and reading it as +-- Nullable(Tuple) is lossless and now supported. DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; --- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader: read a non-nullable (REQUIRED group) file as Nullable(Tuple) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); +(1,'a') Nullable(Tuple(UInt32, String)) +(2,'b') Nullable(Tuple(UInt32, String)) +-- Same REQUIRED group, requested as Nullable(Tuple) with named, reordered subset of elements +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(`2` String, `1` UInt32))'); +('a',1) Nullable(Tuple(`2` String, `1` UInt32)) +('b',2) Nullable(Tuple(`2` String, `1` UInt32)) DROP TABLE test_nullable_tuple_mismatch2; -- Schema inference: inferred type with toTypeName DROP TABLE IF EXISTS test_nullable_tuple_describe; @@ -150,3 +158,12 @@ SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet' hello LowCardinality(Nullable(String)) world LowCardinality(Nullable(String)) DROP TABLE test_nullable_tuple_lc_string; +-- REQUIRED inner group under an OPTIONAL outer group, inner subcolumn requested as Nullable(Tuple). +-- The outer group's definition-level nulls are not propagated to the inner Nullable, so accepting +-- the hint would silently drop those nulls. Must reject rather than lose data. +DROP TABLE IF EXISTS test_nullable_tuple_opt_ancestor; +CREATE TABLE test_nullable_tuple_opt_ancestor (c0 Nullable(Tuple(inner Tuple(a UInt32)))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_opt_ancestor VALUES (((1,),)), (NULL), (((3,),)); +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_opt_ancestor; +SELECT `c0.inner` FROM file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet', '`c0.inner` Nullable(Tuple(a UInt32))'); -- { serverError TYPE_MISMATCH } +DROP TABLE test_nullable_tuple_opt_ancestor; diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql index 327fb913e874..55e3e2fdca66 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql @@ -138,15 +138,20 @@ SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', DROP TABLE test_nullable_tuple_multi; --- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) +-- Type hint mismatch: file has a REQUIRED Tuple group, read as Nullable(Tuple(...)) (issue #109605). +-- A REQUIRED group is always defined, so the outer Nullable is always-non-null and reading it as +-- Nullable(Tuple) is lossless and now supported. DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; --- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader: read a non-nullable (REQUIRED group) file as Nullable(Tuple) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); + +-- Same REQUIRED group, requested as Nullable(Tuple) with named, reordered subset of elements +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(`2` String, `1` UInt32))'); DROP TABLE test_nullable_tuple_mismatch2; @@ -210,3 +215,16 @@ INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_lc_str.parquet', 'P SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))'); DROP TABLE test_nullable_tuple_lc_string; + +-- REQUIRED inner group under an OPTIONAL outer group, inner subcolumn requested as Nullable(Tuple). +-- The outer group's definition-level nulls are not propagated to the inner Nullable, so accepting +-- the hint would silently drop those nulls. Must reject rather than lose data. +DROP TABLE IF EXISTS test_nullable_tuple_opt_ancestor; +CREATE TABLE test_nullable_tuple_opt_ancestor (c0 Nullable(Tuple(inner Tuple(a UInt32)))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_opt_ancestor VALUES (((1,),)), (NULL), (((3,),)); + +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_opt_ancestor; + +SELECT `c0.inner` FROM file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet', '`c0.inner` Nullable(Tuple(a UInt32))'); -- { serverError TYPE_MISMATCH } + +DROP TABLE test_nullable_tuple_opt_ancestor; diff --git a/tests/queries/0_stateless/data_parquet/04065_optional_list_wrapper_required_element.parquet b/tests/queries/0_stateless/data_parquet/04065_optional_list_wrapper_required_element.parquet new file mode 100644 index 0000000000000000000000000000000000000000..24e89f2e6b328d425d7cd646a35a1358e6f81f40 GIT binary patch literal 660 zcmaJ<$w~u35UpvOF^3!yp@(jmLk>D9i7Uo~Bw$SvO>j+cLGcp95Chpv)c9SV#ILie z2EmgBT}#({wKRQ&on51iHQ|}@4XglUOCA6yI!km6n1Wd?Q7%Wv53S}~s~Ho(`kyGD zr>k5=h>EqZ)Y2*#?H8Fh_fL%_8$tW3fWYij1 z?*jtuLKj958s*C1=Umh43gZ{+eHaz^!KW8|d(lyp`F})LqHU=-FP0-y*@(1q2Q)1& zL`8Izp+ew7o5fD&^6oGl+!nF=8KUeYuTe#B=s)h%L4h|5}O~ ze09T$JhFY}H@OZ;o586x&Z#JorKpgkYnm5*&r@7n^HgnC46k=&G$Ex>p-wm#bBmMP z`NQ4r@M$cqC@FXf+B`2w59@4*oHpg4B_JA-L|pO2ZAvTJZ^3+mq>n&=SeC|W6>MSJ roxQvzubaK;bhA5~O%{dsWHuj6#)ZB8t(~nM7l!owMfx>Y^dEl!z88b$ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04065_optional_map_wrapper_required_value.parquet b/tests/queries/0_stateless/data_parquet/04065_optional_map_wrapper_required_value.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ae97480953275d782dac0e779478843a09ac5830 GIT binary patch literal 961 zcmaJ=U279T6umRMh6n)@afTUKBnVw9Zfg^o5=Ft;q$Xm-WTh?mBAc*mC7U*;A8GR| zeDu*rAAIo9pWvfE$$Mw5iHZ*7-kE#OJ@J%#{*f9fcgqRJ&)!urbVU%EPL zfGV7f4wFeNI5fCo6Tt=V1t`&Q)%9_5`Q|*%&Jy8*D6N5VFtP?ck8n`ea*TuEh_MTh zTzfk3!4oCjfI?X}*<>j~$%0E)zGw0S^Pl*|-)Gmf6V#%~%IF@+qV_r0JGcleNU}Va z?XJq^U9bx){fWsh%>T~(kIeh;LjEdtccqm4$|d)U@*4;3sgjAJmj@5?J*m&meL9OY zG{y|@N9VaQa}%cKsWGW(r8X8K;RW4V#ylgB?<_#Yx z$yU@R*{V5i?U1@?FR-Zxb83y~?a1=2$*EMjW$Rl#Gs1V64UeMB_+WowI+TAeUFO@- zi1f3SWxpXo-Pec1?tVu2q>oum%5T!MWtpL!Bkz}@cHVHCZnzKJ3(h;w_dzl9%37Rz zKscU}X_Ld|mm+(%sSd?|tsbvbs&kqp1EQ|1&X~ubs?&J(?kqXo7)_@e@oaWHubwBf blefo5)%y14*5;ND@N=dwy5a!z@z47M?$x|A literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04065_optional_struct_under_list.parquet b/tests/queries/0_stateless/data_parquet/04065_optional_struct_under_list.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cbf128c666989087d6ef42b64e56b442892abdf6 GIT binary patch literal 727 zcmaJ47x9ma=J!nwyQN@60Jxr>tqGDSSAGcaUEVMDEYWkZz`n&uQ z&g>HKRDA~kD)dQ^4Y{@@JxY7h^Oeg{<#N6*HvV|a z%M_}$1f3O+iz>(9RD^W(Bn0KnMlbPnC Date: Wed, 29 Jul 2026 18:12:23 +0000 Subject: [PATCH 58/86] Backport #112297 to 26.6: Snapshot backup entries should not use backup io thread --- src/Backups/BackupImpl.cpp | 19 ++++++++++--------- src/Backups/BackupImpl.h | 1 + src/Backups/BackupsWorker.cpp | 7 +++++++ src/Backups/IBackup.h | 2 ++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 03bbdc15d2a7..3d1791a9126c 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -1065,15 +1065,6 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) if (entry->isReference()) return; - if (entry->isFromRemoteFile()) - { - LOG_TRACE(log, "Writing backup for file {} : skipped because of lightweight snapshot", info.data_file_name); - std::lock_guard lock{mutex}; - original_endpoint = entry->getEndpointURI(); - original_namespace = entry->getNamespace(); - return; - } - if (open_mode == OpenMode::READ) throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for reading. Something is wrong internally"); @@ -1194,6 +1185,16 @@ void BackupImpl::setCompressedSize() } +void BackupImpl::setOriginalEndpointAndNamespaceIfEmpty(const String & endpoint_, const String & namespace_) noexcept +{ + if (original_endpoint.empty()) + { + original_endpoint = endpoint_; + original_namespace = namespace_; + } +} + + bool BackupImpl::setIsCorrupted() noexcept { try diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 553ed09ca450..21e2d3e78273 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -89,6 +89,7 @@ class BackupImpl : public IBackup void finalizeWriting() override; bool setIsCorrupted() noexcept override; bool tryRemoveAllFiles() noexcept override; + void setOriginalEndpointAndNamespaceIfEmpty(const String & endpoint_, const String & namespace_) noexcept override; private: void open(); diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index a78dc4a7c09b..50d028b7920a 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -791,6 +791,13 @@ void BackupsWorker::writeBackupEntries( size_t index = !writing_order.empty() ? writing_order[i] : i; auto & entry = backup_entries[index].second; + + if (entry->isFromRemoteFile()) + { + backup->setOriginalEndpointAndNamespaceIfEmpty(entry->getEndpointURI(), entry->getNamespace()); + continue; + } + const auto & file_info = file_infos[index]; /// Using references here is fine as the variables reference objects either belonging to `this` or passed as references in the diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h index acca401964ce..b73891e2b466 100644 --- a/src/Backups/IBackup.h +++ b/src/Backups/IBackup.h @@ -122,6 +122,8 @@ class IBackup : public std::enable_shared_from_this /// Puts a new entry to the backup. virtual void writeFile(const BackupFileInfo & file_info, BackupEntryPtr entry) = 0; + virtual void setOriginalEndpointAndNamespaceIfEmpty(const String & endpoint_, const String & namespace_) noexcept = 0; + /// Whether it's possible to add new entries to the backup in multiple threads. virtual bool supportsWritingInMultipleThreads() const = 0; From d244f8e3b0b45680bcbdda698abdb318090e0ef0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 08:33:36 +0000 Subject: [PATCH 59/86] Backport #112010 to 26.6: Fix Paimon background refresh crash on concurrent LATEST hint rewrite --- .../DataLakes/Paimon/PaimonClient.cpp | 45 ++-- .../Paimon/tests/gtest_paimon_latest_hint.cpp | 198 ++++++++++++++++++ 2 files changed, 229 insertions(+), 14 deletions(-) create mode 100644 src/Storages/ObjectStorage/DataLakes/Paimon/tests/gtest_paimon_latest_hint.cpp diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp index adeda37789e1..28a552717e89 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -18,9 +19,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -52,6 +55,10 @@ extern const int FILE_DOESNT_EXIST; extern const int CANNOT_PARSE_NUMBER; } +/// A valid `LATEST` contains one positive decimal `Int64` and fits well within this limit. +/// Use the same value as the minimum read buffer size so it is read in one underlying request. +constexpr size_t PAIMON_HINT_FILE_SIZE = 64; + PaimonSnapshot::PaimonSnapshot(const Poco::JSON::Object::Ptr & json_object) { Paimon::getValueFromJSON(id, json_object, "id"); @@ -167,32 +174,41 @@ std::optional> PaimonTableClient::getLatestTableSnapsho /// try to read latest hint Int64 snapshot_version{-1}; String latest_snapshot_path; - RelativePathWithMetadata relative_path_with_metadata( - std::filesystem::path(table_location) / PAIMON_SNAPSHOT_DIR / PAIMON_SNAPSHOT_LATEST_HINT); + String latest_hint_path = std::filesystem::path(table_location) / PAIMON_SNAPSHOT_DIR / PAIMON_SNAPSHOT_LATEST_HINT; + StoredObject latest_hint_object(latest_hint_path); try { - if (object_storage->exists(StoredObject(relative_path_with_metadata.relative_path))) + if (object_storage->exists(latest_hint_object)) { - auto buf = createReadBuffer(relative_path_with_metadata, object_storage, getContext(), log); - String hint_version_string; - readStringUntilEOF(hint_version_string, *buf); + /// Do not utilize filesystem cache if more precise cache enabled. + /// This mirrors the Iceberg pattern in `StatelessMetadataFileGetter.cpp`. + auto read_settings = getContext()->getReadSettings(); + read_settings.enable_filesystem_cache = false; + read_settings.local_fs_settings.buffer_size + = std::max(read_settings.local_fs_settings.buffer_size, PAIMON_HINT_FILE_SIZE); + read_settings.remote_fs_settings.buffer_size + = std::max(read_settings.remote_fs_settings.buffer_size, PAIMON_HINT_FILE_SIZE); + + auto hint_data + = object_storage->readSmallObjectAndGetObjectMetadata(latest_hint_object, read_settings, PAIMON_HINT_FILE_SIZE); + const String & hint_version_string = hint_data.data; { - auto [_, ec] - = std::from_chars(hint_version_string.data(), hint_version_string.data() + hint_version_string.size(), snapshot_version); - if (ec != std::errc()) + const auto * end = hint_version_string.data() + hint_version_string.size(); + auto [ptr, ec] = std::from_chars(hint_version_string.data(), end, snapshot_version); + if (ec != std::errc() || ptr != end || snapshot_version <= 0 || snapshot_version == std::numeric_limits::max()) { throw Exception( ErrorCodes::CANNOT_PARSE_NUMBER, "The Paimon snapshot hint file content: {} is invalid.", hint_version_string); } } latest_snapshot_path - = std::filesystem::path(table_location) / (PAIMON_SNAPSHOT_DIR) / (PAIMON_SNAPSHOT_PREFIX + std::to_string(snapshot_version)); + = std::filesystem::path(table_location) / PAIMON_SNAPSHOT_DIR / (PAIMON_SNAPSHOT_PREFIX + std::to_string(snapshot_version)); } } catch (...) { - LOG_WARNING(log, "Failed to read Paimon LATEST hint file, falling back to snapshot listing: {}", - getCurrentExceptionMessage(false)); + LOG_WARNING( + log, "Failed to read Paimon LATEST hint file, falling back to snapshot listing: {}", getCurrentExceptionMessage(false)); snapshot_version = -1; latest_snapshot_path.clear(); } @@ -201,10 +217,11 @@ std::optional> PaimonTableClient::getLatestTableSnapsho if (!latest_snapshot_path.empty()) { Int64 next_snapshot_version = snapshot_version + 1; - StoredObject store_object( + StoredObject snapshot_object(latest_snapshot_path); + StoredObject next_snapshot_object( std::filesystem::path(table_location) / (PAIMON_SNAPSHOT_DIR) / (PAIMON_SNAPSHOT_PREFIX + std::to_string(next_snapshot_version))); - if (!object_storage->exists(store_object)) + if (object_storage->exists(snapshot_object) && !object_storage->exists(next_snapshot_object)) { return std::make_pair(snapshot_version, latest_snapshot_path); } diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/tests/gtest_paimon_latest_hint.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/tests/gtest_paimon_latest_hint.cpp new file mode 100644 index 000000000000..4efd213c46ee --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/tests/gtest_paimon_latest_hint.cpp @@ -0,0 +1,198 @@ +#include + +#include + +#if USE_AVRO + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace DB; + +namespace +{ + +struct ScopedTempDir +{ + fs::path path; + + explicit ScopedTempDir(const std::string & name) + : path(fs::temp_directory_path() / (name + "_" + std::to_string(::getpid()))) + { + std::error_code ec; + fs::remove_all(path, ec); + fs::create_directories(path); + } + + ~ScopedTempDir() + { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +void writeFile(const fs::path & path, const std::string & contents) +{ + fs::create_directories(path.parent_path()); + std::ofstream out; + out.exceptions(std::ios::failbit | std::ios::badbit); + out.open(path, std::ios::binary | std::ios::trunc); + out << contents; + out.close(); +} + +void replaceFileAtomically(const fs::path & path, const std::string & contents, size_t sequence) +{ + fs::path temporary_path(path.string() + ".tmp." + std::to_string(sequence)); + writeFile(temporary_path, contents); + fs::rename(temporary_path, path); +} + +fs::path makePaimonTable(const fs::path & root, const std::vector & snapshot_ids, const std::string & latest_hint) +{ + auto table = root / "test.db" / "test_table"; + for (Int64 snapshot_id : snapshot_ids) + { + writeFile(table / Paimon::PAIMON_SNAPSHOT_DIR / (std::string(Paimon::PAIMON_SNAPSHOT_PREFIX) + std::to_string(snapshot_id)), "{}"); + } + writeFile(table / Paimon::PAIMON_SNAPSHOT_DIR / Paimon::PAIMON_SNAPSHOT_LATEST_HINT, latest_hint); + return table; +} + +class CountingLocalObjectStorage : public LocalObjectStorage +{ +public: + using LocalObjectStorage::LocalObjectStorage; + + SmallObjectDataWithMetadata readSmallObjectAndGetObjectMetadata( + const StoredObject & object, + const ReadSettings & read_settings, + size_t max_size_bytes, + std::optional read_hint) const override + { + small_object_reads.fetch_add(1, std::memory_order_relaxed); + last_local_buffer_size.store(read_settings.local_fs_settings.buffer_size, std::memory_order_relaxed); + last_remote_buffer_size.store(read_settings.remote_fs_settings.buffer_size, std::memory_order_relaxed); + last_max_size_bytes.store(max_size_bytes, std::memory_order_relaxed); + return IObjectStorage::readSmallObjectAndGetObjectMetadata(object, read_settings, max_size_bytes, read_hint); + } + + size_t getSmallObjectReads() const { return small_object_reads.load(std::memory_order_relaxed); } + size_t getLastLocalBufferSize() const { return last_local_buffer_size.load(std::memory_order_relaxed); } + size_t getLastRemoteBufferSize() const { return last_remote_buffer_size.load(std::memory_order_relaxed); } + size_t getLastMaxSizeBytes() const { return last_max_size_bytes.load(std::memory_order_relaxed); } + +private: + mutable std::atomic small_object_reads{0}; + mutable std::atomic last_local_buffer_size{0}; + mutable std::atomic last_remote_buffer_size{0}; + mutable std::atomic last_max_size_bytes{0}; +}; + +std::shared_ptr makeLocalObjectStorage(const fs::path & root) +{ + return std::make_shared( + LocalObjectStorageSettings("test_paimon_latest_hint", root.string(), /*read_only_=*/false)); +} + +} + +TEST(PaimonLatestHint, ReadsConcurrentlyReplacedHintAsSmallObject) +{ + ScopedTempDir temporary_directory("ch_gtest_paimon_latest_hint"); + auto table = makePaimonTable(temporary_directory.path, {9, 10}, "9"); + auto hint_path = table / Paimon::PAIMON_SNAPSHOT_DIR / Paimon::PAIMON_SNAPSHOT_LATEST_HINT; + + auto context = Context::createCopy(getContext().context); + context->setSetting("remote_filesystem_read_method", String("threadpool")); + context->setSetting("remote_filesystem_read_prefetch", Field(true)); + + auto object_storage = makeLocalObjectStorage(temporary_directory.path); + PaimonTableClient client(object_storage, table.string(), context); + + std::atomic stop{false}; + std::exception_ptr writer_exception; + { + std::thread writer( + [&] + { + try + { + size_t sequence = 0; + while (!stop.load(std::memory_order_relaxed)) + { + replaceFileAtomically(hint_path, (sequence % 2 == 0) ? "10" : "9", sequence); + ++sequence; + } + } + catch (...) + { + writer_exception = std::current_exception(); + stop.store(true, std::memory_order_relaxed); + } + }); + + SCOPE_EXIT({ + stop.store(true, std::memory_order_relaxed); + writer.join(); + }); + + for (size_t iteration = 0; iteration < 2000; ++iteration) + { + auto snapshot_info = client.getLatestTableSnapshotInfo(); + ASSERT_TRUE(snapshot_info.has_value()); + EXPECT_EQ(snapshot_info->first, 10); + EXPECT_TRUE(fs::exists(snapshot_info->second)); + } + } + + if (writer_exception) + { + try + { + std::rethrow_exception(writer_exception); + } + catch (const std::exception & exception) + { + FAIL() << "Hint writer failed: " << exception.what(); + } + } + + EXPECT_GT(object_storage->getSmallObjectReads(), 0); + EXPECT_EQ(object_storage->getLastMaxSizeBytes(), 64); + EXPECT_GE(object_storage->getLastLocalBufferSize(), 64); + EXPECT_GE(object_storage->getLastRemoteBufferSize(), 64); +} + +TEST(PaimonLatestHint, FallsBackToListingForInvalidHint) +{ + ScopedTempDir temporary_directory("ch_gtest_paimon_invalid_latest_hint"); + auto table = makePaimonTable(temporary_directory.path, {1, 2}, "3trailing"); + + auto object_storage = makeLocalObjectStorage(temporary_directory.path); + PaimonTableClient client(object_storage, table.string(), getContext().context); + + auto snapshot_info = client.getLatestTableSnapshotInfo(); + ASSERT_TRUE(snapshot_info.has_value()); + EXPECT_EQ(snapshot_info->first, 2); + EXPECT_TRUE(fs::exists(snapshot_info->second)); + EXPECT_EQ(object_storage->getSmallObjectReads(), 1); +} + +#endif From f8f30ff5274a07496af6fafdfd312c30e05c2708 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 09:39:14 +0000 Subject: [PATCH 60/86] Backport #109898 to 26.6: Parquet v3: read a physically nullable struct as Nullable(Tuple) --- .../Formats/Impl/Parquet/Reader.cpp | 45 +++++++- src/Processors/Formats/Impl/Parquet/Reader.h | 22 ++++ .../Formats/Impl/Parquet/SchemaConverter.cpp | 95 +++++++++++++++-- .../Formats/Impl/Parquet/SchemaConverter.h | 5 + .../0_stateless/00900_long_parquet_load_2.sh | 1 + ..._list_map_wrapper_nullable_tuple.reference | 4 +- ...ptional_list_map_wrapper_nullable_tuple.sh | 16 ++- ...nside_nullable_parquet_roundtrip.reference | 97 +++++++++++++++--- ...uple_inside_nullable_parquet_roundtrip.sql | 81 ++++++++++++--- ...al_struct_nullable_leaf_under_list.parquet | Bin 0 -> 737 bytes 10 files changed, 316 insertions(+), 50 deletions(-) create mode 100644 tests/queries/0_stateless/data_parquet/04065_optional_struct_nullable_leaf_under_list.parquet diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index d94b6ae42b81..e0422d65ce2d 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -1459,7 +1460,7 @@ void Reader::decodePrimitiveColumn(ColumnChunk & column, const PrimitiveColumnIn throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid repetition/definition levels for arrays in column {}", column_info.name); } - if (subchunk.null_map && !column_info.output_nullable && !options.format.null_as_default) + if (subchunk.null_map && !column_info.output_nullable && !column_info.group_nullable && !options.format.null_as_default) { const auto & null_map = assert_cast(*subchunk.null_map).getData(); /// null_map uses standard ClickHouse convention: 1 = NULL, 0 = NOT NULL. @@ -1472,9 +1473,23 @@ void Reader::decodePrimitiveColumn(ColumnChunk & column, const PrimitiveColumnIn if (subchunk.null_map) { const auto & null_map = assert_cast(*subchunk.null_map).getData(); + /// Fill defaults at null rows so the column reaches full size. For a group_nullable leaf, + /// the null map is the group null map: defaults fill the struct-null rows. subchunk.column->expand(null_map, /*inverted*/ true); } + if (column_info.group_nullable && subchunk.null_map) + { + /// Leaf of a physically-nullable struct read as Nullable(Tuple(...)): its def-level null map + /// is the group null map. Move it aside now, before the output_nullable block below can + /// consume `null_map` into a leaf-level ColumnNullable. formOutputColumn reads it from the + /// group's first leaf to wrap the assembled ColumnTuple in ColumnNullable. If the leaf is + /// itself Nullable, it gets a fresh all-non-null map below (the file leaf is REQUIRED, so it + /// has no element-level nulls; the struct nulls are represented by the outer ColumnNullable). + subchunk.group_null_map = std::move(subchunk.null_map); + subchunk.null_map.reset(); + } + if (subchunk.arrays_offsets.empty() && subchunk.column->size() != row_subgroup.filter.rows_pass) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected number of rows in column subchunk {} {}", subchunk.column->size(), row_subgroup.filter.rows_pass); @@ -2146,7 +2161,26 @@ MutableColumnPtr Reader::formOutputColumn(RowSubgroup & row_subgroup, size_t out return res; } - TypeIndex kind = output_info.input_type->getColumnType(); + /// Physically-nullable struct read as Nullable(Tuple(...)). input_type is Nullable(Tuple), but + /// we assemble the inner ColumnTuple from the leaves and then wrap it in ColumnNullable using + /// the group null map. Every leaf shares the same def-level null map (the subtree is + /// all-REQUIRED), which decodePrimitiveColumn moved into `group_null_map` on each leaf before + /// any leaf-level Nullable wrapping could consume it. Take it from the first leaf. Dispatch on + /// the unwrapped type. + MutableColumnPtr nullable_group_null_map; + if (output_info.nullable_group) + { + ColumnSubchunk & first_leaf = row_subgroup.columns.at(output_info.primitive_start); + if (first_leaf.group_null_map) + nullable_group_null_map = IColumn::mutate(std::move(first_leaf.group_null_map)); + else + /// No struct-level nulls (all rows defined): all-non-null map. + nullable_group_null_map = ColumnUInt8::create(num_rows, UInt8(0)); + } + + TypeIndex kind = output_info.nullable_group + ? removeNullable(output_info.input_type)->getColumnType() + : output_info.input_type->getColumnType(); if (output_info.is_primitive) { @@ -2206,6 +2240,13 @@ MutableColumnPtr Reader::formOutputColumn(RowSubgroup & row_subgroup, size_t out res = ColumnMap::create(std::move(nested)); } + if (output_info.nullable_group) + { + /// Wrap the assembled ColumnTuple in ColumnNullable using the reconstructed group null map. + chassert(nullable_group_null_map->size() == res->size()); + res = ColumnNullable::create(std::move(res), std::move(nullable_group_null_map)); + } + chassert(res->getDataType() == output_info.input_type->getColumnType()); if (output_info.needs_cast) diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 53b598aa5524..0ac46ac11f31 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -161,6 +161,13 @@ struct Reader DataTypePtr decoded_type; // what decoder outputs, not Nullable DataTypePtr output_type; // maybe Nullable bool output_nullable = false; + /// This leaf is inside a Tuple group that is requested as Nullable(Tuple(...)) and is + /// eligible for it (the OPTIONAL group has no optional/nullable ancestor and an all-REQUIRED, + /// non-array subtree). Then this leaf's definition-level null map is exactly the group's null + /// map. We keep that null map (instead of throwing CANNOT_INSERT_NULL) and fill defaults at + /// the null rows; the group null map is later used to wrap the assembled ColumnTuple in + /// ColumnNullable. See OutputColumnInfo::nullable_group. + bool group_nullable = false; /// TODO [parquet]: Consider also adding output_low_cardinality to allow producing LowCardinality /// column directly from parquet dictionary+indices. This is not straightforward /// because ColumnLowCardinality requires values to be unique and the first value to @@ -202,6 +209,13 @@ struct Reader bool is_missing_column = false; bool needs_cast = false; // if output_type is different from input_type + /// If set, the assembled column (a ColumnTuple) is wrapped in ColumnNullable using the group + /// null map reconstructed from the leaves' definition levels. Used to read a physically + /// nullable parquet struct (OPTIONAL group) as Nullable(Tuple(...)). Only set when the group + /// has no optional/nullable ancestor and an all-REQUIRED, non-array subtree, so every leaf's + /// null map equals the group null map. `needs_cast` (if any) is applied after wrapping. + bool nullable_group = false; + /// If type is Array, this is the repetition level of that array. /// `rep - 1` is index in ColumnChunk::arrays_offsets. UInt8 rep = 0; @@ -349,6 +363,14 @@ struct Reader MutableColumnPtr null_map; + /// For a leaf of a physically-nullable struct read as Nullable(Tuple(...)) (see + /// PrimitiveColumnInfo::group_nullable): the group's definition-level null map, moved here + /// in decodePrimitiveColumn before any leaf-level Nullable wrapping can consume `null_map`. + /// formOutputColumn reads it from the group's first leaf to wrap the assembled ColumnTuple + /// in ColumnNullable. Kept separate from `null_map` so it survives even when the leaf itself + /// is materialized as Nullable(...) (which moves `null_map` into the leaf's ColumnNullable). + MutableColumnPtr group_null_map; + /// If this primitive column is inside an array, this is the offsets for `ColumnArray`s at /// all nesting levels, from outer to inner. Index is repetition level - 1. /// Derived from parquet's repetition/definition levels. See comment on LevelInfo. diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index 2c6d294d0ab3..62d9aba02ad9 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -420,6 +420,10 @@ bool SchemaConverter::processSubtreePrimitive(TraversalNode & node) primitive.name = node.name; primitive.levels = levels; primitive.output_nullable = output_nullable || (output_nullable_if_not_json && !typeid_cast(inferred_type.get())); + /// Leaf of a physically-nullable struct read as Nullable(Tuple(...)): its def-level null map is + /// the group null map. Keep that null map (don't throw on the group-null rows) and fill defaults + /// there; the group null map wraps the ColumnTuple in Reader::formOutputColumn. + primitive.group_nullable = nullable_tuple_group_depth > 0; primitive.decoder = std::move(decoder); primitive.decoded_type = decoded_type; for (const auto & level : levels) @@ -605,6 +609,41 @@ bool SchemaConverter::processSubtreeArrayInner(TraversalNode & node) return true; } +/// Whether the subtree rooted at `schema[root_idx]` (a group) contains only REQUIRED, non-repeated +/// elements below the root. If so, none of its descendants add a definition level, so every leaf's +/// definition-level null map is exactly the root group's null map. This lets us reconstruct the +/// group null map from any leaf and read a physically nullable struct (OPTIONAL group) as +/// Nullable(Tuple(...)) losslessly. Returns false for any OPTIONAL/REPEATED descendant. +static bool tupleSubtreeIsAllRequired(const std::vector & schema, size_t root_idx) +{ + /// schema is a flattened pre-order tree; num_children counts direct children, laid out + /// contiguously in pre-order. Walk the root's subtree with an explicit stack of + /// remaining-children counters for the groups we descended into. + if (root_idx >= schema.size()) + return false; + std::vector stack; + stack.push_back(size_t(schema.at(root_idx).num_children)); + size_t idx = root_idx + 1; + while (!stack.empty()) + { + if (stack.back() == 0) + { + stack.pop_back(); + continue; + } + if (idx >= schema.size()) + return false; // malformed schema; caller handles elsewhere + stack.back() -= 1; + const parq::SchemaElement & elem = schema.at(idx); + if (elem.repetition_type != parq::FieldRepetitionType::REQUIRED) + return false; + idx += 1; + if (elem.__isset.num_children && elem.num_children > 0) + stack.push_back(size_t(elem.num_children)); + } + return true; +} + void SchemaConverter::processSubtreeTuple(TraversalNode & node) { /// Tuple (possibly a Map key_value tuple): @@ -614,24 +653,45 @@ void SchemaConverter::processSubtreeTuple(TraversalNode & node) /// ... /// The requested type may wrap the tuple in Nullable (e.g. `Nullable(Tuple(...))` is a legal - /// type). Unwrap it, match elements against the inner Tuple, and let the outer wrapper be - /// restored via outer_type_hint (needs_cast) in processSubtree. - /// Only unwrap when the tuple is always defined (REQUIRED group, no optional struct-group - /// ancestor), so the restored outer Nullable is always-non-null and lossless. Only Nullable + /// type). Unwrap it, match elements against the inner Tuple, and restore the wrapper below. + /// + /// Two eligible cases, both requiring no optional/nullable STRUCT-group ancestor. Only Nullable /// levels nested below the innermost array count: a Nullable level at or before it is the /// optional wrapper of a LIST/MAP, whose nulls are normalized to empty collections by /// processRepDefLevelsForArray and never reach the inner tuple null-map. + /// 1. REQUIRED group: always defined, so the outer Nullable is always-non-null. Restored via + /// outer_type_hint (needs_cast) as an all-non-null wrapper. + /// 2. OPTIONAL group with an all-REQUIRED, non-array subtree: physically nullable struct. No + /// descendant adds a definition level, so every leaf's def-level null map equals the group + /// null map. We mark the leaves and the output so the assembled ColumnTuple is wrapped in + /// ColumnNullable using that reconstructed null map (see OutputColumnInfo::nullable_group). + /// Otherwise keep the hint wrapped and let the check below reject it with TYPE_MISMATCH rather + /// than lose nulls. For an OPTIONAL group, processSubtree has already pushed this group's own + /// (non-array) level as levels.back(); exclude it when scanning for an ancestor. size_t innermost_array_idx = 0; for (size_t i = 0; i < levels.size(); ++i) if (levels[i].is_array) innermost_array_idx = i; + const bool group_is_optional = node.element->repetition_type == parq::FieldRepetitionType::OPTIONAL; + const size_t ancestor_end = levels.size() - (group_is_optional ? 1 : 0); bool has_optional_ancestor = false; - for (size_t i = innermost_array_idx + 1; i < levels.size(); ++i) + for (size_t i = innermost_array_idx + 1; i < ancestor_end; ++i) has_optional_ancestor |= !levels[i].is_array; - if (node.type_hint && node.type_hint->isNullable() - && node.element->repetition_type == parq::FieldRepetitionType::REQUIRED - && !has_optional_ancestor) - node.type_hint = assert_cast(*node.type_hint).getNestedType(); + bool nullable_group = false; + if (node.type_hint && node.type_hint->isNullable() && !has_optional_ancestor) + { + if (node.element->repetition_type == parq::FieldRepetitionType::REQUIRED) + node.type_hint = assert_cast(*node.type_hint).getNestedType(); + else if (group_is_optional && tupleSubtreeIsAllRequired(file_metadata.schema, schema_idx - 1)) + { + node.type_hint = assert_cast(*node.type_hint).getNestedType(); + nullable_group = true; + } + } + + /// Mark leaves recursed below as belonging to a physically-nullable group (case 2 above). + nullable_tuple_group_depth += nullable_group ? 1 : 0; + SCOPE_EXIT({ nullable_tuple_group_depth -= nullable_group ? 1 : 0; }); const DataTypeTuple * tuple_type_hint = typeid_cast(node.type_hint.get()); if (node.type_hint && !tuple_type_hint && !typeid_cast(node.type_hint.get())) @@ -775,6 +835,22 @@ void SchemaConverter::processSubtreeTuple(TraversalNode & node) output_type = std::make_shared(types, names); } + /// Physically-nullable struct (OPTIONAL group, case 2 above): the assembled ColumnTuple must be + /// wrapped in ColumnNullable using the group null map. Make input_type Nullable(Tuple(...)) so + /// the outer restore in processSubtree sees no type change (needs_cast stays off); the wrapping + /// is done in Reader::formOutputColumn keyed by OutputColumnInfo::nullable_group. + /// The group null map is reconstructed from a physical leaf's definition levels, so at least one + /// leaf must actually be read. With allow_missing_columns, every requested element can be a + /// synthetic default (no physical leaf); then the null map is unrecoverable, so reject rather + /// than fabricate an all-non-null map that silently drops the struct nulls. + if (nullable_group && primitive_start == primitive_columns.size()) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Requested type of column {} doesn't match parquet schema: physically nullable Tuple has no " + "physical elements to read (all requested elements are missing), so its null map cannot be " + "reconstructed; requested type is {}", node.getNameForLogging(), node.type_hint->getName()); + if (nullable_group) + output_type = makeNullable(output_type); + node.output_idx = output_columns.size(); OutputColumnInfo & output = output_columns.emplace_back(); output.name = node.name; @@ -783,6 +859,7 @@ void SchemaConverter::processSubtreeTuple(TraversalNode & node) output.input_type = std::move(output_type); output.output_type = output.input_type; output.nested_columns = elements; + output.nullable_group = nullable_group; } void SchemaConverter::processPrimitiveColumn( diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h index 7f6c825ff9e3..f5bd40089f7d 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h @@ -35,6 +35,11 @@ struct SchemaConverter /// Actual recursion depth of processSubtree. Tracked unconditionally because the def-level /// counter only advances for OPTIONAL/REPEATED nodes, so REQUIRED-group nesting would bypass it. size_t recursion_depth = 0; + /// >0 while recursing inside a physically-nullable Tuple group (OPTIONAL group requested as + /// Nullable(Tuple(...)) and eligible for lossless reading). Leaves under it get + /// PrimitiveColumnInfo::group_nullable set: their definition-level null map equals the group + /// null map, so we keep it and later wrap the assembled ColumnTuple in ColumnNullable. + size_t nullable_tuple_group_depth = 0; /// The key is the parquet column name, without ColumnMapper. std::unordered_map geo_columns; diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 5031de6a4dfc..3fb1dc1874b5 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -47,6 +47,7 @@ EXCLUDE=( 04065_optional_list_wrapper_required_element.parquet 04065_optional_map_wrapper_required_value.parquet 04065_optional_struct_under_list.parquet + 04065_optional_struct_nullable_leaf_under_list.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference index 9c2f0a06b963..739cfa520216 100644 --- a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference +++ b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.reference @@ -3,5 +3,7 @@ [(3)] Array(Nullable(Tuple(x UInt32))) -- optional MAP wrapper, REQUIRED value group: Map(String, Nullable(Tuple)) accepted {'k1':(1),'k2':(2)} Map(String, Nullable(Tuple(x UInt32))) --- optional element group under a list: genuine struct-level nulls, still rejected +-- optional element group with all-REQUIRED subtree under a list: struct nulls reconstructed losslessly, accepted +[((1)),NULL,((3))] Array(Nullable(Tuple(inner Tuple(x UInt32)))) +-- optional element group with a NULLABLE leaf under a list: leaf null map != struct null map, still rejected TYPE_MISMATCH diff --git a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh index b5909b9d2fc9..80a54c585756 100755 --- a/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh +++ b/tests/queries/0_stateless/04065_parquet_optional_list_map_wrapper_nullable_tuple.sh @@ -12,8 +12,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # # The optional wrapper's nulls are normalized to empty collections by the reader and never reach # the inner tuple null-map, so an always-defined REQUIRED element/value read as Nullable(Tuple) -# is lossless and must be accepted (issue #109605 follow-up). A genuinely OPTIONAL inner group -# carries real struct-level nulls and must still be rejected. +# is lossless and must be accepted (issue #109605 follow-up). +# +# An OPTIONAL element/struct group whose own subtree is all-REQUIRED carries genuine struct-level +# nulls, but every leaf's definition-level null map then equals the group null map, so it can be +# reconstructed and the tuple wrapped in Nullable losslessly (#109898) -- accepted. Only when the +# subtree adds another definition level (e.g. a nullable leaf) is the group null map no longer +# recoverable from a leaf, so that case must still be rejected. DATA="$CURDIR/data_parquet" @@ -25,5 +30,8 @@ $CLICKHOUSE_LOCAL $opts -q "SELECT a, toTypeName(a) FROM file('$DATA/04065_optio echo "-- optional MAP wrapper, REQUIRED value group: Map(String, Nullable(Tuple)) accepted" $CLICKHOUSE_LOCAL $opts -q "SELECT m, toTypeName(m) FROM file('$DATA/04065_optional_map_wrapper_required_value.parquet', 'Parquet', 'm Map(String, Nullable(Tuple(x UInt32)))')" -echo "-- optional element group under a list: genuine struct-level nulls, still rejected" -$CLICKHOUSE_LOCAL $opts -q "SELECT a FROM file('$DATA/04065_optional_struct_under_list.parquet', 'Parquet', 'a Array(Nullable(Tuple(inner Tuple(x UInt32))))')" 2>&1 | grep -o "TYPE_MISMATCH" | head -1 +echo "-- optional element group with all-REQUIRED subtree under a list: struct nulls reconstructed losslessly, accepted" +$CLICKHOUSE_LOCAL $opts -q "SELECT a, toTypeName(a) FROM file('$DATA/04065_optional_struct_under_list.parquet', 'Parquet', 'a Array(Nullable(Tuple(inner Tuple(x UInt32))))')" + +echo "-- optional element group with a NULLABLE leaf under a list: leaf null map != struct null map, still rejected" +$CLICKHOUSE_LOCAL $opts -q "SELECT a FROM file('$DATA/04065_optional_struct_nullable_leaf_under_list.parquet', 'Parquet', 'a Array(Nullable(Tuple(inner Tuple(x UInt32))))')" 2>&1 | grep -o "TYPE_MISMATCH" | head -1 diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference index a5758ea71f26..7c0798c8a367 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference @@ -7,17 +7,39 @@ DROP TABLE IF EXISTS test_nullable_tuple_basic; CREATE TABLE test_nullable_tuple_basic (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader: physically nullable struct (OPTIONAL group, all-REQUIRED subtree) +-- read as Nullable(Tuple). The group null map is reconstructed from the leaves (issue #109605). +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); +(1,'a') +\N +(3,'c') DROP TABLE test_nullable_tuple_basic; -- Both struct and element nullable: Nullable(Tuple(Nullable(UInt32), String)) DROP TABLE IF EXISTS test_nullable_tuple_both; CREATE TABLE test_nullable_tuple_both (c0 Nullable(Tuple(Nullable(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ((4, 'd')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; --- Parquet V3 native reader (not yet supported) +-- Parquet V3 native reader: nullable element makes the subtree not all-REQUIRED, so the group +-- null map cannot be separated from the element null map. Reject rather than lose data. SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_both; +-- Physically-nullable struct (all-REQUIRED leaves) read as Nullable(Tuple) with a leaf hint that +-- materializes the first leaf as Nullable. decodePrimitiveColumn moves the shared group null map +-- into the leaf's ColumnNullable, so the group null map must be preserved separately or the middle +-- (struct-level NULL) row would be silently returned as a non-null tuple. Middle row must stay NULL. +DROP TABLE IF EXISTS test_nullable_tuple_leaf_nullable; +CREATE TABLE test_nullable_tuple_leaf_nullable (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_leaf_nullable VALUES ((1, 'a')), (NULL), ((3, 'c')); +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_leaf_nullable; +SELECT c0 FROM file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); +(1,'a') +\N +(3,'c') +SELECT c0 IS NULL FROM file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); +0 +1 +0 +DROP TABLE test_nullable_tuple_leaf_nullable; -- Non-nullable struct with nullable elements DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; @@ -43,39 +65,52 @@ DROP TABLE IF EXISTS test_nullable_tuple_named; CREATE TABLE test_nullable_tuple_named (c0 Nullable(Tuple(a UInt32, b String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; --- Parquet V3 native reader named (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader named: physically nullable struct read as Nullable(Tuple) +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); +(1,'x') +\N +(3,'z') DROP TABLE test_nullable_tuple_named; -- All-NULL column DROP TABLE IF EXISTS test_nullable_tuple_allnull; CREATE TABLE test_nullable_tuple_allnull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; --- Parquet V3 native reader all null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader all null: every row is a struct-level NULL +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); +\N +\N +\N DROP TABLE test_nullable_tuple_allnull; -- No-NULL column (nullable type, zero actual NULLs) DROP TABLE IF EXISTS test_nullable_tuple_nonull; CREATE TABLE test_nullable_tuple_nonull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; --- Parquet V3 native reader no null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader no null: nullable struct type, zero actual struct-level NULLs +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); +(1,'a') +(2,'b') +(3,'c') DROP TABLE test_nullable_tuple_nonull; -- Single-element tuple DROP TABLE IF EXISTS test_nullable_tuple_single; CREATE TABLE test_nullable_tuple_single (c0 Nullable(Tuple(UInt32))) ENGINE = Memory; INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; --- Parquet V3 native reader single (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader single: single-element physically nullable struct +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); +(1) +\N +(3) DROP TABLE test_nullable_tuple_single; -- Deeply nested: nullable tuple inside nullable tuple DROP TABLE IF EXISTS test_nullable_tuple_deep; CREATE TABLE test_nullable_tuple_deep (c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20)), (((4, 'd'), 40)); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; --- Parquet V3 native reader deep nested (not yet supported) +-- Parquet V3 native reader deep nested: inner Nullable(Tuple) is an OPTIONAL inner group, so the +-- subtree is not all-REQUIRED. Reject rather than lose the inner struct nulls. SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_deep; -- Nullable tuple with Array element @@ -83,7 +118,8 @@ DROP TABLE IF EXISTS test_nullable_tuple_arr; CREATE TABLE test_nullable_tuple_arr (c0 Nullable(Tuple(Array(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; --- Parquet V3 native reader array elem (not yet supported) +-- Parquet V3 native reader array elem: the Array element adds a repetition level, so the subtree +-- is not all-REQUIRED and the leaf null maps no longer equal the group null map. Reject. SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr; -- Multiple nullable tuple columns @@ -91,8 +127,11 @@ DROP TABLE IF EXISTS test_nullable_tuple_multi; CREATE TABLE test_nullable_tuple_multi (c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ((3, 'c'), NULL); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; --- Parquet V3 native reader multi col (not yet supported) -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader multi col: two independent physically nullable structs +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); +(1,'a') (1.5) +\N (2.5) +(3,'c') \N DROP TABLE test_nullable_tuple_multi; -- Type hint mismatch: file has a REQUIRED Tuple group, read as Nullable(Tuple(...)) (issue #109605). -- A REQUIRED group is always defined, so the outer Nullable is always-non-null and reading it as @@ -110,6 +149,17 @@ SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parqu ('a',1) Nullable(Tuple(`2` String, `1` UInt32)) ('b',2) Nullable(Tuple(`2` String, `1` UInt32)) DROP TABLE test_nullable_tuple_mismatch2; +-- Physically nullable outer struct with a REQUIRED nested struct (all-REQUIRED subtree): the outer +-- group's def-level null map is reconstructed and the nested tuple is preserved (issue #109605). +DROP TABLE IF EXISTS test_nullable_tuple_nested_required; +CREATE TABLE test_nullable_tuple_nested_required (c0 Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_nested_required VALUES (((1, 'a'), 10)), (NULL), (((3, 'c'), 30)); +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nested_required.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_nested_required; +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_nested_required.parquet', 'Parquet', 'c0 Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64))') ORDER BY c0.c; +((1,'a'),10) Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64)) +((3,'c'),30) Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64)) +\N Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64)) +DROP TABLE test_nullable_tuple_nested_required; -- Schema inference: inferred type with toTypeName DROP TABLE IF EXISTS test_nullable_tuple_describe; CREATE TABLE test_nullable_tuple_describe (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; @@ -136,8 +186,11 @@ DROP TABLE IF EXISTS test_nullable_tuple_arr_unnamed; CREATE TABLE test_nullable_tuple_arr_unnamed (c0 Array(Nullable(Tuple(UInt32, String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; --- Parquet V3 native reader unnamed (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader unnamed: Array(Nullable(Tuple)) with an all-REQUIRED tuple subtree. An +-- array ancestor is not an optional struct-group ancestor (array nulls become empty arrays), so +-- each leaf's null map is exactly the per-element tuple null map and the tuple is wrapped losslessly. +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); +[(1,'a'),NULL,(3,'c')] DROP TABLE test_nullable_tuple_arr_unnamed; -- Array(Nullable(Tuple)) with Array element inside: import_nested flattens DROP TABLE IF EXISTS test_nullable_tuple_arr_nested_elem; @@ -167,3 +220,13 @@ INSERT INTO test_nullable_tuple_opt_ancestor VALUES (((1,),)), (NULL), (((3,),)) INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_opt_ancestor; SELECT `c0.inner` FROM file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet', '`c0.inner` Nullable(Tuple(a UInt32))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_opt_ancestor; +-- Physically nullable struct read as Nullable(Tuple) where every requested element is missing and +-- synthesized (input_format_parquet_allow_missing_columns). With no physical leaf, the group null +-- map cannot be reconstructed, so reject rather than fabricate an all-non-null map (would abort / +-- read past the decoded-leaf array otherwise). +DROP TABLE IF EXISTS test_nullable_tuple_all_missing; +CREATE TABLE test_nullable_tuple_all_missing (c0 Nullable(Tuple(a UInt32, b UInt64))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_all_missing VALUES ((1, 10)), (NULL), ((3, 30)); +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_all_missing.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_all_missing; +SELECT c0 FROM file(currentDatabase() || '_04065_all_missing.parquet', 'Parquet', 'c0 Nullable(Tuple(z String))') SETTINGS input_format_parquet_allow_missing_columns = 1; -- { serverError TYPE_MISMATCH } +DROP TABLE test_nullable_tuple_all_missing; diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql index 55e3e2fdca66..67b4d4760ec2 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql @@ -13,8 +13,9 @@ INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader: physically nullable struct (OPTIONAL group, all-REQUIRED subtree) +-- read as Nullable(Tuple). The group null map is reconstructed from the leaves (issue #109605). +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); DROP TABLE test_nullable_tuple_basic; @@ -25,11 +26,27 @@ INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ( INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; --- Parquet V3 native reader (not yet supported) +-- Parquet V3 native reader: nullable element makes the subtree not all-REQUIRED, so the group +-- null map cannot be separated from the element null map. Reject rather than lose data. SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_both; +-- Physically-nullable struct (all-REQUIRED leaves) read as Nullable(Tuple) with a leaf hint that +-- materializes the first leaf as Nullable. decodePrimitiveColumn moves the shared group null map +-- into the leaf's ColumnNullable, so the group null map must be preserved separately or the middle +-- (struct-level NULL) row would be silently returned as a non-null tuple. Middle row must stay NULL. +DROP TABLE IF EXISTS test_nullable_tuple_leaf_nullable; +CREATE TABLE test_nullable_tuple_leaf_nullable (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_leaf_nullable VALUES ((1, 'a')), (NULL), ((3, 'c')); + +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_leaf_nullable; + +SELECT c0 FROM file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); +SELECT c0 IS NULL FROM file(currentDatabase() || '_04065_leaf_nullable.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); + +DROP TABLE test_nullable_tuple_leaf_nullable; + -- Non-nullable struct with nullable elements DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; @@ -61,8 +78,8 @@ INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; --- Parquet V3 native reader named (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader named: physically nullable struct read as Nullable(Tuple) +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); DROP TABLE test_nullable_tuple_named; @@ -73,8 +90,8 @@ INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; --- Parquet V3 native reader all null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader all null: every row is a struct-level NULL +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); DROP TABLE test_nullable_tuple_allnull; @@ -85,8 +102,8 @@ INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')) INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; --- Parquet V3 native reader no null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader no null: nullable struct type, zero actual struct-level NULLs +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); DROP TABLE test_nullable_tuple_nonull; @@ -97,8 +114,8 @@ INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; --- Parquet V3 native reader single (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader single: single-element physically nullable struct +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); DROP TABLE test_nullable_tuple_single; @@ -109,7 +126,8 @@ INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20 INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; --- Parquet V3 native reader deep nested (not yet supported) +-- Parquet V3 native reader deep nested: inner Nullable(Tuple) is an OPTIONAL inner group, so the +-- subtree is not all-REQUIRED. Reject rather than lose the inner struct nulls. SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_deep; @@ -121,7 +139,8 @@ INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')) INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; --- Parquet V3 native reader array elem (not yet supported) +-- Parquet V3 native reader array elem: the Array element adds a repetition level, so the subtree +-- is not all-REQUIRED and the leaf null maps no longer equal the group null map. Reject. SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr; @@ -133,8 +152,8 @@ INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ( INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; --- Parquet V3 native reader multi col (not yet supported) -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader multi col: two independent physically nullable structs +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); DROP TABLE test_nullable_tuple_multi; @@ -155,6 +174,18 @@ SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parqu DROP TABLE test_nullable_tuple_mismatch2; +-- Physically nullable outer struct with a REQUIRED nested struct (all-REQUIRED subtree): the outer +-- group's def-level null map is reconstructed and the nested tuple is preserved (issue #109605). +DROP TABLE IF EXISTS test_nullable_tuple_nested_required; +CREATE TABLE test_nullable_tuple_nested_required (c0 Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_nested_required VALUES (((1, 'a'), 10)), (NULL), (((3, 'c'), 30)); + +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nested_required.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_nested_required; + +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_nested_required.parquet', 'Parquet', 'c0 Nullable(Tuple(inner Tuple(a UInt32, b String), c UInt64))') ORDER BY c0.c; + +DROP TABLE test_nullable_tuple_nested_required; + -- Schema inference: inferred type with toTypeName DROP TABLE IF EXISTS test_nullable_tuple_describe; CREATE TABLE test_nullable_tuple_describe (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; @@ -187,8 +218,10 @@ INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; --- Parquet V3 native reader unnamed (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } +-- Parquet V3 native reader unnamed: Array(Nullable(Tuple)) with an all-REQUIRED tuple subtree. An +-- array ancestor is not an optional struct-group ancestor (array nulls become empty arrays), so +-- each leaf's null map is exactly the per-element tuple null map and the tuple is wrapped losslessly. +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); DROP TABLE test_nullable_tuple_arr_unnamed; @@ -228,3 +261,17 @@ INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_opt_ancestor.parque SELECT `c0.inner` FROM file(currentDatabase() || '_04065_opt_ancestor.parquet', 'Parquet', '`c0.inner` Nullable(Tuple(a UInt32))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_opt_ancestor; + +-- Physically nullable struct read as Nullable(Tuple) where every requested element is missing and +-- synthesized (input_format_parquet_allow_missing_columns). With no physical leaf, the group null +-- map cannot be reconstructed, so reject rather than fabricate an all-non-null map (would abort / +-- read past the decoded-leaf array otherwise). +DROP TABLE IF EXISTS test_nullable_tuple_all_missing; +CREATE TABLE test_nullable_tuple_all_missing (c0 Nullable(Tuple(a UInt32, b UInt64))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_all_missing VALUES ((1, 10)), (NULL), ((3, 30)); + +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_all_missing.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_all_missing; + +SELECT c0 FROM file(currentDatabase() || '_04065_all_missing.parquet', 'Parquet', 'c0 Nullable(Tuple(z String))') SETTINGS input_format_parquet_allow_missing_columns = 1; -- { serverError TYPE_MISMATCH } + +DROP TABLE test_nullable_tuple_all_missing; diff --git a/tests/queries/0_stateless/data_parquet/04065_optional_struct_nullable_leaf_under_list.parquet b/tests/queries/0_stateless/data_parquet/04065_optional_struct_nullable_leaf_under_list.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e08c42a0b391e50cf16132786041fabceda5e7a1 GIT binary patch literal 737 zcmaJ<%WB(D5FOpBiWgqkg)1b`!UQT>$ipoal#)Qlj{Bf#YnwJsvr}B*7;MLp)VR0o?2t+36Wp*h{%%NwdtM>YzKVW*fB z(_*F(MUTg`Sy7^5%~K4mDKv2sCn+V?#2E#(_}t5NdOPPPytOr95dQYUdoOLKmH)Gj z_aGcAL~9N1=y_`V_>9OH5hE}rtpJMRB&u@1|8?-jiHZK0Lpf`1F>?=(oKMUvZjJaI z^N8Cc&9%_*S>D>rAH^fydPE&bsk!U|Z-uD0`Yy@$VjQ&Wa7}7kR}xR9Kl=vrTB={L z6yO4|X%$v&rS?IR7lZZWa&Wf%$tN8Q9bzf=C`r_HuBU3Gnc3liG>-C9Do&KH$_Lr< z;#{Smz8M8m6p{QPR8+SX6$a?aoQD#ssBIp;(DxCa==@N=*XDHGm(=L^n4-o!FK?E` m?f!5+-_OhPdeyiq%EkD4)_D2qpn1@YlLXiI3wPMX&;1vq_?t!m literal 0 HcmV?d00001 From 20ff763e33638677b9fa2278bfdb249cffac78f4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 14:34:22 +0000 Subject: [PATCH 61/86] Backport #111278 to 26.6: Bump contrib/thrift to Apache Thrift v0.24.0 --- contrib/thrift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/thrift b/contrib/thrift index 248688d401a4..13d30d6a6cdf 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 248688d401a48b0c34d7aa06a577a47a992a32dc +Subproject commit 13d30d6a6cdfbd44c07977b8b762e88d9d5eb6df From b3eb9f4fc21baf2934b6dcd75bb7c622227768ee Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 15:33:08 +0000 Subject: [PATCH 62/86] Backport #111629 to 26.6: Fix overflow while parsing in system.zookeeper_info --- src/Coordination/FourLetterCommand.cpp | 14 +-- .../KeeperAsynchronousMetrics.cpp | 29 +++-- src/Coordination/KeeperAsynchronousMetrics.h | 11 ++ .../gtest_keeper_asynchronous_metrics.cpp | 39 +++++++ .../System/StorageSystemZooKeeperInfo.cpp | 71 ++++++------ .../__init__.py | 0 .../configs/enable_keeper.xml | 22 ++++ .../configs/use_keeper.xml | 12 +++ .../mock_keeper_4lw.py | 62 +++++++++++ .../test.py | 102 ++++++++++++++++++ 10 files changed, 319 insertions(+), 43 deletions(-) create mode 100644 src/Coordination/tests/gtest_keeper_asynchronous_metrics.cpp create mode 100644 tests/integration/test_zookeeper_info_number_overflow/__init__.py create mode 100644 tests/integration/test_zookeeper_info_number_overflow/configs/enable_keeper.xml create mode 100644 tests/integration/test_zookeeper_info_number_overflow/configs/use_keeper.xml create mode 100644 tests/integration/test_zookeeper_info_number_overflow/mock_keeper_4lw.py create mode 100644 tests/integration/test_zookeeper_info_number_overflow/test.py diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp index ed1d5fbd6e09..b3db893334d6 100644 --- a/src/Coordination/FourLetterCommand.cpp +++ b/src/Coordination/FourLetterCommand.cpp @@ -333,12 +333,14 @@ String MonitorCommand::run() print(ret, "latest_snapshot_size", state_machine.getLatestSnapshotSize()); #if defined(OS_LINUX) || defined(OS_DARWIN) - print(ret, "open_file_descriptor_count", getCurrentProcessFDCount()); - auto max_file_descriptor_count = getMaxFileDescriptorCount(); - if (max_file_descriptor_count.has_value()) - print(ret, "max_file_descriptor_count", *max_file_descriptor_count); - else - print(ret, "max_file_descriptor_count", -1); + /// An undetermined value is reported as the textual `-1`, as ZooKeeper does. + /// It must not go through the `uint64_t` overload of `print`: `-1` would wrap around + /// to 2^64 - 1, which is indistinguishable from an unlimited `RLIMIT_NOFILE` (`RLIM_INFINITY`). + const Int64 open_file_descriptor_count = getCurrentProcessFDCount(); + print(ret, "open_file_descriptor_count", toString(open_file_descriptor_count)); + + const auto max_file_descriptor_count = getMaxFileDescriptorCount(); + print(ret, "max_file_descriptor_count", max_file_descriptor_count.has_value() ? toString(*max_file_descriptor_count) : String("-1")); #endif if (keeper_info.is_leader) diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 7b5f80e80a8a..869b08bba495 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -11,6 +11,20 @@ namespace DB { +void setKeeperFileDescriptorMetrics( + AsynchronousMetricValues & new_values, Int64 open_file_descriptor_count, std::optional max_file_descriptor_count) +{ + new_values["KeeperOpenFileDescriptorCount"] + = {open_file_descriptor_count, "The number of open file descriptors in ClickHouse Keeper. `-1` if the value cannot be determined."}; + if (max_file_descriptor_count.has_value()) + new_values["KeeperMaxFileDescriptorCount"] = { + *max_file_descriptor_count, + "The maximum number of open file descriptors in ClickHouse Keeper. `-1` if the value cannot be determined."}; + else + new_values["KeeperMaxFileDescriptorCount"] + = {-1, "The maximum number of open file descriptors in ClickHouse Keeper. `-1` if the value cannot be determined."}; +} + void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousMetricValues & new_values) { #if USE_NURAFT @@ -23,8 +37,13 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM size_t ephemerals_count = 0; size_t approximate_data_size = 0; size_t key_arena_size = 0; - size_t open_file_descriptor_count = 0; - std::optional max_file_descriptor_count = 0; + /// Signed on purpose: `getCurrentProcessFDCount` reports an undetermined count as `-1`, + /// and it must not wrap around to 2^64 - 1, which is indistinguishable from an unlimited + /// `RLIMIT_NOFILE`. This matches the contract of the `mntr` four-letter command. + /// The values are assigned only on Linux and macOS, so start from "undetermined", + /// not from 0, for the same reason. + Int64 open_file_descriptor_count = -1; + std::optional max_file_descriptor_count; size_t followers = 0; size_t synced_followers = 0; size_t zxid = 0; @@ -80,11 +99,7 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM /// it needs to be fixed and it needs to be atomic to avoid deadlock ///new_values["KeeperLatestSnapshotSize"] = { latest_snapshot_size, "The uncompressed size in bytes of the latest snapshot created by ClickHouse Keeper." }; - new_values["KeeperOpenFileDescriptorCount"] = { open_file_descriptor_count, "The number of open file descriptors in ClickHouse Keeper." }; - if (max_file_descriptor_count.has_value()) - new_values["KeeperMaxFileDescriptorCount"] = { *max_file_descriptor_count, "The maximum number of open file descriptors in ClickHouse Keeper." }; - else - new_values["KeeperMaxFileDescriptorCount"] = { -1, "The maximum number of open file descriptors in ClickHouse Keeper." }; + setKeeperFileDescriptorMetrics(new_values, open_file_descriptor_count, max_file_descriptor_count); new_values["KeeperFollowers"] = { followers, "The number of followers of ClickHouse Keeper." }; new_values["KeeperSyncedFollowers"] = { synced_followers, "The number of followers of ClickHouse Keeper who are also in-sync." }; diff --git a/src/Coordination/KeeperAsynchronousMetrics.h b/src/Coordination/KeeperAsynchronousMetrics.h index 1f5a5eb5f2e5..277da7f6f74a 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.h +++ b/src/Coordination/KeeperAsynchronousMetrics.h @@ -3,12 +3,23 @@ #include #include +#include + +#include + namespace DB { class KeeperDispatcher; void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousMetricValues & new_values); +/// Fills `KeeperOpenFileDescriptorCount` and `KeeperMaxFileDescriptorCount`. +/// An undetermined count is reported as `-1`: it is signed on purpose, so that the sentinel does not +/// wrap around to 2^64 - 1, which is indistinguishable from an unlimited `RLIMIT_NOFILE`. +/// Exposed separately from `updateKeeperInformation` to make that contract testable. +void setKeeperFileDescriptorMetrics( + AsynchronousMetricValues & new_values, Int64 open_file_descriptor_count, std::optional max_file_descriptor_count); + class KeeperAsynchronousMetrics : public AsynchronousMetrics { public: diff --git a/src/Coordination/tests/gtest_keeper_asynchronous_metrics.cpp b/src/Coordination/tests/gtest_keeper_asynchronous_metrics.cpp new file mode 100644 index 000000000000..7d7cb660ad18 --- /dev/null +++ b/src/Coordination/tests/gtest_keeper_asynchronous_metrics.cpp @@ -0,0 +1,39 @@ +#include + +#include + +#include +#include + +using namespace DB; + +/// `getCurrentProcessFDCount` returns `-1` and `getMaxFileDescriptorCount` returns `std::nullopt` when the count +/// cannot be determined. The sentinel must survive as `-1` instead of wrapping around to 2^64 - 1, which is +/// indistinguishable from an unlimited `RLIMIT_NOFILE`. This is the branch that a running Keeper does not take +/// on Linux and macOS, so it is checked here directly. +TEST(KeeperAsynchronousMetrics, UndeterminedFileDescriptorCounts) +{ + AsynchronousMetricValues values; + setKeeperFileDescriptorMetrics(values, -1, std::nullopt); + + ASSERT_EQ(values.at("KeeperOpenFileDescriptorCount").value, -1.0); + ASSERT_EQ(values.at("KeeperMaxFileDescriptorCount").value, -1.0); +} + +TEST(KeeperAsynchronousMetrics, DeterminedFileDescriptorCounts) +{ + AsynchronousMetricValues values; + setKeeperFileDescriptorMetrics(values, 42, 1024); + + ASSERT_EQ(values.at("KeeperOpenFileDescriptorCount").value, 42.0); + ASSERT_EQ(values.at("KeeperMaxFileDescriptorCount").value, 1024.0); +} + +/// An unlimited `RLIMIT_NOFILE` is reported verbatim and must not be confused with the `-1` sentinel. +TEST(KeeperAsynchronousMetrics, UnlimitedMaxFileDescriptorCount) +{ + AsynchronousMetricValues values; + setKeeperFileDescriptorMetrics(values, 42, std::numeric_limits::max()); + + ASSERT_EQ(values.at("KeeperMaxFileDescriptorCount").value, static_cast(std::numeric_limits::max())); +} diff --git a/src/Storages/System/StorageSystemZooKeeperInfo.cpp b/src/Storages/System/StorageSystemZooKeeperInfo.cpp index 9da40416d22d..a17b48cb7f05 100644 --- a/src/Storages/System/StorageSystemZooKeeperInfo.cpp +++ b/src/Storages/System/StorageSystemZooKeeperInfo.cpp @@ -168,37 +168,37 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP // /* 7 */ {"avg_latency", std::make_shared(), "The average latency."}, if (const auto it = mntr_responses_map.find("zk_avg_latency"); it != mntr_responses_map.end()) - res_columns[7]->insert(parse(it->second)); + res_columns[7]->insert(parse(it->second)); else res_columns[7]->insertDefault(); // /* 8 */ {"max_latency", std::make_shared(), "The max latency."}, if (const auto it = mntr_responses_map.find("zk_max_latency"); it != mntr_responses_map.end()) - res_columns[8]->insert(parse(it->second)); + res_columns[8]->insert(parse(it->second)); else res_columns[8]->insertDefault(); // /* 9 */ {"min_latency", std::make_shared(), "The min latency."}, if (const auto it = mntr_responses_map.find("zk_min_latency"); it != mntr_responses_map.end()) - res_columns[9]->insert(parse(it->second)); + res_columns[9]->insert(parse(it->second)); else res_columns[9]->insertDefault(); // /* 10 */ {"packets_received", std::make_shared(), "The number of packets received."}, if (const auto it = mntr_responses_map.find("zk_packets_received"); it != mntr_responses_map.end()) - res_columns[10]->insert(parse(it->second)); + res_columns[10]->insert(parse(it->second)); else res_columns[10]->insertDefault(); // /* 11 */ {"packets_sent", std::make_shared(), "The number of packets sent."}, if (const auto it = mntr_responses_map.find("zk_packets_sent"); it != mntr_responses_map.end()) - res_columns[11]->insert(parse(it->second)); + res_columns[11]->insert(parse(it->second)); else res_columns[11]->insertDefault(); // /* 12 */ {"outstanding_requests", std::make_shared(), "The number of outstanding requests."}, if (const auto it = mntr_responses_map.find("zk_outstanding_requests"); it != mntr_responses_map.end()) - res_columns[12]->insert(parse(it->second)); + res_columns[12]->insert(parse(it->second)); else res_columns[12]->insertDefault(); @@ -209,12 +209,12 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP res_columns[13]->insertDefault(); ///* 15 */ {"znode_count", std::make_shared(), "The znode count."}, - int followers = 0; + UInt64 followers = 0; if (const auto it = mntr_responses_map.find("zk_followers"); it != mntr_responses_map.end()) { auto followers_in_string = mntr_responses_map["zk_followers"]; if (!followers_in_string.empty()) - followers = parse(followers_in_string); + followers = parse(followers_in_string); ///* 14 */ {"is_leader", std::make_shared(), "Is this zookeeper leader."}, res_columns[14]->insert(followers > 0); @@ -223,31 +223,31 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP res_columns[14]->insertDefault(); if (const auto it = mntr_responses_map.find("zk_znode_count"); it != mntr_responses_map.end()) - res_columns[15]->insert(parse(it->second)); + res_columns[15]->insert(parse(it->second)); else res_columns[15]->insertDefault(); // /* 16 */ {"watch_count", std::make_shared(), "The watch count."}, if (const auto it = mntr_responses_map.find("zk_watch_count"); it != mntr_responses_map.end()) - res_columns[16]->insert(parse(it->second)); + res_columns[16]->insert(parse(it->second)); else res_columns[16]->insertDefault(); // /* 17 */ {"ephemerals_count", std::make_shared(), "The ephemerals count."}, if (const auto it = mntr_responses_map.find("zk_ephemerals_count"); it != mntr_responses_map.end()) - res_columns[17]->insert(parse(it->second)); + res_columns[17]->insert(parse(it->second)); else res_columns[17]->insertDefault(); // /* 18 */ {"approximate_data_size", std::make_shared(), "The approximate data size."}, if (const auto it = mntr_responses_map.find("zk_approximate_data_size"); it != mntr_responses_map.end()) - res_columns[18]->insert(parse(it->second)); + res_columns[18]->insert(parse(it->second)); else res_columns[18]->insertDefault(); // /* 19 */ {"followers", std::make_shared(), "The followers of the leader. This field is only exposed by the leader."}, if (const auto it = mntr_responses_map.find("zk_followers"); it != mntr_responses_map.end()) - res_columns[19]->insert(parse(it->second)); + res_columns[19]->insert(parse(it->second)); else res_columns[19]->insertDefault(); @@ -255,7 +255,7 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP // /* 21 */ {"pending_syncs", std::make_shared(), "The pending syncs of the leader. This field is only exposed by the leader."}, if (const auto it = mntr_responses_map.find("zk_synced_followers"); it != mntr_responses_map.end()) { - int synced_followers = parse(it->second); + UInt64 synced_followers = parse(it->second); res_columns[20]->insert(synced_followers); res_columns[21]->insert(followers - synced_followers); } @@ -265,15 +265,26 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP res_columns[21]->insertDefault(); } + /// Keeper reports -1 for the file descriptor counts it could not determine. + /// Otherwise the values are unsigned: `max_file_descriptor_count` is printed as a `size_t` + /// and equals 2^64 - 1 (`RLIM_INFINITY`) when the limit is unlimited, so it must not be parsed as `Int64`. + auto insert_if_non_negative = [](IColumn & column, std::string_view value) + { + if (value == "-1") + column.insertDefault(); + else + column.insert(parse(value)); + }; + // /* 22 */ {"open_file_descriptor_count", std::make_shared(), "The open file descriptor count. Only available on Unix platforms."}, if (const auto it = mntr_responses_map.find("zk_open_file_descriptor_count"); it != mntr_responses_map.end()) - res_columns[22]->insert(parse(it->second)); + insert_if_non_negative(*res_columns[22], it->second); else res_columns[22]->insertDefault(); // /* 23 */ {"max_file_descriptor_count", std::make_shared(), "The max file descriptor count. Only available on Unix platforms."}, if (const auto it = mntr_responses_map.find("zk_max_file_descriptor_count"); it != mntr_responses_map.end()) - res_columns[23]->insert(parse(it->second)); + insert_if_non_negative(*res_columns[23], it->second); else res_columns[23]->insertDefault(); } @@ -287,25 +298,25 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP ///* 24 */ {"connections", std::make_shared(), "The ZooKeeper connections."}, if (const auto it = srvr_responses_map.find("Connections"); it != srvr_responses_map.end()) - res_columns[24]->insert(parse(it->second)); + res_columns[24]->insert(parse(it->second)); else res_columns[24]->insertDefault(); ///* 25 */ {"outstanding", std::make_shared(), "The ZooKeeper outstanding."}, if (const auto it = srvr_responses_map.find("Outstanding"); it != srvr_responses_map.end()) - res_columns[25]->insert(parse(it->second)); + res_columns[25]->insert(parse(it->second)); else res_columns[25]->insertDefault(); //* 26 */ {"zxid", std::make_shared(), "The ZooKeeper zxid."}, if (const auto it = srvr_responses_map.find("Zxid"); it != srvr_responses_map.end()) - res_columns[26]->insert(parseIntInBase<16, int>(it->second.substr(2))); /// we skip the 0x prefix + res_columns[26]->insert(parseIntInBase<16, Int64>(it->second.substr(2))); /// we skip the 0x prefix else res_columns[26]->insertDefault(); //* 27 */ {"node_count", std::make_shared(), "The ZooKeeper node count."}, if (const auto it = srvr_responses_map.find("Node count"); it != srvr_responses_map.end()) - res_columns[27]->insert(parse(it->second)); + res_columns[27]->insert(parse(it->second)); else res_columns[27]->insertDefault(); } @@ -320,13 +331,13 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP //* 28 */ {"snapshot_dir_size", std::make_shared(), "The ZooKeeper snapshot directory size."}, if (const auto it = dirs_responses_map.find("snapshot_dir_size"); it != dirs_responses_map.end()) - res_columns[28]->insert(parse(it->second)); + res_columns[28]->insert(parse(it->second)); else res_columns[28]->insertDefault(); //* 29 */ {"log_dir_size", std::make_shared(), "The ZooKeeper log directory size."}, if (const auto it = dirs_responses_map.find("log_dir_size"); it != dirs_responses_map.end()) - res_columns[29]->insert(parse(it->second)); + res_columns[29]->insert(parse(it->second)); else res_columns[29]->insertDefault(); } @@ -340,49 +351,49 @@ void StorageSystemZooKeeperInfo::fillData(MutableColumns & res_columns, ContextP // /* 30 */ {"first_log_idx", std::make_shared(), "The ZooKeeper first log index."}, if (const auto it = lgif_responses_map.find("first_log_idx"); it != lgif_responses_map.end()) - res_columns[30]->insert(parse(it->second)); + res_columns[30]->insert(parse(it->second)); else res_columns[30]->insertDefault(); // /* 31 */ {"first_log_term", std::make_shared(), "The ZooKeeper first log term."}, if (const auto it = lgif_responses_map.find("first_log_term"); it != lgif_responses_map.end()) - res_columns[31]->insert(parse(it->second)); + res_columns[31]->insert(parse(it->second)); else res_columns[31]->insertDefault(); // /* 32 */ {"last_log_idx", std::make_shared(), "The ZooKeeper last log index."}, if (const auto it = lgif_responses_map.find("last_log_idx"); it != lgif_responses_map.end()) - res_columns[32]->insert(parse(it->second)); + res_columns[32]->insert(parse(it->second)); else res_columns[32]->insertDefault(); // /* 33 */ {"last_log_term", std::make_shared(), "The ZooKeeper last log term."}, if (const auto it = lgif_responses_map.find("last_log_term"); it != lgif_responses_map.end()) - res_columns[33]->insert(parse(it->second)); + res_columns[33]->insert(parse(it->second)); else res_columns[33]->insertDefault(); // /* 34 */ {"last_committed_idx", std::make_shared(), "The ZooKeeper last committed index."}, if (const auto it = lgif_responses_map.find("last_committed_log_idx"); it != lgif_responses_map.end()) - res_columns[34]->insert(parse(it->second)); + res_columns[34]->insert(parse(it->second)); else res_columns[34]->insertDefault(); // /* 35 */ {"leader_committed_log_idx", std::make_shared(), "The ZooKeeper leader committed log index."}, if (const auto it = lgif_responses_map.find("leader_committed_log_idx"); it != lgif_responses_map.end()) - res_columns[35]->insert(parse(it->second)); + res_columns[35]->insert(parse(it->second)); else res_columns[35]->insertDefault(); // /* 36 */ {"target_committed_log_idx", std::make_shared(), "The ZooKeeper target committed log index."}, if (const auto it = lgif_responses_map.find("target_committed_log_idx"); it != lgif_responses_map.end()) - res_columns[36]->insert(parse(it->second)); + res_columns[36]->insert(parse(it->second)); else res_columns[36]->insertDefault(); // /* 37 */ {"last_snapshot_idx", std::make_shared(), "The ZooKeeper last snapshot index."}, if (const auto it = lgif_responses_map.find("last_snapshot_idx"); it != lgif_responses_map.end()) - res_columns[37]->insert(parse(it->second)); + res_columns[37]->insert(parse(it->second)); else res_columns[37]->insertDefault(); } diff --git a/tests/integration/test_zookeeper_info_number_overflow/__init__.py b/tests/integration/test_zookeeper_info_number_overflow/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_zookeeper_info_number_overflow/configs/enable_keeper.xml b/tests/integration/test_zookeeper_info_number_overflow/configs/enable_keeper.xml new file mode 100644 index 000000000000..f07176396ea9 --- /dev/null +++ b/tests/integration/test_zookeeper_info_number_overflow/configs/enable_keeper.xml @@ -0,0 +1,22 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + + 1 + keeper + 9234 + + + + diff --git a/tests/integration/test_zookeeper_info_number_overflow/configs/use_keeper.xml b/tests/integration/test_zookeeper_info_number_overflow/configs/use_keeper.xml new file mode 100644 index 000000000000..39c551b4b30b --- /dev/null +++ b/tests/integration/test_zookeeper_info_number_overflow/configs/use_keeper.xml @@ -0,0 +1,12 @@ + + + + keeper + 9181 + + + keeper + 9998 + + + diff --git a/tests/integration/test_zookeeper_info_number_overflow/mock_keeper_4lw.py b/tests/integration/test_zookeeper_info_number_overflow/mock_keeper_4lw.py new file mode 100644 index 000000000000..21b27ca06a8f --- /dev/null +++ b/tests/integration/test_zookeeper_info_number_overflow/mock_keeper_4lw.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""A fake Keeper four letter command endpoint which responds to `mntr`, +`srvr`, `dirs` and `lgif` with values above 2^31 - 1, as on a Keeper which has +committed more than 2^31 - 1 transactions.""" + +import socket + +RESPONSES = { + b"ruok": "imok", + b"isro": "rw", + b"mntr": ( + "zk_version\tv26.6.1.1-testing\n" + "zk_avg_latency\t0\n" + "zk_packets_received\t3000000000\n" + "zk_packets_sent\t3000000000\n" + "zk_open_file_descriptor_count\t-1\n" + "zk_max_file_descriptor_count\t18446744073709551615\n" + ), + b"srvr": ( + "ClickHouse Keeper version: v26.6.1.1-testing\n" + "Latency min/avg/max: 0/0/0\n" + "Received: 0\n" + "Sent: 0\n" + "Connections: 1\n" + "Outstanding: 0\n" + "Zxid: 0x80000000\n" + "Mode: leader\n" + "Node count: 5\n" + ), + b"dirs": ( + "snapshot_dir_size: 3000000000\n" + "log_dir_size: 4000000000\n" + ), + b"lgif": ( + "first_log_idx\t1\n" + "first_log_term\t1\n" + "last_log_idx\t5000000000\n" + "last_log_term\t1\n" + "last_committed_log_idx\t5000000000\n" + "leader_committed_log_idx\t5000000000\n" + "target_committed_log_idx\t5000000000\n" + "last_snapshot_idx\t2500000000\n" + ), +} + + +def main(): + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", 9998)) + server.listen(10) + while True: + connection, _ = server.accept() + try: + command = connection.recv(4) + connection.sendall(RESPONSES.get(command, "").encode()) + finally: + connection.close() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_zookeeper_info_number_overflow/test.py b/tests/integration/test_zookeeper_info_number_overflow/test.py new file mode 100644 index 000000000000..ccc8d6bc8306 --- /dev/null +++ b/tests/integration/test_zookeeper_info_number_overflow/test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import os +import socket +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) + +# Keeper and server from the current build. The `zookeeper` config contains a +# second host `keeper:9998` - a mock which responds to the four letter commands +# with values above 2^31 - 1, as on a Keeper which has committed more than +# 2^31 - 1 transactions. +keeper = cluster.add_instance( + "keeper", + main_configs=["configs/enable_keeper.xml", "configs/use_keeper.xml"], + stay_alive=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + keeper.copy_file_to_container( + os.path.join(SCRIPT_DIR, "mock_keeper_4lw.py"), "/mock_keeper_4lw.py" + ) + keeper.exec_in_container(["python3", "/mock_keeper_4lw.py"], detach=True) + + mock_address = (cluster.get_instance_ip("keeper"), 9998) + for _ in range(100): + try: + socket.create_connection(mock_address, timeout=1).close() + break + except OSError: + time.sleep(0.1) + else: + raise Exception("Mock four letter command endpoint did not start") + + yield cluster + finally: + cluster.shutdown() + + +def test_zookeeper_info_number_overflow(started_cluster): + # The current version returns values above 2^31 - 1 correctly, turns + # the -1 reported for an unknown file descriptor count into NULL, and + # accepts 2^64 - 1 (RLIM_INFINITY, an unlimited file descriptor limit). + assert ( + keeper.query( + "SELECT zxid, node_count, packets_received, open_file_descriptor_count," + " max_file_descriptor_count, snapshot_dir_size, log_dir_size," + " last_log_idx, last_committed_idx" + " FROM system.zookeeper_info WHERE port = 9998" + ) + == "2147483648\t5\t3000000000\t\\N\t18446744073709551615\t3000000000\t4000000000" + "\t5000000000\t5000000000\n" + ) + + +def test_zookeeper_info_file_descriptor_counts(started_cluster): + # The same values from the real Keeper of this build: `mntr` prints the file + # descriptor counts as unsigned numbers, and the textual -1 it reports for an + # undetermined value must not wrap around to 2^64 - 1 (which is a valid value + # on its own: an unlimited RLIMIT_NOFILE). + open_fd, max_fd = ( + keeper.query( + "SELECT open_file_descriptor_count, max_file_descriptor_count" + " FROM system.zookeeper_info WHERE port = 9181" + ) + .strip() + .split("\t") + ) + + assert open_fd != "\\N" and 0 < int(open_fd) < 2**31 + assert max_fd != "\\N" and int(max_fd) >= int(open_fd) + + +def test_keeper_asynchronous_metrics_file_descriptor_counts(started_cluster): + # The same contract on the sibling surface: `system.asynchronous_metrics` must + # report an undetermined file descriptor count as -1, never as 2^64 - 1. + keeper.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + open_fd, max_fd = ( + keeper.query( + "SELECT" + " maxIf(value, metric = 'KeeperOpenFileDescriptorCount')," + " maxIf(value, metric = 'KeeperMaxFileDescriptorCount')" + " FROM system.asynchronous_metrics" + " WHERE metric IN ('KeeperOpenFileDescriptorCount', 'KeeperMaxFileDescriptorCount')" + ) + .strip() + .split("\t") + ) + + assert 0 < float(open_fd) < 2**31 + assert float(max_fd) == -1 or float(max_fd) >= float(open_fd) From 6d17ffd7692d2787981bc9910fc44cc780525a01 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 15:37:31 +0000 Subject: [PATCH 63/86] Backport #109356 to 26.6: Fix data loss when a merge races ALTER RENAME COLUMN on a column without a default expression --- src/Storages/MergeTree/MergeTask.cpp | 79 ++++-- ...ame_column_no_default_merge_race.reference | 1 + ...ter_rename_column_no_default_merge_race.sh | 224 ++++++++++++++++++ 3 files changed, 287 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.reference create mode 100755 tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.sh diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7307c448cd25..06061e0aafba 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -622,6 +622,24 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const const auto & patch_parts = global_ctx->future_part->patch_parts; + /// Snapshot of pending mutations for the source parts, fetched once and reused for + /// `alter_conversions` below, so the expired-columns check observes the same mutations. + auto parts_info = MergeTreeData::getPartsSnapshotInfo(global_ctx->future_part->parts); + + MergeTreeData::IMutationsSnapshot::Params params + { + .metadata_version = global_ctx->metadata_snapshot->getMetadataVersion(), + .min_part_metadata_version = parts_info.min_metadata_version, + .min_part_data_versions = nullptr, + .max_mutation_versions = nullptr, + .need_data_mutations = false, + .need_alter_mutations = !patch_parts.empty(), + .need_patch_parts = false, + .has_lightweight_delete_parts = parts_info.has_lightweight_delete_parts, + }; + + auto mutations_snapshot = global_ctx->data->getMutationsSnapshot(params); + /// Determine columns that are absent in all source parts—either fully expired or never written—and mark them as /// expired to avoid unnecessary reads or writes during merges. /// @@ -646,12 +664,55 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const columns_present_in_parts.emplace(col.name); } + /// The only live values of a column may sit in the patch parts selected for this merge: a + /// column added by `ADD COLUMN` and then filled by a lightweight `UPDATE` is physically + /// absent from all base parts. Such a column is not expired - expiring it would drop the + /// column from the read set, so the patch would never be requested and its values lost. + NameSet columns_present_in_patch_parts; + for (const auto & patch_part : patch_parts) + { + for (const auto & col : patch_part->getColumns()) + columns_present_in_patch_parts.emplace(col.name); + } + + NameSet storage_column_names; + storage_column_names.reserve(global_ctx->storage_columns.size()); + for (const auto & storage_column : global_ctx->storage_columns) + storage_column_names.emplace(storage_column.name); + + /// A pending `RENAME COLUMN old -> new` is applied on-fly at read time: `storage_columns` + /// already carries `new`, while the source parts still physically store `old`. Treat `new` + /// as present whenever a base or patch part holds the matching `old` name, so the merge does not wrongly + /// expire and drop a not-yet-materialized rename target of a column with no default + /// expression (see #80648). If `old` is itself a live storage column (re-added while the + /// rename is pending), the physical data will belong to `new` only after the rename + /// materializes, so fall back to expiring `new` and let the rename mutation re-derive it. + NameSet renamed_column_targets; + for (const auto & part : global_ctx->future_part->parts) + { + auto conversions = MergeTreeData::getAlterConversionsForPart(part, mutations_snapshot, global_ctx->context +#if CLICKHOUSE_CLOUD + , nullptr +#endif + ); + for (const auto & rename : conversions->getRenameMap()) + { + if ((columns_present_in_parts.contains(rename.rename_from) + || columns_present_in_patch_parts.contains(rename.rename_from)) + && !storage_column_names.contains(rename.rename_from)) + renamed_column_targets.emplace(rename.rename_to); + } + } + const auto & columns_desc = global_ctx->metadata_snapshot->getColumns(); /// Any storage column not present in any part and without a default expression is considered expired for (const auto & storage_column : global_ctx->storage_columns) { - if (!columns_present_in_parts.contains(storage_column.name) && !columns_desc.getDefault(storage_column.name)) + if (!columns_present_in_parts.contains(storage_column.name) + && !columns_present_in_patch_parts.contains(storage_column.name) + && !renamed_column_targets.contains(storage_column.name) + && !columns_desc.getDefault(storage_column.name)) global_ctx->new_data_part->expired_columns.emplace(storage_column.name); } } @@ -736,22 +797,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const addGatheringColumn(global_ctx, BlockOffsetColumn::name, BlockOffsetColumn::type); } - auto parts_info = MergeTreeData::getPartsSnapshotInfo(global_ctx->future_part->parts); - - MergeTreeData::IMutationsSnapshot::Params params - { - .metadata_version = global_ctx->metadata_snapshot->getMetadataVersion(), - .min_part_metadata_version = parts_info.min_metadata_version, - .min_part_data_versions = nullptr, - .max_mutation_versions = nullptr, - .need_data_mutations = false, - .need_alter_mutations = !patch_parts.empty(), - .need_patch_parts = false, - .has_lightweight_delete_parts = parts_info.has_lightweight_delete_parts, - }; - - auto mutations_snapshot = global_ctx->data->getMutationsSnapshot(params); - if (!patch_parts.empty()) { LOG_DEBUG(ctx->log, "Will apply {} patches up to version {}", patch_parts.size(), global_ctx->future_part->part_info.getMutationVersion()); diff --git a/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.reference b/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.reference new file mode 100644 index 000000000000..d86bac9de59a --- /dev/null +++ b/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.sh b/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.sh new file mode 100755 index 000000000000..435943d8d98a --- /dev/null +++ b/tests/queries/0_stateless/04648_alter_rename_column_no_default_merge_race.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/80648, the facet where the +# renamed column has no default expression, so a merge that wrongly expires it loses the values for +# good. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +# The window requires the RENAME mutation to stay unselected while `OPTIMIZE ... FINAL` merges the +# parts, so the parts still store the old column name while the metadata already carries the new +# one. `mt_select_parts_to_mutate_no_free_threads` is the only mechanism that opens it: the pool-size +# thresholds are bypassed on an idle server by the `occupied <= 1` short circuit in +# CompactionStatistics::getMaxSourcePartBytesForMutation, and `SYSTEM STOP MERGES` cannot substitute +# for it because it aborts the explicit OPTIMIZE too ("Cancelled merging parts"). +# +# The failpoint is server-global, so a concurrent copy of this test can clear it mid-window. That +# only ever costs coverage here, never a red, because every assertion holds in both states: the +# positive ones read `system.parts_columns` for a column the merge had to keep either way, and the +# phase-3 one reads the values, which the rename preserves whether or not it has materialized. The +# remaining effect of an early release is that the rename may materialize on only some source parts, +# so OPTIMIZE legitimately refuses a mixed mutation version - `optimize_or_skip` below turns that one +# refusal into a skip. Same trade-off as 03830_vertical_merge_inject_column_after_drop, untagged too. +# +# Every assertion is about what a merge decided, so OPTIMIZE always runs with +# `optimize_throw_if_noop = 1`: a silently skipped merge would otherwise read as a lost column. +disable_failpoint() { + ${CLICKHOUSE_CLIENT} --query="SYSTEM DISABLE FAILPOINT mt_select_parts_to_mutate_no_free_threads" 2>/dev/null || true +} +trap disable_failpoint EXIT + +# A concurrent copy of this test clearing the server-global failpoint lets the pending rename +# materialize on only some of the source parts, and OPTIMIZE then legitimately refuses to merge parts +# with different mutation versions. Return non-zero for that one reason so the phase can skip, and +# keep failing on every other refusal - that is what `optimize_throw_if_noop = 1` is for. +optimize_or_skip() { + local table="$1" err + if err=$(${CLICKHOUSE_CLIENT} --query="OPTIMIZE TABLE ${table} FINAL SETTINGS optimize_throw_if_noop = 1" 2>&1); then + return 0 + fi + case "$err" in + *"have different mutation version"*) return 1 ;; + esac + echo "FAIL (${table}): OPTIMIZE did not run: ${err}" + exit 1 +} + +# For a column with no default, dropping it from the merged part is the data loss - there is nothing +# left to refill it from. Require both that the merged part still carries the column and that it +# carries at least `min_bytes` of data for it, so a column present but rewritten as empty defaults +# does not pass. +assert_kept() { + local table="$1" column="$2" min_bytes="$3" label="$4" + local kept + kept=$(${CLICKHOUSE_CLIENT} --query=" + SELECT sumIf(column_data_uncompressed_bytes, column = '${column}') >= ${min_bytes} + FROM system.parts_columns + WHERE database = currentDatabase() AND table = '${table}' AND active") + if [ "$kept" != "1" ]; then + echo "FAIL (${label}): the merge did not keep the values of ${column}" + ${CLICKHOUSE_CLIENT} --query=" + SELECT name, column, column_data_uncompressed_bytes FROM system.parts_columns + WHERE database = currentDatabase() AND table = '${table}' AND active + ORDER BY name, column" + exit 1 + fi +} + +# Phase 1: a plain column with no default at all. +${CLICKHOUSE_CLIENT} --query=" + DROP TABLE IF EXISTS t_rename_no_default; + CREATE TABLE t_rename_no_default (id UInt64, d String) + ENGINE = MergeTree() ORDER BY id + SETTINGS min_bytes_for_wide_part = 0; + INSERT INTO t_rename_no_default SELECT number, 'payload_value_' || toString(number) FROM numbers(500); + INSERT INTO t_rename_no_default SELECT number, 'payload_value_' || toString(number) FROM numbers(500, 500); + SYSTEM ENABLE FAILPOINT mt_select_parts_to_mutate_no_free_threads; + ALTER TABLE t_rename_no_default RENAME COLUMN d TO d1 SETTINGS alter_sync = 0; +" +if optimize_or_skip t_rename_no_default; then + assert_kept t_rename_no_default d1 5000 "String, no default" +fi +disable_failpoint +${CLICKHOUSE_CLIENT} --query="DROP TABLE t_rename_no_default" + +# Phase 2: the column is Dynamic. Any type reproduces the loss; Dynamic is kept because it cannot +# carry a default expression, so it is a type where the bug is unavoidable rather than observable. +${CLICKHOUSE_CLIENT} --query=" + DROP TABLE IF EXISTS t_rename_no_default_dynamic; + SET allow_experimental_dynamic_type = 1; + CREATE TABLE t_rename_no_default_dynamic (x UInt64, y UInt64) + ENGINE = MergeTree() ORDER BY x + SETTINGS min_bytes_for_wide_part = 0; + INSERT INTO t_rename_no_default_dynamic SELECT number, number FROM numbers(3); + ALTER TABLE t_rename_no_default_dynamic ADD COLUMN d Dynamic SETTINGS mutations_sync = 1; + INSERT INTO t_rename_no_default_dynamic SELECT number, number, number FROM numbers(3, 3); + INSERT INTO t_rename_no_default_dynamic SELECT number, number, 'str_' || toString(number) FROM numbers(6, 3); + INSERT INTO t_rename_no_default_dynamic SELECT number, number, NULL FROM numbers(9, 3); + SYSTEM ENABLE FAILPOINT mt_select_parts_to_mutate_no_free_threads; + ALTER TABLE t_rename_no_default_dynamic RENAME COLUMN d TO d1 SETTINGS alter_sync = 0; +" +if optimize_or_skip t_rename_no_default_dynamic; then + assert_kept t_rename_no_default_dynamic d1 1 "Dynamic, no default" +fi +disable_failpoint +${CLICKHOUSE_CLIENT} --query="DROP TABLE t_rename_no_default_dynamic" + +# Phase 3: the opposite direction - the keep-alive must not over-apply. The old name is re-added +# before the pending rename materializes, so the metadata carries both a and b while the parts still +# store only the pre-rename a. That physical a belongs to b once the rename materializes, so the +# merge must keep it under its own name and must not also claim b as present: doing so would bind one +# set of bytes to two logical columns. +${CLICKHOUSE_CLIENT} --query=" + DROP TABLE IF EXISTS t_rename_no_default_reuse; + CREATE TABLE t_rename_no_default_reuse (id UInt64, a String) + ENGINE = MergeTree() ORDER BY id + SETTINGS min_bytes_for_wide_part = 0; + INSERT INTO t_rename_no_default_reuse VALUES (1, 'AAA'), (2, 'BBB'); + INSERT INTO t_rename_no_default_reuse VALUES (3, 'CCC'), (4, 'DDD'); + SYSTEM STOP MERGES t_rename_no_default_reuse; + SYSTEM ENABLE FAILPOINT mt_select_parts_to_mutate_no_free_threads; + ALTER TABLE t_rename_no_default_reuse RENAME COLUMN a TO b SETTINGS alter_sync = 0; + ALTER TABLE t_rename_no_default_reuse ADD COLUMN a String DEFAULT 'reused_default' SETTINGS alter_sync = 0; + SYSTEM START MERGES t_rename_no_default_reuse; +" +optimize_or_skip t_rename_no_default_reuse || true +# The two ALTERs have to take effect as one step. A merge that runs between them sees no live a, so it +# correctly materializes a physical b that the later re-add cannot remove. STOP MERGES closes that +# window, as in phases 4 and 5. +# +# Assert the values rather than the physical column set. Which columns a part stores is not stable +# here: while the rename is pending only a exists, and once it materializes a part legitimately holds +# both b and the re-added a, so no predicate over `system.parts_columns` can tell the correct shapes +# apart from the defect. A mutation-progress probe cannot fix that either - it aggregates over every +# mutation on the table, so an unrelated pending one makes it claim this rename is unfinished after it +# has materialized. +# +# What the defect costs is b's data: over-applying the keep-alive lets the re-added a claim the +# physical column, so the merge rewrites it from a's default and the pre-rename values are gone. That +# holds whether or not the rename has materialized, so assert exactly it. +# +# Deliberately not asserted: while the rename is still pending, reading logical a also returns b's +# data instead of a's default. That is a read-path defect that reproduces on master with no merge at +# all, so it is out of scope here and is tracked separately. +wrong=$(${CLICKHOUSE_CLIENT} --query=" + SELECT countIf(b NOT IN ('AAA', 'BBB', 'CCC', 'DDD')) + FROM t_rename_no_default_reuse") +if [ "$wrong" != "0" ]; then + echo "FAIL (rename target reused): the merge lost b's data to the re-added a" + ${CLICKHOUSE_CLIENT} --query="SELECT id, b, a FROM t_rename_no_default_reuse ORDER BY id" + ${CLICKHOUSE_CLIENT} --query=" + SELECT name, column, column_data_uncompressed_bytes FROM system.parts_columns + WHERE database = currentDatabase() AND table = 't_rename_no_default_reuse' AND active + ORDER BY name, column" + exit 1 +fi +disable_failpoint +${CLICKHOUSE_CLIENT} --query="DROP TABLE t_rename_no_default_reuse" + +# Phase 4: a column whose only live values are in a patch part. `ADD COLUMN a` plus a lightweight +# `UPDATE` leaves every base part without a physical a, so the merge must not expire it - neither +# under its own name nor as a pending rename target - otherwise the patch is never requested and the +# updated value is silently lost. The own-name case needs no failpoint, so it asserts the values. +${CLICKHOUSE_CLIENT} --query=" + DROP TABLE IF EXISTS t_rename_no_default_patch; + CREATE TABLE t_rename_no_default_patch (id UInt64, v String) + ENGINE = MergeTree() ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 1; + SYSTEM STOP MERGES t_rename_no_default_patch; + INSERT INTO t_rename_no_default_patch VALUES (1, 'x'), (2, 'y'); + INSERT INTO t_rename_no_default_patch VALUES (3, 'z'), (4, 'w'); + ALTER TABLE t_rename_no_default_patch ADD COLUMN a String SETTINGS mutations_sync = 1; +" +${CLICKHOUSE_CLIENT} --enable_lightweight_update=1 --query="UPDATE t_rename_no_default_patch SET a = 'patched' WHERE id = 2" +${CLICKHOUSE_CLIENT} --query="SYSTEM START MERGES t_rename_no_default_patch" +${CLICKHOUSE_CLIENT} --query="OPTIMIZE TABLE t_rename_no_default_patch FINAL SETTINGS optimize_throw_if_noop = 1" + +count=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM t_rename_no_default_patch WHERE a = if(id = 2, 'patched', '')") +if [ "$count" != "4" ]; then + echo "FAIL (patch-only column, own name): expected 4 rows with the patched a preserved, got $count" + ${CLICKHOUSE_CLIENT} --query="SELECT id, v, a FROM t_rename_no_default_patch ORDER BY id" + exit 1 +fi + +${CLICKHOUSE_CLIENT} --query="DROP TABLE t_rename_no_default_patch" + +# Phase 5: the same patch-only column, but as the target of a pending rename. +${CLICKHOUSE_CLIENT} --query=" + DROP TABLE IF EXISTS t_rename_no_default_patch_rename; + CREATE TABLE t_rename_no_default_patch_rename (id UInt64, v String) + ENGINE = MergeTree() ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 1; + SYSTEM STOP MERGES t_rename_no_default_patch_rename; + INSERT INTO t_rename_no_default_patch_rename VALUES (1, 'x'), (2, 'y'); + INSERT INTO t_rename_no_default_patch_rename VALUES (3, 'z'), (4, 'w'); + ALTER TABLE t_rename_no_default_patch_rename ADD COLUMN a String SETTINGS mutations_sync = 1; +" +${CLICKHOUSE_CLIENT} --enable_lightweight_update=1 --query="UPDATE t_rename_no_default_patch_rename SET a = 'patched' WHERE id = 2" +${CLICKHOUSE_CLIENT} --query=" + SYSTEM ENABLE FAILPOINT mt_select_parts_to_mutate_no_free_threads; + ALTER TABLE t_rename_no_default_patch_rename RENAME COLUMN a TO b SETTINGS alter_sync = 0; + SYSTEM START MERGES t_rename_no_default_patch_rename; +" +# The threshold has to exceed what four empty strings occupy (9 bytes here), or preserving the target +# column while dropping the patch value still passes. The value itself cannot be asserted: logical b +# reads back empty until the rename materializes, so an exact-value check would fail on a correct +# server in exactly the state this phase exists to cover. Phase 4 asserts the value because it has no +# pending rename. +if optimize_or_skip t_rename_no_default_patch_rename; then + assert_kept t_rename_no_default_patch_rename b 10 "patch-only column, rename target" +fi +disable_failpoint +${CLICKHOUSE_CLIENT} --query="DROP TABLE t_rename_no_default_patch_rename" + +echo "OK" From 0598788180f94d6c4472896536d2f994c489a151 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 22:50:51 +0000 Subject: [PATCH 64/86] Backport #106928 to 26.6: Fix Invalid number of rows in Chunk in JoiningTransform with additional join filter --- .../HashJoin/HashJoinMethodsImpl.h | 8 +++++++ ...obal_left_semi_chunk_consistency.reference | 2 ++ ...oin_global_left_semi_chunk_consistency.sql | 24 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.reference create mode 100644 tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.sql diff --git a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h index 88ff3c8f13b1..411acf992b31 100644 --- a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h +++ b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h @@ -1195,6 +1195,14 @@ size_t HashJoinMethods::joinRightColumnsWithAddi added_columns.offsets_to_replicate.resize(left_block_rows); added_columns.filter.resize(left_block_rows); } + else if (need_filter) + { + /// The loop above may break early at max_joined_block_rows, producing fewer left rows + /// than the selector size the filter was allocated for. Trim the filter to the number of + /// processed rows so the required right key column built from it matches the left block, + /// which is cut to left_block_rows downstream. + added_columns.filter.resize(left_block_rows); + } added_columns.applyLazyDefaults(); return left_block_rows; } diff --git a/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.reference b/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.reference new file mode 100644 index 000000000000..c461972e40ea --- /dev/null +++ b/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.reference @@ -0,0 +1,2 @@ +41712 +8817 diff --git a/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.sql b/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.sql new file mode 100644 index 000000000000..1171ecef916f --- /dev/null +++ b/tests/queries/0_stateless/04330_join_global_left_semi_chunk_consistency.sql @@ -0,0 +1,24 @@ +-- Tags: no-old-analyzer +-- no-old-analyzer: the mixed equi + inequality JOIN ON is only supported by the analyzer. + +SELECT count() +FROM +( + SELECT t1.generate_series + FROM numbers(41712) AS t0 + LEFT JOIN generateSeries(5297, 67368) AS t1 + ON (t0.number <= t1.generate_series) AND (t1.generate_series = t0.number) + SETTINGS max_joined_block_size_rows = 1000 +); + +SELECT count() +FROM +( + SELECT t1d0.generate_series, [100000000000000000000.] + FROM numbers(41712) AS t0d0 + GLOBAL LEFT JOIN generateSeries(5297, 67368) AS t1d0 + ON (t0d0.number <= t1d0.generate_series) AND (t1d0.generate_series = t0d0.number) + SEMI LEFT JOIN numbers_mt(14075) AS t2d0 USING (number) + LIMIT 39 BY ALL + SETTINGS max_joined_block_size_rows = 1000 +); From 898efe920551aa37f51257b4f6259b59b433fed7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 30 Jul 2026 23:32:16 +0000 Subject: [PATCH 65/86] Backport #112599 to 26.6: Bump `sqlite-amalgamation` from 3.41.2 to 3.53.4 --- contrib/sqlite-amalgamation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/sqlite-amalgamation b/contrib/sqlite-amalgamation index 20598079891d..23c7b7692961 160000 --- a/contrib/sqlite-amalgamation +++ b/contrib/sqlite-amalgamation @@ -1 +1 @@ -Subproject commit 20598079891d27ef1a3ad3f66bbfa3f983c25268 +Subproject commit 23c7b76929611997c5f0315b3149372b1bee1d41 From e5a8309447ececcd99cc2ae7802a7ebbf14c3204 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 08:08:00 +0000 Subject: [PATCH 66/86] Backport #107927 to 26.6: AI functions: retry transient network errors like the url table function --- src/Functions/AI/AnthropicProvider.cpp | 7 +- src/Functions/AI/IAIProvider.cpp | 7 + src/Functions/AI/IAIProvider.h | 25 ++ src/Functions/AI/OpenAIProvider.cpp | 13 +- src/Functions/FunctionBaseAI.cpp | 64 ++++- src/Functions/FunctionBaseAI.h | 7 + src/Functions/aiEmbed.cpp | 21 +- src/IO/HTTPCommon.cpp | 16 ++ src/IO/HTTPCommon.h | 5 + src/IO/ReadWriteBufferFromHTTP.cpp | 16 +- .../test_ai_functions/mock_ai_server.py | 47 +++- tests/integration/test_ai_functions/test.py | 219 ++++++++++++++++++ 12 files changed, 399 insertions(+), 48 deletions(-) diff --git a/src/Functions/AI/AnthropicProvider.cpp b/src/Functions/AI/AnthropicProvider.cpp index d02f1ee04480..3ff042fd3cff 100644 --- a/src/Functions/AI/AnthropicProvider.cpp +++ b/src/Functions/AI/AnthropicProvider.cpp @@ -14,7 +14,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int MALFORMED_AI_PROVIDER_RESPONSE; } @@ -132,9 +131,9 @@ AIResponse AnthropicProvider::call(const AIRequest & ai_request, const Connectio auto status = http_response.getStatus(); if (status != Poco::Net::HTTPResponse::HTTP_OK) { - throw Exception( - ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, - "Anthropic provider error: {}", extractProviderError(response_body, static_cast(status))); + throw AIProviderHTTPException( + status, + PreformattedMessage::create("Anthropic provider error: {}", extractProviderError(response_body, static_cast(status)))); } Poco::JSON::Parser parser; diff --git a/src/Functions/AI/IAIProvider.cpp b/src/Functions/AI/IAIProvider.cpp index f41d88c8446f..81430b683b5d 100644 --- a/src/Functions/AI/IAIProvider.cpp +++ b/src/Functions/AI/IAIProvider.cpp @@ -10,6 +10,13 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; + extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; +} + +AIProviderHTTPException::AIProviderHTTPException(Poco::Net::HTTPResponse::HTTPStatus http_status_, PreformattedMessage msg) + : Exception(std::move(msg), ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER) + , http_status(http_status_) +{ } AIEmbeddingResponse IAIProvider::embed(const AIEmbeddingRequest & /*ai_embedding_request*/, const ConnectionTimeouts & /*timeouts*/) diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index b0652880dedb..0d30a033fd2b 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -1,14 +1,39 @@ #pragma once +#include #include #include #include #include +#include #include namespace DB { +/// Thrown when an AI provider returns a non-2xx HTTP response. Carries the HTTP status code so the +/// retry logic (`FunctionBaseAI::isRetriableProviderError`) can apply the same retriable-status +/// policy as the `url` table function (`isRetriableHTTPError`): deterministic client errors +/// (e.g. 400, 401, 403, 404, 405, 501) are surfaced immediately, while transient/server-side errors +/// are retried. Uses the `RECEIVED_ERROR_FROM_REMOTE_IO_SERVER` error code, as the providers did +/// before the status was preserved, so error messages and `throw_on_error` behavior are unchanged. +class AIProviderHTTPException : public Exception +{ +public: + AIProviderHTTPException(Poco::Net::HTTPResponse::HTTPStatus http_status_, PreformattedMessage msg); + + AIProviderHTTPException * clone() const override { return new AIProviderHTTPException(*this); } + void rethrow() const override { throw *this; } /// NOLINT(cert-err60-cpp) + + Poco::Net::HTTPResponse::HTTPStatus getHTTPStatus() const { return http_status; } + +private: + Poco::Net::HTTPResponse::HTTPStatus http_status; + + const char * name() const noexcept override { return "DB::AIProviderHTTPException"; } + const char * className() const noexcept override { return "DB::AIProviderHTTPException"; } +}; + /** Parameters for a single AI chat completion request. * * Each row processed by an AI function produces one AIRequest. diff --git a/src/Functions/AI/OpenAIProvider.cpp b/src/Functions/AI/OpenAIProvider.cpp index b67b15dffaa1..17875d06806b 100644 --- a/src/Functions/AI/OpenAIProvider.cpp +++ b/src/Functions/AI/OpenAIProvider.cpp @@ -16,7 +16,6 @@ namespace DB namespace ErrorCodes { - extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int MALFORMED_AI_PROVIDER_RESPONSE; } @@ -114,9 +113,9 @@ AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTi auto status = http_response.getStatus(); if (status != Poco::Net::HTTPResponse::HTTP_OK) { - throw Exception( - ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, - "AI provider error: {}", extractProviderError(response_body, static_cast(status))); + throw AIProviderHTTPException( + status, + PreformattedMessage::create("AI provider error: {}", extractProviderError(response_body, static_cast(status)))); } Poco::JSON::Parser parser; @@ -197,9 +196,9 @@ AIEmbeddingResponse OpenAIProvider::embed(const AIEmbeddingRequest & ai_embeddin auto status = http_response.getStatus(); if (status != Poco::Net::HTTPResponse::HTTP_OK) { - throw Exception( - ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, - "AI provider error: {}", extractProviderError(response_body, static_cast(status))); + throw AIProviderHTTPException( + status, + PreformattedMessage::create("AI provider error: {}", extractProviderError(response_body, static_cast(status)))); } Poco::JSON::Parser parser; diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 262918b32655..dc936b30b44d 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -14,6 +17,7 @@ #include #include #include +#include #include #include namespace ProfileEvents @@ -45,7 +49,6 @@ namespace Setting namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int SUPPORT_IS_DISABLED; } @@ -120,6 +123,46 @@ UInt64 FunctionBaseAI::computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 att return delay_ms; } +bool FunctionBaseAI::isRetriableProviderError(std::exception_ptr eptr) +{ + /// Catch order matters: more derived exception types must come first. + try + { + std::rethrow_exception(eptr); + } + catch (const AIProviderHTTPException & e) + { + return isRetriableHTTPError(e.getHTTPStatus()); + } + catch (const NetException &) + { + /// ClickHouse-level network error (e.g. a DNS failure raised by the HTTP connection pool). + return true; + } + catch (const Poco::Net::NetException &) + { + /// Connection refused/reset, TLS connect failure, or an unreachable advertised address. + return true; + } + catch (const Poco::TimeoutException &) + { + /// Connect or receive timeout. + return true; + } + catch (const Poco::IOException & e) + { + /// Write-side transient I/O failure, e.g. a broken pipe (`EPIPE`) when the peer resets the + /// connection mid-request. Out-of-file-descriptors (`EMFILE`) is not retriable. + return e.code() != POCO_EMFILE; + } + catch (...) + { + /// Ok: any other exception is a deterministic argument/usage error (malformed provider + /// response, bad configuration, JSON parse failure, …) — retrying would only repeat it. + return false; + } +} + FunctionBaseAI::ResolvedConfig FunctionBaseAI::resolveConfig() const { auto base = resolveAINamedCollection(getContext(), credentials_collection_name); @@ -230,6 +273,12 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) { + /// Enforce the API-call quota before every provider request, including retries, so a flaky + /// endpoint can't dispatch more than `ai_function_max_api_calls_per_query` requests per query. + /// Kept outside the `try` so a `throw_on_quota_exceeded` throw is not caught by the retry handler. + if (quota.checkQuotas()) + break; + try { AIRequest ai_request; @@ -254,9 +303,11 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, success = true; break; } - catch (const Exception & e) + catch (...) { - if (attempt < max_retries && e.code() == ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER) + /// Retry transient failures (network errors, provider-side HTTP errors) like the + /// `url` table function does; deterministic errors are surfaced immediately. + if (attempt < max_retries && isRetriableProviderError(std::current_exception())) { std::this_thread::sleep_for(std::chrono::milliseconds(computeRetryBackoffMs(retry_delay_ms, attempt))); continue; @@ -267,13 +318,6 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, throw; } - catch (...) /// Handle non-DB exceptions (e.g. Poco network/JSON errors) for throw_on_error semantics - { - if (!throw_on_error) - break; - - throw; - } } result_col->insertData(result.data(), result.size()); diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index ab6bf6da1ff5..0f2ccd146ab1 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -7,6 +7,8 @@ #include #include +#include + namespace DB { @@ -70,6 +72,11 @@ class FunctionBaseAI : public IFunction /// sleep or overflow `std::chrono::milliseconds`. static UInt64 computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt); + /// Whether a failed provider request should be retried: transient network failures and + /// transient/server-side HTTP responses are retriable, deterministic argument/usage errors are not. + /// `eptr` must be the currently handled exception, i.e. `std::current_exception()`. + static bool isRetriableProviderError(std::exception_ptr eptr); + protected: ContextPtr context; ContextPtr getContext() const { return context; } diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index b2b0630dd78e..10d061497715 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -24,6 +24,7 @@ #include #include +#include /// std::current_exception for retry classification #include /// thread::sleep for retry backoff namespace ProfileEvents @@ -56,7 +57,6 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; - extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int SUPPORT_IS_DISABLED; } @@ -220,6 +220,12 @@ class FunctionAiEmbed final : public IFunction bool batch_ok = false; for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) { + /// Enforce the API-call quota before every provider request, including retries, so a flaky + /// endpoint can't dispatch more than `ai_function_max_api_calls_per_query` requests per query. + /// Kept outside the `try` so a `throw_on_quota_exceeded` throw is not caught by the retry handler. + if (quota.checkQuotas()) + break; + try { /// update api_calls/quotas before call so failed calls are still added to total @@ -231,9 +237,11 @@ class FunctionAiEmbed final : public IFunction batch_ok = true; break; } - catch (const Exception & e) + catch (...) { - if (attempt < max_retries && e.code() == ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER) + /// Retry transient failures (network errors, provider-side HTTP errors) like the + /// `url` table function does; deterministic errors are surfaced immediately. + if (attempt < max_retries && FunctionBaseAI::isRetriableProviderError(std::current_exception())) { std::this_thread::sleep_for(std::chrono::milliseconds(FunctionBaseAI::computeRetryBackoffMs(retry_delay_ms, attempt))); continue; @@ -244,13 +252,6 @@ class FunctionAiEmbed final : public IFunction throw; } - catch (...) /// Handle non-DB exceptions (e.g. Poco network/JSON errors) for throw_on_error semantics - { - if (!throw_on_error) /// just skip to next batch, this batch's rows will be filled with empty arrays - break; - - throw; - } } if (!batch_ok) /// failed batch's rows are filled in by the next batch (or the final tail fill) diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 4a5f46449eec..3bd91cb9950c 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -18,6 +18,8 @@ #endif +#include +#include #include #include @@ -60,6 +62,20 @@ HTTPSessionPtr makeHTTPSession( bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status) { return status == Poco::Net::HTTPResponse::HTTP_MOVED_PERMANENTLY || status == Poco::Net::HTTPResponse::HTTP_FOUND || status == Poco::Net::HTTPResponse::HTTP_SEE_OTHER || status == Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT; } +bool isRetriableHTTPError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept +{ + static constexpr std::array non_retriable_errors{ + Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED}; + + return std::all_of( + non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; }); +} + std::istream * receiveResponse( Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, const bool allow_redirects) { diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index abf249cddff2..318f9e356ad3 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -68,6 +68,11 @@ HTTPSessionPtr makeHTTPSession( bool isRedirect(Poco::Net::HTTPResponse::HTTPStatus status); +/// Whether an HTTP error response is worth retrying. Deterministic client errors (bad request, +/// unauthorized, forbidden, not found, method not allowed, not implemented) are not retriable; +/// everything else (transient/server-side errors, rate limiting, …) is. +bool isRetriableHTTPError(Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept; + /** Used to receive response (response headers and possibly body) * after sending data (request headers and possibly body). * Throws exception in case of non HTTP_OK (200) response code. diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index 02bcbe63360d..ed18ae1feafd 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -19,20 +19,6 @@ namespace ProfileEvents namespace { -bool isRetriableError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept -{ - static constexpr std::array non_retriable_errors{ - Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST, - Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED, - Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND, - Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN, - Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED, - Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED}; - - return std::all_of( - non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; }); -} - Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPResponse & response, bool enable_url_encoding) { chassert(DB::isRedirect(response.getStatus())); @@ -343,7 +329,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, } catch (HTTPException & e) { - if (!isRetriableError(e.getHTTPStatus())) + if (!isRetriableHTTPError(e.getHTTPStatus())) is_retriable = false; error_message = e.displayText(); diff --git a/tests/integration/test_ai_functions/mock_ai_server.py b/tests/integration/test_ai_functions/mock_ai_server.py index 1c42fa00e32b..096e11e233ea 100644 --- a/tests/integration/test_ai_functions/mock_ai_server.py +++ b/tests/integration/test_ai_functions/mock_ai_server.py @@ -8,6 +8,11 @@ `aiTranslate`'s `instructions` argument is forwarded in the prompt, or that the `Authorization` header is omitted when the named collection has no `api_key`). Header names are lower-cased for case-insensitive lookup. + GET /set-flaky?count=N — arm the flaky endpoints below to fail their next N requests + with a simulated transient network error (used to exercise retries). `count=0` disarms. + POST /v1/chat/flaky — like `/v1/chat/completions`, but drops the connection without + a response for the first N requests after `/set-flaky?count=N`, then succeeds. + POST /v1/embeddings_flaky — like `/v1/embeddings`, but flaky in the same way as above. POST /v1/chat/completions — returns response based on request content: - If response_format with json_schema is present, returns JSON matching the schema with values derived from the user message. @@ -20,13 +25,16 @@ element, exercising the duplicate-index rejection path. POST /v1/embeddings_wrong_count — returns one fewer entry than requested, exercising the cardinality mismatch path. - POST /v1/error — always returns HTTP 500 (used for chat completion errors) + POST /v1/error — always returns HTTP 500, a transient/server-side error that + the url table function (and so the AI functions) retries. + POST /v1/bad_request — always returns HTTP 400, a deterministic client error that + the url table function never retries, used to assert AI functions do not retry it either. POST /v1/embeddings_error — always returns HTTP 500 (used for embedding errors) """ import http.server import json -from urllib.parse import urlparse +from urllib.parse import urlparse, parse_qs MOCK_PORT = 18123 DEFAULT_EMBED_DIM = 4 @@ -34,6 +42,11 @@ # Single-threaded `HTTPServer` handles one request at a time, so a plain dict is safe. LAST_REQUEST = {"path": None, "body": None, "headers": {}} +# Number of upcoming requests to the flaky endpoints (`/v1/chat/flaky`, `/v1/embeddings_flaky`) +# that should fail with a simulated transient network error before they start succeeding. +# Set via `GET /set-flaky?count=N`. Used to exercise the network-error retry path. +FLAKY = {"fails_remaining": 0} + def extract_user_message(body): data = json.loads(body) @@ -143,6 +156,15 @@ def do_GET(self): self._send_json(200, LAST_REQUEST) return + if parsed.path == "/set-flaky": + qs = parse_qs(parsed.query) + FLAKY["fails_remaining"] = int(qs.get("count", ["0"])[0]) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"OK") + return + self.send_response(404) self.end_headers() @@ -155,6 +177,21 @@ def do_POST(self): LAST_REQUEST["body"] = body LAST_REQUEST["headers"] = {k.lower(): v for k, v in self.headers.items()} + if parsed.path in ("/v1/chat/flaky", "/v1/embeddings_flaky"): + if FLAKY["fails_remaining"] > 0: + FLAKY["fails_remaining"] -= 1 + # Simulate a transient network failure: close the connection without sending any + # response, so the client sees EOF — a Poco network exception — rather than an HTTP + # error status. This exercises the network-error retry path, distinct from the HTTP + # 500 path (`/v1/error`). + self.close_connection = True + return + if parsed.path == "/v1/chat/flaky": + self._send_json(200, make_success_response(extract_user_message(body))) + else: + self._send_json(200, make_embeddings_response(body)) + return + if parsed.path == "/v1/chat/completions": user_msg = extract_user_message(body) json_schema = extract_response_format(body) @@ -171,6 +208,12 @@ def do_POST(self): self._send_json(500, make_error_response("permanent failure")) return + if parsed.path == "/v1/bad_request": + # A deterministic client error (e.g. malformed request / bad API key). The url table + # function never retries 400, so neither should the AI functions. + self._send_json(400, make_error_response("invalid request", error_type="invalid_request_error")) + return + if parsed.path == "/v1/embeddings": self._send_json(200, make_embeddings_response(body)) return diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index df03a1b5541a..463b0b060a47 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -101,6 +101,14 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-model', " f"api_key = 'test-key'" ) + # Endpoint returning a deterministic HTTP 400, which the url table function never retries. + instance.query( + f"CREATE NAMED COLLECTION ai_bad_request AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/bad_request', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) # `api_key` is optional (some providers, e.g. a local Ollama, need no auth). # This collection omits it so we can assert no `Authorization` header is sent. instance.query( @@ -137,6 +145,22 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-embed-model', " f"api_key = 'test-key'" ) + # Endpoints that drop the connection for the first N requests (armed via /set-flaky), + # used to test that transient network failures are retried like the url table function. + instance.query( + f"CREATE NAMED COLLECTION ai_flaky AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/flaky', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_embed_flaky AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_flaky', " + f"model = 'test-embed-model', " + f"api_key = 'test-key'" + ) instance.query("CREATE TABLE test_input (x String) ENGINE = Memory") instance.query( @@ -641,3 +665,198 @@ def test_embed_quota_input_tokens_exceeded(started_cluster): # value due to a quota cut, matching the documented `AIRowsSkipped` semantics. assert int(events["rows_processed"]) == 1 assert int(events["rows_skipped"]) == 3 + + +# --------------------------------------------------------------------------- +# Retry on transient network errors (like the url table function) +# --------------------------------------------------------------------------- + + +def set_flaky(count): + """Arm the mock's flaky endpoints to fail their next `count` requests with a dropped + connection (a transient network error). `count=0` disarms them.""" + instance.exec_in_container( + ["curl", "-s", f"http://localhost:{MOCK_PORT}/set-flaky?count={count}"] + ) + + +def test_generate_retries_on_network_error(started_cluster): + """A transient network failure (connection dropped without a response) is retried, matching + the url table function. With enough retries the call recovers and ultimately succeeds.""" + set_flaky(2) + qid = unique_query_id("gen_retry_net") + result = instance.query( + "SELECT aiGenerate('recover me')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_flaky", + "ai_function_max_retries": 5, + }, + query_id=qid, + ) + assert result.strip() == "recover me" + events = get_profile_events(qid) + # 2 failed attempts + 1 successful attempt for the single row. + assert int(events["api_calls"]) == 3 + assert int(events["rows_processed"]) == 1 + + +def test_generate_network_error_not_retried_when_disabled(started_cluster): + """With `ai_function_max_retries = 0`, a network failure is surfaced rather than retried.""" + set_flaky(10) + try: + error = instance.query_and_get_error( + "SELECT aiGenerate('no retry')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_flaky", + "ai_function_max_retries": 0, + }, + ) + assert error # a network/IO error is raised instead of a result + finally: + set_flaky(0) + + +def test_embed_retries_on_network_error(started_cluster): + """The embedding path retries transient network failures too.""" + set_flaky(2) + qid = unique_query_id("embed_retry_net") + result = instance.query( + "SELECT aiEmbed('hello')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_embed_flaky", + "ai_function_max_retries": 5, + }, + query_id=qid, + ) + vec = parse_embedding(result) + assert len(vec) == 4 # DEFAULT_EMBED_DIM in mock server + events = get_profile_events(qid) + assert int(events["api_calls"]) == 3 + assert int(events["rows_processed"]) == 1 + + +# --------------------------------------------------------------------------- +# Provider HTTP-status retry policy (matches the url table function): +# deterministic client errors (400/401/403/404/405/501) are surfaced immediately, +# transient/server-side errors (5xx, …) are retried. +# --------------------------------------------------------------------------- + + +def test_generate_deterministic_http_error_not_retried(started_cluster): + """A deterministic provider HTTP status (400 Bad Request) is surfaced immediately and is NOT + retried, even with `ai_function_max_retries` enabled — exactly like the url table function, + which never retries 400/401/403/404/405/501. Only a single API call is made.""" + qid = unique_query_id("gen_400_no_retry") + result = instance.query( + "SELECT aiGenerate('bad request')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_bad_request", + "ai_function_max_retries": 5, + "ai_function_throw_on_error": 0, + }, + query_id=qid, + ) + # Non-retriable error with throw_on_error = 0: the row is skipped, producing an empty result. + assert result.strip() == "" + events = get_profile_events(qid) + assert int(events["api_calls"]) == 1 # exactly one call: the 400 was not retried + assert int(events["rows_processed"]) == 0 + assert int(events["rows_skipped"]) == 1 + + +def test_generate_deterministic_http_error_throws(started_cluster): + """With the default `ai_function_throw_on_error = 1`, the deterministic 400 surfaces as + `RECEIVED_ERROR_FROM_REMOTE_IO_SERVER` rather than being retried away.""" + error = instance.query_and_get_error( + "SELECT aiGenerate('bad request')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_bad_request", + "ai_function_max_retries": 5, + }, + ) + assert "RECEIVED_ERROR_FROM_REMOTE_IO_SERVER" in error + + +def test_generate_server_error_is_retried(started_cluster): + """Counterpart to the 400 case: an HTTP 500 is a transient/server-side error, so it IS retried + (1 initial attempt + `ai_function_max_retries` retries), matching the url table function.""" + qid = unique_query_id("gen_500_retried") + result = instance.query( + "SELECT aiGenerate('server error')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_error", + "ai_function_max_retries": 2, + "ai_function_retry_initial_delay_ms": 1, # keep the test fast + "ai_function_throw_on_error": 0, + }, + query_id=qid, + ) + assert result.strip() == "" + events = get_profile_events(qid) + assert int(events["api_calls"]) == 3 # 1 + 2 retries + assert int(events["rows_skipped"]) == 1 + + +# --------------------------------------------------------------------------- +# The API-call quota bounds retries: `ai_function_max_api_calls_per_query` caps the +# total number of HTTP requests per query, including retried requests, so a flaky +# endpoint cannot dispatch `1 + ai_function_max_retries` requests for a single row/batch. +# --------------------------------------------------------------------------- + + +def test_generate_retry_respects_api_call_quota(started_cluster): + """An HTTP 500 is retriable, but the API-call quota is enforced before every attempt — including + retries. With `ai_function_max_api_calls_per_query = 1` and `ai_function_max_retries = 5`, only a + single request is dispatched (the quota stops the retries), not `1 + 5`.""" + qid = unique_query_id("gen_quota_caps_retries") + result = instance.query( + "SELECT aiGenerate('server error')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_error", + "ai_function_max_retries": 5, + "ai_function_retry_initial_delay_ms": 1, # keep the test fast + "ai_function_max_api_calls_per_query": 1, + "ai_function_throw_on_error": 0, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + assert result.strip() == "" + events = get_profile_events(qid) + # Without the per-attempt quota check this would be 6 (1 initial + 5 retries). + assert int(events["api_calls"]) == 1 + assert int(events["rows_processed"]) == 0 + assert int(events["rows_skipped"]) == 1 + + +def test_embed_retry_respects_api_call_quota(started_cluster): + """The embedding path enforces the same per-attempt API-call quota: a retriable HTTP 500 is not + retried past `ai_function_max_api_calls_per_query`.""" + qid = unique_query_id("embed_quota_caps_retries") + result = instance.query( + "SELECT aiEmbed('server error')", + settings={ + **AI_SETTINGS, + "ai_function_credentials": "ai_embed_error", + "ai_function_max_retries": 5, + "ai_function_retry_initial_delay_ms": 1, # keep the test fast + "ai_function_max_api_calls_per_query": 1, + "ai_function_throw_on_error": 0, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + # The single live row is skipped (empty array) because its batch never succeeded. + assert parse_embedding(result) == [] + events = get_profile_events(qid) + # Without the per-attempt quota check this would be 6 (1 initial + 5 retries). + assert int(events["api_calls"]) == 1 + assert int(events["rows_processed"]) == 0 + assert int(events["rows_skipped"]) == 1 From e2f2446f8b7b0d1fb7a76dff4017236bb363ef3b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 11:21:03 +0000 Subject: [PATCH 67/86] Backport #111386 to 26.6: Analyzer: fix qualified CTE column references in views --- src/Analyzer/QueryTreeBuilder.cpp | 10 +++- ...er_view_cte_qualified_identifier.reference | 24 +++++++++ ...analyzer_view_cte_qualified_identifier.sql | 52 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.reference create mode 100644 tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.sql diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 1f4101d083bf..5fb65c1bba74 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -1009,7 +1009,15 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(bool is_subquery, const ASTSele auto & subquery_expression = table_expression.subquery->as(); const auto & select_with_union_query = subquery_expression.children[0]; - auto node = buildSelectWithUnionExpression(select_with_union_query, true /*is_subquery*/, {} /*cte_name*/, select_query.aliases(), context); + /// Views store CTE references in FROM as subqueries with cte_name set (ApplyWithSubqueryVisitor). + /// Propagate it so qualified identifiers like `cte_name.column` still bind, as they do when + /// a CTE reference is resolved from the WITH section directly. + auto node = buildSelectWithUnionExpression( + select_with_union_query, + true /*is_subquery*/, + CommonTableExpressionData{.cte_name = subquery_expression.cte_name}, + select_query.aliases(), + context); node->setAlias(subquery_expression.tryGetAlias()); node->setOriginalAST(select_with_union_query); diff --git a/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.reference b/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.reference new file mode 100644 index 000000000000..e007a86a839d --- /dev/null +++ b/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.reference @@ -0,0 +1,24 @@ +-- view created with enable_analyzer = 0, aliased CTE reference +1 +1 +-- view created with enable_analyzer = 1 +1 +1 +-- qualification by both cte name and alias +1 1 +1 1 +-- unaliased CTE reference +1 +1 +-- UNION CTE body +1 +2 +1 +2 +-- SHOW CREATE is unchanged +WITH c AS (SELECT 1 AS x) SELECT c.x AS f FROM c AS s +-- two references to the same CTE with different aliases +1 +-- materialized view with an aliased CTE reference +1 +2 diff --git a/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.sql b/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.sql new file mode 100644 index 000000000000..fcb3ddb06635 --- /dev/null +++ b/tests/queries/0_stateless/04614_analyzer_view_cte_qualified_identifier.sql @@ -0,0 +1,52 @@ +-- Views expand CTE references in FROM into subqueries with cte_name set +-- (ApplyWithSubqueryVisitor). The analyzer must propagate cte_name so +-- qualified identifiers like `c.x` still bind. +-- https://github.com/ClickHouse/clickhouse-private/issues/55715#issuecomment-5004050349 + +DROP TABLE IF EXISTS v_cte_old, v_cte_new, v_cte_both, v_cte_noalias, v_cte_union, v_cte_double, mv_cte, t_mv_src; + +SELECT '-- view created with enable_analyzer = 0, aliased CTE reference'; +SET enable_analyzer = 0; +CREATE VIEW v_cte_old AS WITH c AS (SELECT 1 AS x) SELECT c.x AS f FROM c AS s; +SELECT * FROM v_cte_old SETTINGS enable_analyzer = 0; +SELECT * FROM v_cte_old SETTINGS enable_analyzer = 1; + +SELECT '-- view created with enable_analyzer = 1'; +SET enable_analyzer = 1; +CREATE VIEW v_cte_new AS WITH c AS (SELECT 1 AS x) SELECT c.x AS f FROM c AS s; +SELECT * FROM v_cte_new SETTINGS enable_analyzer = 0; +SELECT * FROM v_cte_new SETTINGS enable_analyzer = 1; + +SELECT '-- qualification by both cte name and alias'; +CREATE VIEW v_cte_both AS WITH c AS (SELECT 1 AS x) SELECT c.x AS cx, s.x AS sx FROM c AS s; +SELECT * FROM v_cte_both SETTINGS enable_analyzer = 0; +SELECT * FROM v_cte_both SETTINGS enable_analyzer = 1; + +SELECT '-- unaliased CTE reference'; +CREATE VIEW v_cte_noalias AS WITH c AS (SELECT 1 AS x) SELECT c.x AS f FROM c; +SELECT * FROM v_cte_noalias SETTINGS enable_analyzer = 0; +SELECT * FROM v_cte_noalias SETTINGS enable_analyzer = 1; + +SELECT '-- UNION CTE body'; +CREATE VIEW v_cte_union AS WITH c AS (SELECT 1 AS x UNION ALL SELECT 2 AS x) SELECT c.x AS f FROM c AS s; +SELECT * FROM v_cte_union ORDER BY f SETTINGS enable_analyzer = 0; +SELECT * FROM v_cte_union ORDER BY f SETTINGS enable_analyzer = 1; + +SELECT '-- SHOW CREATE is unchanged'; +SELECT replaceRegexpOne(create_table_query, '.*AS WITH', 'WITH') FROM system.tables WHERE database = currentDatabase() AND name = 'v_cte_old'; + +SELECT '-- two references to the same CTE with different aliases'; +CREATE VIEW v_cte_double AS WITH c AS (SELECT 1 AS x) SELECT c.x AS f FROM c AS s1, c AS s2; +SELECT * FROM v_cte_double SETTINGS enable_analyzer = 0; -- { serverError AMBIGUOUS_COLUMN_NAME } +SELECT * FROM v_cte_double SETTINGS enable_analyzer = 1; + +SELECT '-- materialized view with an aliased CTE reference'; +CREATE TABLE t_mv_src (a UInt8) ENGINE = MergeTree ORDER BY a; +CREATE MATERIALIZED VIEW mv_cte ENGINE = MergeTree ORDER BY f AS WITH c AS (SELECT a AS x FROM t_mv_src) SELECT c.x AS f FROM c AS s; +SET enable_analyzer = 0; +INSERT INTO t_mv_src VALUES (1); +SET enable_analyzer = 1; +INSERT INTO t_mv_src VALUES (2); +SELECT * FROM mv_cte ORDER BY f; + +DROP TABLE v_cte_old, v_cte_new, v_cte_both, v_cte_noalias, v_cte_union, v_cte_double, mv_cte, t_mv_src; From 6120a6fb27efe0d740c8f075aee499215d2f23e1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 11:22:13 +0000 Subject: [PATCH 68/86] Backport #111103 to 26.6: Cache deduplication data hashes before the Alias hop --- src/Interpreters/InsertDeduplication.cpp | 132 +++++++++++++++--- src/Interpreters/InsertDeduplication.h | 30 +++- .../InsertDependenciesBuilder.cpp | 25 +++- src/Storages/MergeTree/MergeTreeSink.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- src/Storages/StorageAlias.cpp | 14 ++ ...eduplication_alias_hop_row_drift.reference | 9 ++ ...04613_deduplication_alias_hop_row_drift.sh | 122 ++++++++++++++++ ...ion_alias_hop_partitioned_target.reference | 6 + ...uplication_alias_hop_partitioned_target.sh | 83 +++++++++++ ...on_alias_hop_deduplicating_inner.reference | 5 + ...plication_alias_hop_deduplicating_inner.sh | 76 ++++++++++ ...ication_alias_hop_insert_into_mv.reference | 5 + ..._deduplication_alias_hop_insert_into_mv.sh | 91 ++++++++++++ ...ion_alias_hop_regular_table_root.reference | 5 + ...uplication_alias_hop_regular_table_root.sh | 78 +++++++++++ ...n_alias_hop_partitioned_no_dedup.reference | 2 + ...lication_alias_hop_partitioned_no_dedup.sh | 54 +++++++ 18 files changed, 718 insertions(+), 23 deletions(-) create mode 100644 tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.reference create mode 100755 tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.sh create mode 100644 tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.reference create mode 100755 tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.sh create mode 100644 tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.reference create mode 100755 tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.sh create mode 100644 tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.reference create mode 100755 tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.sh create mode 100644 tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.reference create mode 100755 tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.sh create mode 100644 tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.reference create mode 100755 tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.sh diff --git a/src/Interpreters/InsertDeduplication.cpp b/src/Interpreters/InsertDeduplication.cpp index ad04ece0e6e4..c259bf7fb323 100644 --- a/src/Interpreters/InsertDeduplication.cpp +++ b/src/Interpreters/InsertDeduplication.cpp @@ -152,13 +152,33 @@ DeduplicationInfo::FilterResult DeduplicationInfo::deduplicateSelf(bool deduplic } -DeduplicationInfo::Ptr DeduplicationInfo::filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index) const +DeduplicationInfo::Ptr DeduplicationInfo::filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index, bool deduplication_enabled) const { /// An empty selector means the block was not split (single partition); with dedup off or a /// single token there is nothing to attribute. Every token then belongs to this partition. - if (disabled || row_to_partition.empty() || getCount() <= 1) + /// When the sink does not deduplicate at all (`deduplication_enabled` is false, e.g. the + /// deduplication window of the table is 0), the tokens are never registered, so there is + /// nothing to attribute either - and the consistency check below must not reject the insert. + if (disabled || !deduplication_enabled || row_to_partition.empty() || getCount() <= 1) return cloneSelf(); + /// Attributing tokens to partitions walks each token's row range over the selector, which is + /// only possible while the offsets still describe the block that was split. Behind an `Alias` + /// hop over a row-count-changing view the deduplication info is re-anchored to the view-output + /// chunks and there is no mapping from the tokens' source rows to the selector anymore. Refuse + /// loudly instead of reading out of the selector's bounds (see filterImpl). + if (row_to_partition.size() != getRows()) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Cannot attribute {} deduplication tokens to the partitions of the insert: the deduplication info " + "describes {} rows, but the block was split into partitions over {} rows because a materialized view " + "with a row-count-changing inner query was processed before a table with the `Alias` engine. " + "Debug: {}", + getCount(), + getRows(), + row_to_partition.size(), + debug()); + /// Keep only tokens that have at least one row in this partition. std::set absent_offsets; for (size_t i = 0; i < offsets.size(); ++i) @@ -269,10 +289,15 @@ DeduplicationInfo::FilterResult DeduplicationInfo::filterImpl(const std::setoriginal_block = std::make_shared(original_block->cloneEmpty()); @@ -280,8 +305,8 @@ DeduplicationInfo::FilterResult DeduplicationInfo::filterImpl(const std::setoriginal_block, .deduplication_info = new_tokens, - .removed_rows = getTokenRows(0), - .removed_tokens = 1, + .removed_rows = original_block->rows() > 0 ? original_block->rows() : getRows(), + .removed_tokens = getCount(), }; } @@ -289,6 +314,24 @@ DeduplicationInfo::FilterResult DeduplicationInfo::filterImpl(const std::set DeduplicationInfo::getDeduplicationHashes(const s } +void DeduplicationInfo::cacheDataHashes() const +{ + if (disabled) + return; + + for (size_t offset = 0; offset < offsets.size(); ++offset) + { + if (!tokens[offset].by_user.empty() || tokens[offset].data_hash_batch.has_value()) + continue; + + chassert(original_block); + calculateDataHashColumnWise(offset, *original_block); + } +} + + +void DeduplicationInfo::cacheDataHashes(DataHashCache & cache) const +{ + if (disabled) + return; + + /// A cache hit: this info is a sibling clone of the one that filled the cache (same source + /// block, same token boundaries). Copy the already-computed hashes instead of re-hashing. + if (cache.block && cache.block.get() == original_block.get() && cache.offsets == offsets) + { + chassert(cache.hashes.size() == tokens.size()); + for (size_t offset = 0; offset < tokens.size(); ++offset) + if (!tokens[offset].data_hash_batch.has_value()) + tokens[offset].data_hash_batch = cache.hashes[offset]; + return; + } + + cacheDataHashes(); + + /// Remember the computed hashes so the sibling clones of this source block reuse them. + cache.block = original_block; + cache.offsets = offsets; + cache.hashes.resize(tokens.size()); + for (size_t offset = 0; offset < tokens.size(); ++offset) + cache.hashes[offset] = tokens[offset].data_hash_batch; +} + + void DeduplicationInfo::prewarmDataHashes() const { if (!original_block || !original_block->rows()) @@ -905,7 +991,15 @@ void DeduplicationInfo::truncateTokensForRetry() Block DeduplicationInfo::goRetry(SharedHeader && header, Chunk && filtered_data, Ptr filtered_info, const std::string & partition_id, ContextPtr context) const { - bool is_empty = !filtered_data || filtered_data.getNumRows() == 0; + // in case all rows are filtered out + // we should not run the pipeline + // because no data no results + // otherwise we can end up in a cycle when all data is filtered by inner query return not empty aggregate result + /// Do not even build the retry chain: the callers only check that the result is empty, and + /// behind a table with the `Alias` engine the visited views belong to the outer insert chain, + /// so `insert_dependencies` of the nested chain cannot rebuild them (see createRetry). + if (!filtered_data || filtered_data.getNumRows() == 0) + return header->cloneEmpty(); auto builder = QueryPipelineBuilder(); builder.init(Pipe(std::make_shared(std::move(header), std::move(filtered_data)))); @@ -916,13 +1010,6 @@ Block DeduplicationInfo::goRetry(SharedHeader && header, Chunk && filtered_data, auto result_header = pipeline.getSharedHeader(); - // in case all rows are filtered out - // we should not run the pipeline - // because no data no results - // otherwise we can end up in a cycle when all data is filtered by inner query return not empty aggregate result - if (is_empty) - return result_header->cloneEmpty(); - auto filter =[this, filtered_info] (const Chunk & chunk) -> bool { auto info = chunk.getChunkInfos().get(); @@ -1361,7 +1448,20 @@ void DeduplicationInfo::TokenDefinition::doExtend(const TokenDefinition & right) return; data_hash.reset(); // invalidate data hash as token is changed - data_hash_batch.reset(); + + /// A VIEW_NUMBER range extension merges chunks a view produced from the same source block: + /// `canBeExtended` required all preceding extras (including SOURCE_NUMBER) to be equal, and the + /// cached hash is computed over the source block's token range, not over the view-output chunks. + /// So when both sides carry the same cached hash, it is still valid for the merged token — and it + /// must be kept: after the merge the info may be re-anchored to a block whose rows no longer + /// match the offsets (e.g. the nested INSERT behind an Alias hop), and recomputing would read + /// out of the block's bounds. For any other extension the token covers new data — invalidate. + const bool keep_cached_hash = left_last_extra.type == Extra::Type::VIEW_NUMBER + && data_hash_batch.has_value() + && right.data_hash_batch == data_hash_batch; + + if (!keep_cached_hash) + data_hash_batch.reset(); // invalidate cached data hash as the token's data has changed // type is equal but values are different switch (left_last_extra.type) diff --git a/src/Interpreters/InsertDeduplication.h b/src/Interpreters/InsertDeduplication.h index 89c6ef290057..0708aea9d9d4 100644 --- a/src/Interpreters/InsertDeduplication.h +++ b/src/Interpreters/InsertDeduplication.h @@ -97,7 +97,12 @@ class DeduplicationInfo : public ChunkInfo FilterResult deduplicateSelf(bool deduplication_enabled, const std::string & partition_id, ContextPtr context) const; FilterResult deduplicateBlock(const std::vector & existing_block_ids, const std::string & partition_id, ContextPtr context) const; - Ptr filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index) const; + /// `deduplication_enabled` mirrors the flag the sink passes to `deduplicateSelf`: when the target + /// table does not deduplicate, no token is attributed to a partition and the drift check inside + /// must not reject the insert. It is trailing and defaults to the fail-close value (`true`, the + /// check active) so that the signature stays source-compatible with the previous one - the unit + /// tests are compiled against the code without this fix by the bugfix validation job. + Ptr filterToPartition(const PaddedPODArray & row_to_partition, size_t partition_index, bool deduplication_enabled = true) const; std::vector getDeduplicationHashes(const std::string & partition_id, bool deduplication_enabled) const; @@ -128,6 +133,29 @@ class DeduplicationInfo : public ChunkInfo void setInsertDependencies(InsertDependenciesBuilderConstPtr insert_dependencies_); void updateOriginalBlock(const Chunk & chunk, SharedHeader header); + /// Compute and cache the data hashes of all tokens while `original_block` still matches + /// `offsets`. Must be called before handing the info to a nested INSERT pipeline (an `Alias` + /// hop) whose squashing and `AddDeduplicationInfoTransform` re-anchor `original_block` to the + /// chunks of that pipeline: a dependent view with a row-count-changing inner query makes those + /// chunks differ from the rows the offsets describe, and a hash computed after such + /// re-anchoring would read out of the block's bounds. + void cacheDataHashes() const; + + /// Memoizing overload for the `Alias` hop. `RestoreChunkInfosTransform` clones the same + /// source-level info onto every output chunk of a row-count-changing view, and the clones do + /// not share `data_hash_batch` (tokens are copied by value). Caching per chunk would re-hash + /// the whole source block once per emitted chunk - O(source_rows * output_chunks). All those + /// clones carry the same `original_block` and `offsets` and differ only in view-block numbers, + /// which do not affect the hashed range, so the per-token hashes are computed once for a given + /// block and reused across its clones. + struct DataHashCache + { + std::shared_ptr block; + std::vector offsets; + std::vector> hashes; + }; + void cacheDataHashes(DataHashCache & cache) const; + const std::vector & getVisitedViews() const; private: diff --git a/src/Interpreters/InsertDependenciesBuilder.cpp b/src/Interpreters/InsertDependenciesBuilder.cpp index d5c054eebdbf..c1ab8004fe0b 100644 --- a/src/Interpreters/InsertDependenciesBuilder.cpp +++ b/src/Interpreters/InsertDependenciesBuilder.cpp @@ -135,6 +135,7 @@ namespace ErrorCodes { extern const int UNKNOWN_TABLE; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; extern const int TOO_DEEP_RECURSION; } @@ -1744,6 +1745,26 @@ Chain InsertDependenciesBuilder::createRetry(const std::vector starting from {}", fmt::join(path, "/"), partition, start_from); + /// Behind a table with the `Alias` engine the deduplication info travels into a nested insert + /// chain, and its visited views belong to the outer chain's builder, so this builder cannot + /// rebuild them. A foreign element can appear anywhere in the path: at its end (a direct + /// insert into the source table), at its start (a direct insert into a materialized view + /// keeps the view as `start_from`), or in the middle (a regular-table root keeps an empty + /// `start_from`, which every builder "owns" because `inner_tables` always contains the + /// empty root, while the intermediate views of the outer chain are still foreign — and + /// skipping them would silently drop their transformations from the retried rows). Require + /// every element to be owned by this builder and refuse loudly otherwise, instead of + /// failing with a bare `std::out_of_range` or losing rows below. + auto foreign = std::find_if(path.begin(), path.end(), [this](const auto & id) { return !isView(id); }); + if (foreign != path.end() || !isView(start_from)) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Cannot rebuild the deduplication retry chain for '{}': it does not belong to this insert chain. " + "This happens when deduplicated rows have to be recalculated after a table with the `Alias` engine. " + "Retry path: {}", + foreign != path.end() ? *foreign : start_from, + fmt::join(path, "/")); + Chain result; auto it = std::find(path.begin(), path.end(), start_from); @@ -1763,10 +1784,6 @@ Chain InsertDependenciesBuilder::createRetry(const std::vectorfilterToPartition(partition_selector, part_index); + auto current_deduplication_info = deduplication_info->filterToPartition(partition_selector, part_index, deduplicate); { ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 5923efe68540..ef8844c8297c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -340,7 +340,7 @@ void ReplicatedMergeTreeSink::consume(Chunk & chunk) /// Keep only the tokens whose own rows landed in this partition, so a coalesced async /// insert does not register a token in partitions it never wrote to. - auto current_deduplication_info = deduplication_info->filterToPartition(partition_selector, part_index); + auto current_deduplication_info = deduplication_info->filterToPartition(partition_selector, part_index, deduplicate); { ProfileEventTimeIncrement duplication_elapsed(ProfileEvents::DuplicationElapsedMicroseconds); diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index 3cb8bbce7fc2..969989d58bea 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -126,6 +127,16 @@ class AliasSink final : public SinkToStorage, WithContext Chunk non_materialized_chunk(non_materialized_block.getColumns(), non_materialized_block.rows()); non_materialized_chunk.setChunkInfos(chunk.getChunkInfos().clone()); + + /// The nested INSERT re-anchors the deduplication info to its own chunks (its squashing and + /// `AddDeduplicationInfoTransform` call `updateOriginalBlock`). When this sink is fed by a + /// dependent materialized view whose inner query changed the number of rows, those chunks + /// no longer match the rows the info's offsets describe, and computing a data hash after + /// that re-anchoring would read out of the block's bounds. Cache the hashes now, while the + /// info is still consistent. + if (auto deduplication_info = non_materialized_chunk.getChunkInfos().get()) + deduplication_info->cacheDataHashes(data_hash_cache); + executor->push(std::move(non_materialized_chunk)); } @@ -153,6 +164,9 @@ class AliasSink final : public SinkToStorage, WithContext bool async_insert; BlockIO block_io; std::unique_ptr executor; + /// Memoizes the deduplication data hashes across the sibling chunks of one source block, so a + /// row-count-changing view fanned out into many chunks does not re-hash the source per chunk. + DeduplicationInfo::DataHashCache data_hash_cache; }; void StorageAlias::read( diff --git a/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.reference b/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.reference new file mode 100644 index 000000000000..e162cdfaabe1 --- /dev/null +++ b/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.reference @@ -0,0 +1,9 @@ +400 100 100 +800 200 100 +400 100 100 +800 200 100 +400 100 100 +800 200 100 +800 200 200 +1 +200 diff --git a/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.sh b/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.sh new file mode 100755 index 000000000000..a616772bc559 --- /dev/null +++ b/tests/queries/0_stateless/04613_deduplication_alias_hop_row_drift.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# These scenarios pin the deduplication path exactly: the source and view-level squash thresholds +# and the insert/thread counts decide how many deduplication tokens an insert carries and how they +# merge behind the alias hop. Settings randomization perturbs that (e.g. a different squash +# threshold changes the token identity and with it the deduplication outcome), so it is disabled. +# Regression test: a dependent materialized view with a row-count-changing inner query (GROUP BY) +# targeting an Alias, with a deduplicating table behind the alias hop. The AliasSink runs a nested +# INSERT whose squashing and AddDeduplicationInfoTransform re-anchor the DeduplicationInfo's +# original block to the view-output chunks, which no longer match the source rows its offsets +# describe. Computing the deduplication data hash after that re-anchoring read out of the block's +# bounds: an abort on 'block.rows() == getRows()' in debug/sanitizer builds, a garbage hash (broken +# deduplication behind the alias) in release builds. The hashes must be cached at the alias hop, +# while the info is still consistent, so repeated identical inserts deduplicate deterministically. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# max_threads/max_insert_threads are pinned: with parallel processing the interleaving of the +# view-output chunks decides how the tokens merge behind the alias hop, so the deduplication token +# identity - and with it the deduplication outcome at dst - would depend on thread scheduling. +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS drift_dst" + +# Only dst, behind the alias hop, deduplicates. mv1's GROUP BY makes the view output 100 rows from +# the 400-row source block, so the nested INSERT the AliasSink runs sees chunks whose row count +# differs from the rows the restored deduplication info describes. +$CLICKHOUSE_CLIENT -q "CREATE TABLE drift_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE drift_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE drift_dst (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE TABLE drift_alias ENGINE = Alias('drift_inner')" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW drift_mv1 TO drift_alias AS SELECT x FROM drift_src GROUP BY x" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW drift_mv2 TO drift_dst AS SELECT x FROM drift_inner" + +# A data-fed insert (deduplication is not active for INSERT SELECT). 400 rows, 100 distinct. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +# The same insert again: src and inner do not deduplicate and double, while dst must deduplicate +# the repeated block - its deduplication hash is computed from the consistent source block, not +# from whatever the drifted original block points at. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_dst" + +# The same scenario, but the view emits its output in multiple chunks: with a small max_block_size +# the GROUP BY produces 10-row blocks, and a small min_insert_block_size_rows makes the view-level +# squashing pass each of them through. Each chunk carries a clone of the same source-level +# deduplication info, distinguished only by consecutive view-block numbers. The nested INSERT +# behind the alias hop squashes with the target-side threshold - raised back to the default by +# min_insert_block_size_rows_for_materialized_views - so it merges the stamped chunks, extending +# the view-block range of the token. The cached data hashes must survive that merge, or the hash +# is recomputed from the re-anchored (drifted) block: without the fix in +# DeduplicationInfo::TokenDefinition::doExtend the second insert is not deduplicated at dst. +# max_threads=1 keeps the chunk order, and thus the merged token identity, deterministic. +MULTICHUNK_SETTINGS="$SETTINGS --async_insert=0 --max_block_size=10 --max_threads=1 --min_insert_block_size_rows=10 --min_insert_block_size_rows_for_materialized_views=1000000" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $MULTICHUNK_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $MULTICHUNK_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_dst" + +# The same scenario with async inserts: the deduplication info is flagged as an async insert, so a +# collision at the deduplicating table behind the alias hop never takes the single-token fast path +# and used to walk the source-row offsets over the drifted view-output block. With the fix a +# repeated identical flush is deduplicated as a whole, without row-level slicing. +ASYNC_SETTINGS="$SETTINGS --async_insert=1 --wait_for_async_insert=1 --async_insert_deduplicate=1" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $ASYNC_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $ASYNC_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE drift_dst" + +# Two deduplication tokens in one sync insert: the source-side squashing +# (min_insert_block_size_rows, with max_insert_block_size as the parser cap) re-blocks the 800 +# input rows into two 400-row source blocks, each carrying its own token (distinct source numbers +# keep the tokens from merging), and the nested INSERT behind the alias hop squashes them into one block - +# min_insert_block_size_rows_for_materialized_views raises the target-side threshold - so a +# two-token deduplication info rides a drifted view-output block. The first insert carries no +# duplicates and registers both tokens. The second insert repeats the first source block next to a +# fresh one: exactly one of the two tokens collides with the deduplication window, and slicing the +# collided token's rows out of the block is impossible after mv1's GROUP BY collapsed it, so the +# insert is rejected with NOT_IMPLEMENTED - pre-fix this walked the source-row offsets over the +# smaller view-output block: an abort in debug builds, an out-of-bounds read in release builds. +# dst keeps the rows of the first insert only. +# grep -m1 -c prints exactly one count: the server also echoes the exception through +# send_logs_level, so the raw number of matching lines is not stable. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM drift_src), (SELECT count() FROM drift_inner), (SELECT count() FROM drift_dst)" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 201 300; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO drift_src FORMAT TSV" 2>&1 | grep -m1 -c "NOT_IMPLEMENTED" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM drift_dst" + +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE drift_src" diff --git a/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.reference b/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.reference new file mode 100644 index 000000000000..7a9ff6709e50 --- /dev/null +++ b/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.reference @@ -0,0 +1,6 @@ +400 100 100 2 +800 200 100 2 +400 100 100 2 +800 200 100 2 +1 +0 diff --git a/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.sh b/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.sh new file mode 100755 index 000000000000..08a636e6efcb --- /dev/null +++ b/tests/queries/0_stateless/04621_deduplication_alias_hop_partitioned_target.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# The scenarios pin the deduplication path exactly (the squash thresholds, the insert/thread +# counts), so settings randomization is disabled, as in 04613_deduplication_alias_hop_row_drift. +# Companion of 04613_deduplication_alias_hop_row_drift with a PARTITIONED deduplicating target +# behind the alias hop. Partitioning adds a path the unpartitioned test cannot reach: the sink +# splits the block by partition and DeduplicationInfo::filterToPartition attributes each token's +# source-row range to the partitions via the scatter selector. After mv1's row-count-changing +# GROUP BY re-anchored the info to the view-output chunks, the selector describes the smaller +# view-output block while the token ranges still describe the source rows, so the walk read out of +# the selector's bounds. filterToPartition must refuse such an insert with NOT_IMPLEMENTED. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS part_dst" + +# Only dst, behind the alias hop, deduplicates - and it is partitioned, so the sink splits every +# insert by partition before deduplicating. +$CLICKHOUSE_CLIENT -q "CREATE TABLE part_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE part_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE part_dst (x UInt64) ENGINE = MergeTree PARTITION BY x % 2 ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE TABLE part_alias ENGINE = Alias('part_inner')" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW part_mv1 TO part_alias AS SELECT x FROM part_src GROUP BY x" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW part_mv2 TO part_dst AS SELECT x FROM part_inner" + +# A single data-fed insert carries one deduplication token, so the partition split keeps the whole +# info for every partition and only the cached data hash is used: the repeated insert must +# deduplicate in both partitions of dst. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_src), (SELECT count() FROM part_inner), (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_src), (SELECT count() FROM part_inner), (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_dst" + +# The same with async inserts: each flush carries one token, so the partition split stays on the +# single-token fast path and the repeated flush is deduplicated as a whole in both partitions. +ASYNC_SETTINGS="$SETTINGS --async_insert=1 --wait_for_async_insert=1 --async_insert_deduplicate=1" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $ASYNC_SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_src), (SELECT count() FROM part_inner), (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $ASYNC_SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_src), (SELECT count() FROM part_inner), (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE part_dst" + +# Two deduplication tokens in one sync insert: the source-side squashing +# (min_insert_block_size_rows, with max_insert_block_size as the parser cap) re-blocks the 800 +# input rows into two 400-row source blocks, each carrying its own token, and the nested INSERT +# behind the alias hop squashes them into one block. The partitioned sink must then attribute each token's +# source rows to the partitions - impossible after mv1's GROUP BY collapsed the blocks - so the +# insert is rejected with NOT_IMPLEMENTED by filterToPartition (pre-fix this walked the +# source-row ranges over the smaller partition selector: an out-of-bounds read). Nothing reaches +# dst. +# grep -m1 -c prints exactly one count: the server also echoes the exception through +# send_logs_level, so the raw number of matching lines is not stable. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO part_src FORMAT TSV" 2>&1 | grep -m1 -c "NOT_IMPLEMENTED" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM part_dst" + +$CLICKHOUSE_CLIENT -q "DROP TABLE part_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE part_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE part_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE part_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE part_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE part_src" diff --git a/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.reference b/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.reference new file mode 100644 index 000000000000..238383a88802 --- /dev/null +++ b/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.reference @@ -0,0 +1,5 @@ +400 100 +800 100 +800 800 +1 +800 diff --git a/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.sh b/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.sh new file mode 100755 index 000000000000..346950ff4ea4 --- /dev/null +++ b/tests/queries/0_stateless/04635_deduplication_alias_hop_deduplicating_inner.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# These scenarios pin the deduplication path exactly: the squash thresholds and the insert/thread +# counts decide how many deduplication tokens an insert carries behind the alias hop, and a +# randomized deduplication window on the inner table would change the outcome. +# Regression test: the table behind the `Alias` engine deduplicates ITSELF (it is the direct +# target of the nested INSERT the AliasSink runs, not a table behind another materialized view). +# The deduplication info restored onto the nested chain keeps the visited views of the OUTER +# insert chain, which the nested chain's InsertDependenciesBuilder does not know about. When a +# repeated insert collided at the inner table, the deduplication retry looked those views up in +# the nested builder's maps: a bare `std::out_of_range` from `unordered_map::at` in +# `InsertDependenciesBuilder::createRetry`, reported as a logical error (an abort in sanitizer +# builds). Fixed twice over: a fully-collided (empty) retry does not build the retry chain at all, +# and a partial retry that would need the outer chain's views is rejected with a clean +# NOT_IMPLEMENTED. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_inner" + +# The inner table behind the alias deduplicates itself. mv1's GROUP BY makes the view output 100 +# rows from the 400-row source block, so the deduplication info behind the hop is re-anchored to +# a block that no longer matches the rows its offsets describe. +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_alias ENGINE = Alias('hop_inner')" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv1 TO hop_alias AS SELECT x FROM hop_src GROUP BY x" + +# A data-fed insert (deduplication is not active for INSERT SELECT). 400 rows, 100 distinct, +# a single deduplication token. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_src), (SELECT count() FROM hop_inner)" + +# The same insert again: the whole block collides at the inner table. All tokens are filtered, +# nothing is left to retry, so the retry chain - which the nested chain's builder could not even +# construct - must not be built; the repeated block is deduplicated cleanly. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_src), (SELECT count() FROM hop_inner)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_inner" + +# A partial collision that DOES need the retry chain: mv1 is replaced with a row-preserving view, +# so the deduplication info is not drifted and slicing the collided token's rows out of the block +# succeeds, but recalculating the view output for the surviving rows would need the outer chain's +# views, which the nested chain's builder does not know. The insert is rejected with a clean +# NOT_IMPLEMENTED instead of a bare std::out_of_range. The source-side squashing +# (min_insert_block_size_rows) re-blocks the 800 input rows into two 400-row source blocks - two +# tokens - and min_insert_block_size_rows_for_materialized_views makes the nested squash merge +# them into one block behind the hop. +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv1" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv1 TO hop_alias AS SELECT x FROM hop_src" + +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_src), (SELECT count() FROM hop_inner)" + +# The first source block repeats next to a fresh one: exactly one of the two tokens collides. +# grep -m1 -c prints exactly one count: the server also echoes the exception through +# send_logs_level, so the raw number of matching lines is not stable. +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 201 300; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_src FORMAT TSV" 2>&1 | grep -m1 -c "NOT_IMPLEMENTED" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM hop_inner" + +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_src" diff --git a/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.reference b/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.reference new file mode 100644 index 000000000000..4db9d675b61a --- /dev/null +++ b/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.reference @@ -0,0 +1,5 @@ +400 400 +800 400 +800 800 +1 +800 0 diff --git a/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.sh b/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.sh new file mode 100755 index 000000000000..e3e8d4797541 --- /dev/null +++ b/tests/queries/0_stateless/04638_deduplication_alias_hop_insert_into_mv.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# These scenarios pin the deduplication path exactly: the squash thresholds decide how many +# deduplication tokens an insert carries behind the alias hop, and a randomized deduplication +# window on the destination table would change the outcome. +# Regression test: unlike 04635, the insert goes DIRECTLY into a materialized view, so the outer +# insert chain keeps a non-empty root view. The deduplicating table sits behind a dependent +# materialized view of the alias inner table, so the END of the retry path does belong to the +# nested insert chain the AliasSink runs - the guard on the last path element passes - but the +# view the deduplication info anchors its original block at (`start_from` of the retry) belongs +# to the outer chain's InsertDependenciesBuilder. Rebuilding a retry from a view the builder does +# not own is unsound: path elements the builder does not know are silently skipped (dropping +# their transformations on a longer chain), and when the foreign anchor is the first path +# element, `createPreSink` fails with a bare `std::out_of_range` from `unordered_map::at`, +# reported as a logical error (an abort in sanitizer builds). Such retries are now rejected with +# a clean NOT_IMPLEMENTED, and a fully-collided insert is deduplicated cleanly without building +# the retry chain at all. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv0" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_feed" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_inner" + +# The insert goes into hop_mv0, keeping the outer chain's root view non-empty. The rows land in +# hop_src, flow through hop_mv1 into the alias, and behind the hop through hop_mv2 into the +# deduplicating hop_dst. +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_feed (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_dst (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_alias ENGINE = Alias('hop_inner')" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv0 TO hop_src AS SELECT x FROM hop_feed" +# hop_mv1 is row-preserving but NOT an identity: a retry chain that skipped it (because its +# builder does not own the view) would push unshifted rows to the destination - visible data +# corruption instead of a crash. +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv1 TO hop_alias AS SELECT x + 1000000 AS x FROM hop_src" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv2 TO hop_dst AS SELECT x FROM hop_inner" + +# A single-token data-fed insert directly into the materialized view (deduplication is not active +# for INSERT SELECT). +seq 1 400 | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_mv0 FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +# The same insert again: the whole block collides at the destination. Nothing is left to retry, +# so the retry chain - which the nested chain's builder could not even construct - must not be +# built; the repeated block is deduplicated cleanly. +seq 1 400 | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_mv0 FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_dst" + +# A partial collision that DOES need the retry chain. The source-side squashing +# (min_insert_block_size_rows) re-blocks the 800 input rows into two 400-row source blocks - two +# tokens - and min_insert_block_size_rows_for_materialized_views makes the nested squash behind +# the alias hop merge them into one block, so the destination sees one block with two tokens. The +# first source block repeats next to a fresh one: exactly one of the two tokens collides, and +# recalculating the surviving rows would start the retry chain from the outer chain's root view, +# which the nested chain's builder does not know. The insert is rejected with a clean +# NOT_IMPLEMENTED instead of a bare std::out_of_range. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ seq 1 400; seq 401 800; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_mv0 FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +# grep -m1 -c prints exactly one count: the server also echoes the exception through +# send_logs_level, so the raw number of matching lines is not stable. +{ seq 1 400; seq 801 1200; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_mv0 FORMAT TSV" 2>&1 | grep -m1 -c "NOT_IMPLEMENTED" +# The second count is the number of rows that skipped hop_mv1's shift - rows a wrongly-built +# retry chain would have pushed to the destination. +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_dst), (SELECT count() FROM hop_dst WHERE x <= 1000000)" + +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv0" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_feed" diff --git a/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.reference b/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.reference new file mode 100644 index 000000000000..4db9d675b61a --- /dev/null +++ b/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.reference @@ -0,0 +1,5 @@ +400 400 +800 400 +800 800 +1 +800 0 diff --git a/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.sh b/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.sh new file mode 100755 index 000000000000..acb7fd592557 --- /dev/null +++ b/tests/queries/0_stateless/04639_deduplication_alias_hop_regular_table_root.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# These scenarios pin the deduplication path exactly: the squash thresholds decide how many +# deduplication tokens an insert carries behind the alias hop, and a randomized deduplication +# window on the destination table would change the outcome. +# Regression test: unlike 04638, the insert goes into a REGULAR table, so the outer insert +# chain's root view is empty. Every InsertDependenciesBuilder stores the empty root in its +# `inner_tables`, so an ownership check on the retry's anchor alone accepts a foreign empty +# anchor - and the retry loop then silently skipped the outer chain's intermediate views +# (dropping their transformations), after which the rebuilt rows no longer matched the +# surviving tokens and were silently LOST. The retry path must be rejected with a clean +# NOT_IMPLEMENTED whenever any of its elements is not owned by the builder rebuilding it. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS hop_inner" + +# The insert goes into the regular table hop_src (empty root view in the outer chain). The rows +# flow through hop_mv1 into the alias, and behind the hop through hop_mv2 into the +# deduplicating hop_dst. +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_dst (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE TABLE hop_alias ENGINE = Alias('hop_inner')" +# hop_mv1 is row-preserving but NOT an identity: a retry chain that skipped it (because its +# builder does not own the view) would push unshifted rows to the destination - visible data +# corruption instead of a crash. +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv1 TO hop_alias AS SELECT x + 1000000 AS x FROM hop_src" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW hop_mv2 TO hop_dst AS SELECT x FROM hop_inner" + +# A single-token data-fed insert (deduplication is not active for INSERT SELECT). +seq 1 400 | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +# The same insert again: the whole block collides at the destination. Nothing is left to retry, +# so the retry chain must not be built; the repeated block is deduplicated cleanly. +seq 1 400 | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_inner" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE hop_dst" + +# A partial collision that DOES need the retry chain. The source-side squashing +# (min_insert_block_size_rows) re-blocks the 800 input rows into two 400-row source blocks - two +# tokens - and min_insert_block_size_rows_for_materialized_views makes the nested squash behind +# the alias hop merge them into one block, so the destination sees one block with two tokens. The +# first source block repeats next to a fresh one: exactly one of the two tokens collides, and the +# retry path contains hop_mv1, which the nested chain's builder does not own. The insert is +# rejected with a clean NOT_IMPLEMENTED instead of silently losing the surviving rows. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ seq 1 400; seq 401 800; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_inner), (SELECT count() FROM hop_dst)" + +# grep -m1 -c prints exactly one count: the server also echoes the exception through +# send_logs_level, so the raw number of matching lines is not stable. +{ seq 1 400; seq 801 1200; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO hop_src FORMAT TSV" 2>&1 | grep -m1 -c "NOT_IMPLEMENTED" +# The second count is the number of rows that skipped hop_mv1's shift - rows a wrongly-built +# retry chain would have pushed to the destination; the first would show silently LOST rows. +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM hop_dst), (SELECT count() FROM hop_dst WHERE x <= 1000000)" + +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE hop_src" diff --git a/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.reference b/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.reference new file mode 100644 index 000000000000..d114d1131b5e --- /dev/null +++ b/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.reference @@ -0,0 +1,2 @@ +800 200 200 2 +1600 400 400 2 diff --git a/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.sh b/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.sh new file mode 100755 index 000000000000..e96382bed87a --- /dev/null +++ b/tests/queries/0_stateless/04645_deduplication_alias_hop_partitioned_no_dedup.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# The scenario pins the deduplication path exactly (the squash thresholds, the insert/thread +# counts), so settings randomization is disabled, as in 04613_deduplication_alias_hop_row_drift. +# Companion of 04621_deduplication_alias_hop_partitioned_target with the deduplication window of +# the partitioned target set to 0: the sink does not deduplicate, so the tokens are never +# registered and DeduplicationInfo::filterToPartition has nothing to attribute. The multi-token +# insert whose deduplication info drifted behind the alias hop (mv1's row-count-changing GROUP BY +# re-anchored it to the view-output chunks) must succeed instead of being rejected with +# NOT_IMPLEMENTED by the consistency check of filterToPartition. +# See https://github.com/ClickHouse/ClickHouse/issues/111100 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_src" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nodedup_dst" + +# No table in the chain deduplicates - dst is partitioned, so the sink still splits every insert +# by partition and calls filterToPartition, but with deduplication disabled it must pass the +# tokens through untouched. +$CLICKHOUSE_CLIENT -q "CREATE TABLE nodedup_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nodedup_inner (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nodedup_dst (x UInt64) ENGINE = MergeTree PARTITION BY x % 2 ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nodedup_alias ENGINE = Alias('nodedup_inner')" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW nodedup_mv1 TO nodedup_alias AS SELECT x FROM nodedup_src GROUP BY x" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW nodedup_mv2 TO nodedup_dst AS SELECT x FROM nodedup_inner" + +# Two deduplication tokens in one sync insert, exactly as in the rejection scenario of 04621: the +# source-side squashing re-blocks the 800 input rows into two 400-row source blocks, each carrying +# its own token, and the nested INSERT behind the alias hop squashes them into one drifted block. +# Since dst does not deduplicate, the insert must succeed and fill both partitions. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO nodedup_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM nodedup_src), (SELECT count() FROM nodedup_inner), (SELECT count() FROM nodedup_dst), (SELECT count(DISTINCT _partition_id) FROM nodedup_dst)" + +# The repeated insert is not deduplicated anywhere: every count doubles. +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO nodedup_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM nodedup_src), (SELECT count() FROM nodedup_inner), (SELECT count() FROM nodedup_dst), (SELECT count(DISTINCT _partition_id) FROM nodedup_dst)" + +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_mv2" +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_mv1" +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_alias" +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_inner" +$CLICKHOUSE_CLIENT -q "DROP TABLE nodedup_src" From 35339621613d731adeb0ad738be20b24906dd18d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 12:06:36 +0000 Subject: [PATCH 69/86] Backport #112490 to 26.6: Fix `ATTEMPT_TO_READ_AFTER_EOF` on merge of text index with an empty part --- src/Storages/MergeTree/MergeTask.cpp | 4 ++ ...text_index_merge_with_empty_part.reference | 8 +++ ...04654_text_index_merge_with_empty_part.sql | 51 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 tests/queries/0_stateless/04654_text_index_merge_with_empty_part.reference create mode 100644 tests/queries/0_stateless/04654_text_index_merge_with_empty_part.sql diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7307c448cd25..06ee2610c974 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -2275,6 +2275,10 @@ bool MergeTask::MergeTextIndexStage::prepare() const { const auto & part = global_ctx->future_part->parts[part_idx]; + /// An empty part contributes nothing to the merged index and its files are empty. + if (part->rows_count == 0) + continue; + if (index_ptr->getDeserializedFormat(part->checksums, index_ptr->getFileName(), &part->getDataPartStorage())) { /// If text index exists in the source part, take it as is. diff --git a/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.reference b/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.reference new file mode 100644 index 000000000000..f3ed02ba6f8d --- /dev/null +++ b/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.reference @@ -0,0 +1,8 @@ +parts before optimize +1 +0 +parts after optimize +1 +1 +0 +0 diff --git a/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.sql b/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.sql new file mode 100644 index 000000000000..0dbe0dad6e91 --- /dev/null +++ b/tests/queries/0_stateless/04654_text_index_merge_with_empty_part.sql @@ -0,0 +1,51 @@ +-- Regression test: merging a text index with an empty source part threw `ATTEMPT_TO_READ_AFTER_EOF`. +-- A mutation that deletes all rows of a part leaves the text index files empty (no granules are +-- serialized, so not even the header is written) but still listed in the part's checksums, and +-- `MergeTextIndexesTask` tried to read the header of such a file. + +DROP TABLE IF EXISTS t_text_index_merge_with_empty_part; + +CREATE TABLE t_text_index_merge_with_empty_part +( + ts DateTime, + body String, + INDEX idx lower(body) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY ts +SETTINGS + remove_empty_parts = 0, -- keep the empty part until OPTIMIZE + max_bytes_to_merge_at_max_space_in_pool = 1; -- no background merges before OPTIMIZE + +INSERT INTO t_text_index_merge_with_empty_part VALUES ('2026-01-01 00:00:00', 'keeper row timeout'); +INSERT INTO t_text_index_merge_with_empty_part VALUES ('2026-01-01 00:00:01', 'doomed row'); + +-- Turn the second part into an empty part that still carries the (empty) text index files. +ALTER TABLE t_text_index_merge_with_empty_part DELETE WHERE body = 'doomed row' SETTINGS mutations_sync = 2; + +SELECT 'parts before optimize'; +SELECT rows FROM system.parts +WHERE database = currentDatabase() AND table = 't_text_index_merge_with_empty_part' AND active +ORDER BY name; + +OPTIMIZE TABLE t_text_index_merge_with_empty_part FINAL; + +SELECT 'parts after optimize'; +SELECT rows FROM system.parts +WHERE database = currentDatabase() AND table = 't_text_index_merge_with_empty_part' AND active +ORDER BY name; + +SELECT count() FROM t_text_index_merge_with_empty_part +WHERE hasToken(lower(body), 'keeper') +SETTINGS force_data_skipping_indices = 'idx'; + +SELECT count() FROM t_text_index_merge_with_empty_part +WHERE hasToken(lower(body), 'doomed') +SETTINGS force_data_skipping_indices = 'idx'; + +-- A merge where every source part is empty. +ALTER TABLE t_text_index_merge_with_empty_part DELETE WHERE 1 SETTINGS mutations_sync = 2; +OPTIMIZE TABLE t_text_index_merge_with_empty_part FINAL; +SELECT count() FROM t_text_index_merge_with_empty_part; + +DROP TABLE t_text_index_merge_with_empty_part; From 63af5c39780f886ca82d204554179b8f6890187c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 16:45:44 +0000 Subject: [PATCH 70/86] Backport #107251 to 26.6: Feature: Support additional storage classes in S3 --- .../engines/table-engines/integrations/s3.md | 2 +- docs/en/sql-reference/table-functions/s3.md | 2 +- src/IO/S3RequestSettings.cpp | 4 ++-- .../registerStorageObjectStorage.cpp | 2 +- ...3000_s3_storage_class_validation.reference | 10 +++++++++ .../03000_s3_storage_class_validation.sql | 21 +++++++++++++++++++ 6 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/03000_s3_storage_class_validation.reference create mode 100644 tests/queries/0_stateless/03000_s3_storage_class_validation.sql diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index d54b85697864..8cfa19df394d 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -47,7 +47,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) - `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension. - `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). - `partition_columns_in_data_file` - Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. -- `storage_class_name` - Options: `STANDARD` or `INTELLIGENT_TIERING`, allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). +- `storage_class_name` - Options: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER_IR`, `EXPRESS_ONEZONE`. Only S3 storage classes that allow immediate retrieval are supported (archival classes such as `GLACIER` and `DEEP_ARCHIVE` are not). Allows to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). - `extra_credentials` - Optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. ### Data cache {#data-cache} diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index c0f052ee2c42..9d4800f285c3 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -51,7 +51,7 @@ For GCS, substitute your HMAC key and HMAC secret where you see `access_key_id` | `partition_strategy` | Parameter is optional. Supported values: `wildcard` or `hive`. `wildcard` requires a `{_partition_id}` in the path, which is replaced with the partition key. `hive` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `wildcard` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`wildcard` under `compatibility` settings older than `26.6`, `hive` otherwise). | | `partition_columns_in_data_file` | Parameter is optional. Only used with `hive` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. | | `extra_credentials` | Parameter is optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. | -| `storage_class_name` | Parameter is optional. Supported values: `STANDARD` or `INTELLIGENT_TIERING`. Allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). Defaults to `STANDARD`. | +| `storage_class_name` | Parameter is optional. Supported values: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER_IR`, `EXPRESS_ONEZONE`. Only S3 storage classes that allow immediate retrieval are supported (archival classes such as `GLACIER` and `DEEP_ARCHIVE` are not). Allows to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). Defaults to `STANDARD`. | :::note GCS The GCS url is in this format as the endpoint for the Google XML API is different than the JSON API: diff --git a/src/IO/S3RequestSettings.cpp b/src/IO/S3RequestSettings.cpp index 287befe014b4..1c362f39cb6f 100644 --- a/src/IO/S3RequestSettings.cpp +++ b/src/IO/S3RequestSettings.cpp @@ -236,11 +236,11 @@ void S3RequestSettings::validateUploadSettings() (*this)[S3RequestSetting::upload_part_size_multiply_factor].value, ReadableSize((*this)[S3RequestSetting::max_upload_part_size].value)); } - NameSet storage_class_names {"STANDARD", "INTELLIGENT_TIERING"}; + NameSet storage_class_names {"STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER_IR", "EXPRESS_ONEZONE"}; if (!(*this)[S3RequestSetting::storage_class_name].value.empty() && !storage_class_names.contains((*this)[S3RequestSetting::storage_class_name])) throw Exception( ErrorCodes::INVALID_SETTING_VALUE, - "Setting storage_class has invalid value {} which only supports STANDARD and INTELLIGENT_TIERING", + "Setting storage_class has invalid value {}: this storage class is not supported for ClickHouse S3 disks", (*this)[S3RequestSetting::storage_class_name].value); /// TODO: it's possible to set too small limits. diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index d90a89743117..cef5d1559856 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -304,7 +304,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) - `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension. - `partition_strategy` – Options: `WILDCARD` or `HIVE`. `WILDCARD` requires a `{_partition_id}` in the path, which is replaced with the partition key. `HIVE` does not allow wildcards, assumes the path is the table root, and generates Hive-style partitioned directories with Snowflake IDs as filenames and the file format as the extension. If the path contains a `{_partition_id}` placeholder, defaults to `WILDCARD` — the only strategy compatible with such a path. Otherwise defaults to the `file_like_engine_default_partition_strategy` setting (`WILDCARD` under `compatibility` settings older than `26.6`, `HIVE` otherwise). - `partition_columns_in_data_file` - Only used with `HIVE` partition strategy. Tells ClickHouse whether to expect partition columns to be written in the data file. Defaults `false`. -- `storage_class_name` - Options: `STANDARD` or `INTELLIGENT_TIERING`, allow to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). +- `storage_class_name` - Options: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER_IR`, `EXPRESS_ONEZONE`. Only S3 storage classes that allow immediate retrieval are supported (archival classes such as `GLACIER` and `DEEP_ARCHIVE` are not). Allows to specify [AWS S3 Intelligent Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). - `extra_credentials` - Optional. Used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. ### Data cache {#data-cache} diff --git a/tests/queries/0_stateless/03000_s3_storage_class_validation.reference b/tests/queries/0_stateless/03000_s3_storage_class_validation.reference new file mode 100644 index 000000000000..82ef2ccfcd82 --- /dev/null +++ b/tests/queries/0_stateless/03000_s3_storage_class_validation.reference @@ -0,0 +1,10 @@ +Testing invalid S3 storage classes +Testing valid S3 storage classes +4 +4 +4 +4 +4 +4 +4 +Testing rejected archival S3 storage classes diff --git a/tests/queries/0_stateless/03000_s3_storage_class_validation.sql b/tests/queries/0_stateless/03000_s3_storage_class_validation.sql new file mode 100644 index 000000000000..df2cde8788cf --- /dev/null +++ b/tests/queries/0_stateless/03000_s3_storage_class_validation.sql @@ -0,0 +1,21 @@ +-- Tags: no-fasttest +SELECT 'Testing invalid S3 storage classes'; +SELECT * FROM s3('http://localhost:11111/test/bucket', 'CSV', 'x String', storage_class_name='FSX_OPENZFS'); -- { serverError INVALID_SETTING_VALUE } +SELECT * FROM s3('http://localhost:11111/test/bucket', 'CSV', 'x String', storage_class_name='INVALID_CLASS'); -- { serverError INVALID_SETTING_VALUE } + +-- The newly accepted classes must pass validation. If the allow-list were reverted to the old +-- {STANDARD, INTELLIGENT_TIERING} set, these queries would throw INVALID_SETTING_VALUE instead of +-- reading the data, so this gives positive regression coverage without depending on an S3 upload. +SELECT 'Testing valid S3 storage classes'; +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='STANDARD'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='REDUCED_REDUNDANCY'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='STANDARD_IA'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='INTELLIGENT_TIERING'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='ONEZONE_IA'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='GLACIER_IR'); +SELECT count() FROM s3('http://localhost:11111/test/a.tsv', 'TSV', 'a UInt8, b UInt8, c UInt8', storage_class_name='EXPRESS_ONEZONE'); + +-- Archival classes require an asynchronous restore before an object can be read, so they remain rejected. +SELECT 'Testing rejected archival S3 storage classes'; +SELECT * FROM s3('http://localhost:11111/test/bucket', 'CSV', 'x String', storage_class_name='GLACIER'); -- { serverError INVALID_SETTING_VALUE } +SELECT * FROM s3('http://localhost:11111/test/bucket', 'CSV', 'x String', storage_class_name='DEEP_ARCHIVE'); -- { serverError INVALID_SETTING_VALUE } From 0e4136a3383dc83ff5588385262343bd60fd7bef Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 20:03:32 +0000 Subject: [PATCH 71/86] Backport #109861 to 26.6: Don't copy all file infos when writing backup metadata --- src/Backups/BackupCoordinationFileInfos.cpp | 7 ++--- src/Backups/BackupCoordinationFileInfos.h | 5 ++-- src/Backups/BackupCoordinationLocal.cpp | 4 +-- src/Backups/BackupCoordinationLocal.h | 2 +- src/Backups/BackupCoordinationOnCluster.cpp | 6 ++--- src/Backups/BackupCoordinationOnCluster.h | 2 +- src/Backups/BackupImpl.cpp | 29 ++++++++++++--------- src/Backups/IBackupCoordination.h | 8 +++++- 8 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index 086bd3c50b84..25c36a260517 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -33,14 +33,11 @@ BackupFileInfos BackupCoordinationFileInfos::getFileInfos(const String & host_id return it->second; } -BackupFileInfos BackupCoordinationFileInfos::getFileInfosForAllHosts() const +void BackupCoordinationFileInfos::forEachFileInfoForAllHosts(const std::function & callback) const { prepare(); - BackupFileInfos res; - res.reserve(file_infos_for_all_hosts.size()); for (const auto * file_info : file_infos_for_all_hosts) - res.emplace_back(*file_info); - return res; + callback(*file_info); } BackupFileInfo BackupCoordinationFileInfos::getFileInfoByDataFileIndex(size_t data_file_index) const diff --git a/src/Backups/BackupCoordinationFileInfos.h b/src/Backups/BackupCoordinationFileInfos.h index 49aebe57a17b..99862794f283 100644 --- a/src/Backups/BackupCoordinationFileInfos.h +++ b/src/Backups/BackupCoordinationFileInfos.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,8 @@ class BackupCoordinationFileInfos /// Returns file infos for the specified host after preparation. BackupFileInfos getFileInfos(const String & host_id) const; - /// Returns file infos for all hosts after preparation. - BackupFileInfos getFileInfosForAllHosts() const; + /// Iterates the file infos of all hosts in place, without copying them into a vector. + void forEachFileInfoForAllHosts(const std::function & callback) const; /// Returns a file info by data file index (see BackupFileInfo::data_file_index). BackupFileInfo getFileInfoByDataFileIndex(size_t data_file_index) const; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 4eb7cbff30f6..87a08a42fd64 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -116,10 +116,10 @@ BackupFileInfos BackupCoordinationLocal::getFileInfos() const return file_infos.getFileInfos(""); } -BackupFileInfos BackupCoordinationLocal::getFileInfosForAllHosts() const +void BackupCoordinationLocal::forEachFileInfoForAllHosts(const std::function & callback) const { std::lock_guard lock{file_infos_mutex}; - return file_infos.getFileInfosForAllHosts(); + file_infos.forEachFileInfoForAllHosts(callback); } bool BackupCoordinationLocal::startWritingFile(size_t data_file_index) diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 9e99c423978c..3688e8420119 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -63,7 +63,7 @@ class BackupCoordinationLocal : public IBackupCoordination void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; - BackupFileInfos getFileInfosForAllHosts() const override; + void forEachFileInfoForAllHosts(const std::function & callback) const override; bool startWritingFile(size_t data_file_index) override; ZooKeeperRetriesInfo getOnClusterInitializationKeeperRetriesInfo() const override; diff --git a/src/Backups/BackupCoordinationOnCluster.cpp b/src/Backups/BackupCoordinationOnCluster.cpp index 7530f046565f..b0622fee3d20 100644 --- a/src/Backups/BackupCoordinationOnCluster.cpp +++ b/src/Backups/BackupCoordinationOnCluster.cpp @@ -776,12 +776,12 @@ BackupFileInfos BackupCoordinationOnCluster::getFileInfos() const return file_infos->getFileInfos(current_host); } -BackupFileInfos BackupCoordinationOnCluster::getFileInfosForAllHosts() const +void BackupCoordinationOnCluster::forEachFileInfoForAllHosts(const std::function & callback) const { - auto component_guard = Coordination::setCurrentComponent("BackupCoordinationOnCluster::getFileInfosForAllHosts"); + auto component_guard = Coordination::setCurrentComponent("BackupCoordinationOnCluster::forEachFileInfoForAllHosts"); std::lock_guard lock{file_infos_mutex}; prepareFileInfos(); - return file_infos->getFileInfosForAllHosts(); + file_infos->forEachFileInfoForAllHosts(callback); } void BackupCoordinationOnCluster::prepareFileInfos() const diff --git a/src/Backups/BackupCoordinationOnCluster.h b/src/Backups/BackupCoordinationOnCluster.h index 74d28ab61ed5..299a69e2196e 100644 --- a/src/Backups/BackupCoordinationOnCluster.h +++ b/src/Backups/BackupCoordinationOnCluster.h @@ -79,7 +79,7 @@ class BackupCoordinationOnCluster : public IBackupCoordination void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; - BackupFileInfos getFileInfosForAllHosts() const override; + void forEachFileInfoForAllHosts(const std::function & callback) const override; bool startWritingFile(size_t data_file_index) override; ZooKeeperRetriesInfo getOnClusterInitializationKeeperRetriesInfo() const override; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 3d1791a9126c..50a3c1ae5580 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -416,20 +416,21 @@ void BackupImpl::writeBackupMetadata() *out << "" << SettingFieldBackupDataFileNameGeneratorTypeTraits::toString(data_file_name_generator) << ""; - auto all_file_infos = coordination->getFileInfosForAllHosts(); + /// Iterate in place instead of copying all file infos (a backup can contain millions). + size_t num_all_file_infos = 0; + bool base_backup_in_use = false; + coordination->forEachFileInfoForAllHosts([&](const BackupFileInfo & info) + { + ++num_all_file_infos; + if (info.base_size) + base_backup_in_use = true; + }); - if (all_file_infos.empty()) + if (num_all_file_infos == 0) throw Exception(ErrorCodes::BACKUP_IS_EMPTY, "Backup must not be empty"); if (base_backup_info) { - bool base_backup_in_use = false; - for (const auto & info : all_file_infos) - { - if (info.base_size) - base_backup_in_use = true; - } - if (base_backup_in_use) { /// Persist base backup locators without inline `S3` credentials. @@ -462,13 +463,13 @@ void BackupImpl::writeBackupMetadata() *out << "" << original_namespace << ""; } - num_files = all_file_infos.size(); + num_files = num_all_file_infos; total_size = 0; num_entries = 0; size_of_entries = 0; *out << ""; - for (const auto & info : all_file_infos) + coordination->forEachFileInfoForAllHosts([&](const BackupFileInfo & info) { *out << ""; @@ -512,7 +513,7 @@ void BackupImpl::writeBackupMetadata() } *out << ""; - } + }); *out << ""; *out << ""; @@ -1254,8 +1255,10 @@ bool BackupImpl::tryRemoveAllFiles() noexcept else { files_to_remove.push_back(".backup"); - for (const auto & file_info : coordination->getFileInfosForAllHosts()) + coordination->forEachFileInfoForAllHosts([&](const BackupFileInfo & file_info) + { files_to_remove.push_back(file_info.data_file_name); + }); } if (!checkLockFile(false)) diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 5c31e62cb32d..48503cdde64f 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -109,7 +110,12 @@ class IBackupCoordination /// If specified checksum+size are new for this IBackupContentsInfo the function sets `is_data_file_required`. virtual void addFileInfos(BackupFileInfos && file_infos) = 0; virtual BackupFileInfos getFileInfos() const = 0; - virtual BackupFileInfos getFileInfosForAllHosts() const = 0; + + /// Iterates the file infos of all hosts in place, without copying them into a vector + /// (a backup can contain millions). + /// The callback may be called while an internal coordination mutex is held; it must not call back + /// into IBackupCoordination (risk of deadlocks). Prefer keeping the callback lightweight to avoid long critical sections. + virtual void forEachFileInfoForAllHosts(const std::function & callback) const = 0; /// Starts writing a specified file, the function returns false if that file is already being written concurrently. virtual bool startWritingFile(size_t data_file_index) = 0; From 974ca8e9e6dba9e5aa482e898870c7c497003740 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 31 Jul 2026 21:07:43 +0000 Subject: [PATCH 72/86] Backport #109232 to 26.6: AI functions parameter map --- .../sql-reference/functions/ai-functions.md | 81 +++-- src/Common/QueryFuzzer.cpp | 12 +- src/Core/Settings.cpp | 9 +- src/Core/SettingsChangesHistory.cpp | 3 +- src/Functions/FunctionBaseAI.cpp | 281 +++++++++++++----- src/Functions/FunctionBaseAI.h | 111 ++++--- src/Functions/aiClassify.cpp | 29 +- src/Functions/aiEmbed.cpp | 78 ++--- src/Functions/aiExtract.cpp | 29 +- src/Functions/aiGenerate.cpp | 50 ++-- src/Functions/aiTranslate.cpp | 46 ++- tests/integration/test_ai_functions/test.py | 244 ++++++++++----- .../0_stateless/03300_ai_functions.reference | 65 ++-- .../0_stateless/03300_ai_functions.sql | 241 +++++++++------ ...unctions_named_collection_access.reference | 8 + ...42_ai_functions_named_collection_access.sh | 36 ++- ...ai_functions_default_credentials.reference | 12 + ...04492_ai_functions_default_credentials.sql | 78 +++++ 18 files changed, 968 insertions(+), 445 deletions(-) create mode 100644 tests/queries/0_stateless/04492_ai_functions_default_credentials.reference create mode 100644 tests/queries/0_stateless/04492_ai_functions_default_credentials.sql diff --git a/docs/en/sql-reference/functions/ai-functions.md b/docs/en/sql-reference/functions/ai-functions.md index 2152b0ab7f21..7bfc8c041887 100644 --- a/docs/en/sql-reference/functions/ai-functions.md +++ b/docs/en/sql-reference/functions/ai-functions.md @@ -23,38 +23,31 @@ All functions are sharing a common infrastructure that provides: ## Configuration {#configuration} -AI functions resolve provider credentials and configuration from a [**named collection**](/operations/named-collections). To set a named collection to use for credentials, use the [`ai_function_credentials`](/operations/settings/settings#ai_function_credentials) setting. +AI functions reference a [**named collection**](/operations/named-collections) that stores provider credentials and configuration. Different named collections can be created and used for different functions or functions calls. For example you may want to define a different named collection to use with the text functions (`aiGenerate`, `aiClassify`, `aiExtract`, `aiTranslate`) vs the `aiEmbed` function, which require different endpoints and usually use different models. -Example statement to create a named collection with provider credentials: +Example statement to create a named collection with provider credentials, one with a chat endpoint and another with an embedding endpoint: ```sql -CREATE NAMED COLLECTION my_ai_credentials AS +CREATE NAMED COLLECTION ai_text_credentials AS provider = 'openai', endpoint = 'https://api.openai.com/v1/chat/completions', model = 'gpt-4o-mini', api_key = 'sk-...'; -``` - -Select the collection with the `ai_function_credentials` setting, for the session or for a single query: -```sql --- For the session: -SET allow_experimental_ai_functions = 1; -SET ai_function_credentials = 'my_ai_credentials'; -SELECT aiClassify('I love this product!', ['positive', 'negative', 'neutral']); --- Or for a single query: -SELECT aiClassify('I love this product!', ['positive', 'negative', 'neutral']) -SETTINGS allow_experimental_ai_functions = 1, ai_function_credentials = 'my_ai_credentials'; +-- `aiEmbed` does not read `model` from the named collection; pass it as a positional argument instead. +-- Defining `model` in an `aiEmbed` collection is an error, not silently ignored. +CREATE NAMED COLLECTION ai_embedding_credentials AS + provider = 'openai', + endpoint = 'https://api.openai.com/v1/embeddings', + api_key = 'sk-...'; ``` -When `ai_function_credentials` is empty (the default), an exception is raised. - ### Named collection parameters {#named-collection-parameters} | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `provider` | String | — | Model provider. Supported: `'openai'`, `'anthropic'`. See note below. | | `endpoint` | String | — | API endpoint URL. | -| `model` | String | — | Model name (e.g. `'gpt-4o-mini'`, `'text-embedding-3-small'`). | +| `model` | String | — | Model name (e.g. `'gpt-4o-mini'`). Used by the text functions; `aiEmbed` requires `model` as a positional argument and errors if `model` is specified in the named collection. | | `api_key` | String | — | Authentication key for the provider. Optional: when omitted, the auth header is not sent, which allows targeting OpenAI-compatible servers that do not require authentication. | | `max_tokens` | UInt64 | `1024` | Maximum number of output tokens per API call. | | `api_version` | String | — | API version string. Used by Anthropic (`'2023-06-01'`). | @@ -63,27 +56,65 @@ When `ai_function_credentials` is empty (the default), an exception is raised. Any OpenAI-compatible API (e.g. vLLM, Ollama, LiteLLM) can be used by setting `provider = 'openai'` and pointing the `endpoint` to your service. ::: +### Selecting credentials {#selecting-credentials} + +A function resolves the named collection to use from, in order: + +1. the `credentials` key of its parameter map, when present; +2. otherwise the applicable default-credentials setting: + - [`ai_function_text_default_credentials`](/operations/settings/settings#ai_function_text_default_credentials) for the text functions (`aiGenerate`, `aiClassify`, `aiExtract`, `aiTranslate`); + - [`ai_function_embedding_default_credentials`](/operations/settings/settings#ai_function_embedding_default_credentials) for `aiEmbed`. + +If neither is set, the call fails. The text and embedding functions use separate default settings because a chat-completions endpoint differs from an embeddings one. + +```sql +SET ai_function_text_default_credentials = 'ai_text_credentials'; + +-- Uses ai_text_credentials from the setting: +SELECT aiGenerate('What is 2 + 2? Reply with just the number.'); + +-- Overrides the default for this call: +SELECT aiGenerate('Bonjour', map('credentials', 'other_credentials')); +``` + +### Parameter map {#parameter-map} + +Each function accepts an optional trailing `Map(String, String)` of parameters. All values are strings (quote numbers, e.g. `'0.2'`). Unknown keys are rejected. A key that is present overrides the corresponding named-collection value; a key that is absent falls back to the named collection (for `model`/`max_tokens`) or the built-in default. The exception is `aiEmbed`, which takes `model` as a required positional argument (`aiEmbed(text, model[, params])`) and errors if it is instead set in the parameter map or named collection. + +The following parameters are common to all the AI functions: + +| Key | Description | +|-----|-------------| +| `credentials` | Named collection to use (see above). | +| `model` | Overrides the collection's `model` (text functions only; `aiEmbed` takes `model` as a required positional argument, not a map key). | + +Individual functions accept additional, function-specific parameters (such as `max_tokens`, `temperature`, `system_prompt`, `instructions`, and `dimensions`). See each function's reference below for the parameters it accepts and their defaults. + +```sql +SELECT aiGenerate(body, map('temperature', '0.2', 'system_prompt', 'You are terse.')) FROM articles; +``` + ### Query-level settings {#query-level-settings} -Which named collection to use is controlled by the [`ai_function_credentials`](/operations/settings/settings#ai_function_credentials) setting. Other AI-related settings are listed in [Settings](/operations/settings/settings) under the `ai_function_` prefix. +All AI-related settings are listed in [Settings](/operations/settings/settings) under the `ai_function_` prefix. ### Use in `DEFAULT` and `MATERIALIZED` columns {#default-and-materialized-columns} -The `ai_function_credentials` setting is read when the default expression is evaluated, NOT when the column is defined. The collection name is not stored in the column definition: +A default-credentials setting is read when the default expression is evaluated, NOT when the column is defined. The collection name is not stored in the column definition unless the expression passes `credentials` in its parameter map: ```sql -CREATE TABLE t (id UInt32, doc String, vector Array(Float32) DEFAULT aiEmbed(doc)) ...; --- The stored default is `aiEmbed(doc)`; no collection is captured. +CREATE TABLE t (id UInt32, doc String, vector Array(Float32) DEFAULT aiEmbed(doc, 'text-embedding-3-small')) ...; +-- The stored default is `aiEmbed(doc, 'text-embedding-3-small')`; no collection is captured. ``` -Evaluating the expression requires three things: `allow_experimental_ai_functions` and `ai_function_credentials` must be set, and the evaluating user must hold `GRANT NAMED COLLECTION` on the collection (resolving the credentials runs a `NAMED COLLECTION` access check). Any of them missing raises an exception (`SUPPORT_IS_DISABLED`, an empty-credentials error, or `ACCESS_DENIED`). +Evaluating the expression requires three things: `allow_experimental_ai_functions` must be set, the credentials must resolve (from the expression's `credentials` parameter or the applicable default-credentials setting), and the evaluating user must hold `GRANT NAMED COLLECTION` on the collection (resolving the credentials runs a `NAMED COLLECTION` access check). Any of them missing raises an exception (`SUPPORT_IS_DISABLED`, an empty-credentials error, or `ACCESS_DENIED`). A `DEFAULT` column is evaluated at `INSERT`, so both settings must be set in the inserting session or query: ```sql -GRANT NAMED COLLECTION ON my_ai_credentials TO user; +GRANT NAMED COLLECTION ON ai_embedding_credentials TO user; SET allow_experimental_ai_functions = 1; -SET ai_function_credentials = 'my_ai_credentials'; +SET ai_function_embedding_default_credentials = 'ai_embedding_credentials'; INSERT INTO t (id, doc) VALUES (1, 'hello'); ``` @@ -93,7 +124,7 @@ To make such tables insertable without setting these per session, set both in a 1 - my_ai_credentials + ai_embedding_credentials ``` diff --git a/src/Common/QueryFuzzer.cpp b/src/Common/QueryFuzzer.cpp index ad8498c40e3c..0eae045b7a6d 100644 --- a/src/Common/QueryFuzzer.cpp +++ b/src/Common/QueryFuzzer.cpp @@ -3712,11 +3712,13 @@ static const std::vector> & swapFuncs {"naiveBayesClassifier", "detectCharset", "detectLanguage", "detectLanguageUnknown", "detectLanguageMixed", "detectTonality"}, /// Word-level NLP (language/extension + word) {"stem", "lemmatize", "synonyms"}, - /// AI text functions: (text[, system_prompt | instruction_or_schema | target_language][, ...]). - /// All take a leading String `text` followed by String-typed semantic arguments, so a name swap keeps - /// the call parseable. `aiEmbed` (second arg `dimensions` is `UInt`) and `aiClassify` (second arg - /// `categories` is `Array(String)`) have incompatible argument types and are intentionally excluded. - {"aiGenerate", "aiExtract", "aiTranslate"}, + /// AI text generation: text + optional params map + {"aiGenerate"}, + /// aiEmbed takes (text, model[, params]); its arity matches no other AI function, so it is not + /// grouped for name-swapping (a swap would produce arity-mismatched calls). + {"aiEmbed"}, + /// AI functions: text + a per-function arg (categories / instruction / target_language) + optional params map + {"aiClassify", "aiExtract", "aiTranslate"}, /// Geo distance functions (lon1, lat1, lon2, lat2 → Float64) {"greatCircleDistance", "geoDistance", "greatCircleAngle"}, /// Consistent hash functions (value, num_buckets → Int32) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 1e6a02bcb7fd..dd2155cf5814 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8348,9 +8348,6 @@ Maximum number of WebAssembly UDF instances that can run in parallel per functio /* AI function settings */ \ DECLARE(Bool, allow_experimental_ai_functions, false, R"( Enable experimental AI functions (e.g. `aiGenerateContent`). These functions make external HTTP calls to AI providers. -)", EXPERIMENTAL) \ - DECLARE(String, ai_function_credentials, "", R"( -Name of the named collection that AI functions use for provider credentials and configuration (`provider`, `endpoint`, `model`, optional `api_key`, etc.). When empty, an exception is raised. )", EXPERIMENTAL) \ DECLARE(UInt64, ai_function_request_timeout_sec, 60, R"( Timeout in seconds for individual HTTP requests made by AI functions (AI chat completions and embedding API calls). If a request does not complete within this time, it is considered failed and may be retried according to `ai_function_max_retries`. @@ -8382,6 +8379,12 @@ If true (default), exceeding an AI function quota limit (`ai_function_max_input_ )", EXPERIMENTAL) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by `aiEmbed`. Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. +)", EXPERIMENTAL) \ + DECLARE(String, ai_function_text_default_credentials, "", R"( +Name of the named collection used by the text AI functions (`aiGenerate`, `aiClassify`, `aiExtract`, `aiTranslate`) when the call does not pass `credentials` in its parameter map. Empty means no default: such calls must pass `credentials` explicitly. A chat-completions endpoint differs from an embeddings one, so this is separate from `ai_function_embedding_default_credentials`. +)", EXPERIMENTAL) \ + DECLARE(String, ai_function_embedding_default_credentials, "", R"( +Name of the named collection used by `aiEmbed` when the call does not pass `credentials` in its parameter map. Empty means no default: such calls must pass `credentials` explicitly. `aiEmbed` takes `model` as a required positional argument, not from the named collection. Kept separate from `ai_function_text_default_credentials` because an embeddings endpoint differs from a chat one. )", EXPERIMENTAL) \ /* ############ END OF EXPERIMENTAL FEATURES ############# */ \ /* ####################################################### */ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index dcf7f8814427..a7784fbad966 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -51,7 +51,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"use_lightweight_primary_key_index_analysis", false, true, "New setting to optimize primary key index analysis for tables with long primary keys"}, {"ai_function_embedding_max_batch_size", 100, 100, "New setting"}, {"enable_nullable_tuple_type", false, false, "Nullable Tuple is now Beta. Added as an alias for 'allow_experimental_nullable_tuple_type'."}, - {"ai_function_credentials", "", "", "New setting"}, + {"ai_function_text_default_credentials", "", "", "New setting"}, + {"ai_function_embedding_default_credentials", "", "", "New setting"}, {"enable_sharding_aggregator", false, false, "New setting to enable sharded `GROUP BY` optimization that distributes rows across threads by hashing the grouping key, so each thread aggregates a disjoint subset of keys without a merge phase; this is efficient for high cardinality keys with evenly distributed data."}, {"allow_experimental_text_index_lazy_apply", false, false, "New setting to gate experimental lazy posting list apply mode"}, {"text_index_posting_list_apply_mode", "materialize", "materialize", "New setting for lazy posting list apply mode"}, diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 262918b32655..d0d5f94b2781 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -13,9 +15,14 @@ #include #include #include +#include #include +#include +#include #include #include +#include + namespace ProfileEvents { extern const Event AIInputTokens; @@ -31,7 +38,6 @@ namespace DB namespace Setting { extern const SettingsBool allow_experimental_ai_functions; - extern const SettingsString ai_function_credentials; extern const SettingsUInt64 ai_function_request_timeout_sec; extern const SettingsUInt64 ai_function_max_retries; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; @@ -40,6 +46,7 @@ namespace Setting extern const SettingsUInt64 ai_function_max_output_tokens_per_query; extern const SettingsUInt64 ai_function_max_api_calls_per_query; extern const SettingsBool ai_function_throw_on_quota_exceeded; + extern const SettingsString ai_function_text_default_credentials; } namespace ErrorCodes @@ -67,123 +74,259 @@ String sanitizeTextForAI(std::string_view input) return output; } +/// Providers serialize integer fields (`max_tokens`, `dimensions`) as `Int64` (Poco JSON has no +/// UInt64). Reject values that would silently become negative after the cast. +void checkUIntFitsInt64(UInt64 value, std::string_view name) +{ + if (value > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI function parameter '{}' exceeds maximum ({})", + name, std::numeric_limits::max()); +} + +/// Parse a map value (string) into a `Field` of the parameter's kind. +Field parseAIParamValue(AIParamKind kind, const String & raw, std::string_view name) +{ + switch (kind) + { + case AIParamKind::String: + return Field(raw); + case AIParamKind::Float: + try + { + return Field(parseFromString(raw)); + } + catch (...) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI function parameter '{}' must be a number, got '{}'", name, raw); + } + case AIParamKind::UInt: + { + /// Special UInt64 handling to avoid potential overflow + UInt64 value = 0; + ReadBufferFromString buf(raw); + if (!tryReadIntText(value, buf) || !buf.eof()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "AI function parameter '{}' must be a non-negative integer, got '{}'", name, raw); + checkUIntFitsInt64(value, name); + return Field(value); + } + } + std::unreachable(); +} + +/// Read a parameter's fallback value from the named collection (used when `inherit_from_collection`). +Field readAIParamFromCollection(AIParamKind kind, const NamedCollectionPtr & collection, std::string_view name) +{ + const String key(name); + switch (kind) + { + case AIParamKind::String: + return Field(collection->get(key)); + case AIParamKind::Float: + return Field(collection->get(key)); + case AIParamKind::UInt: + { + UInt64 value = collection->get(key); + checkUIntFitsInt64(value, name); + return Field(value); + } + } + std::unreachable(); +} + } FunctionBaseAI::FunctionBaseAI(ContextPtr context_) : context(context_) { - const auto & settings = getContext()->getSettingsRef(); - if (!settings[Setting::allow_experimental_ai_functions]) + if (!getContext()->getSettingsRef()[Setting::allow_experimental_ai_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AI functions are experimental. Set `allow_experimental_ai_functions` setting to enable it"); - - credentials_collection_name = settings[Setting::ai_function_credentials]; } -FunctionBaseAI::AINamedCollectionConfig FunctionBaseAI::resolveAINamedCollection(const ContextPtr & context, const String & collection_name) +bool FunctionBaseAI::isStringToStringMap(const IDataType & type) { - AINamedCollectionConfig config; - config.collection_name = collection_name; + const auto * map_type = typeid_cast(&type); + return map_type && isString(map_type->getKeyType()) && isString(map_type->getValueType()); +} - if (config.collection_name.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "AI functions require credentials: set the `ai_function_credentials` setting to the name of a named collection " - "containing the provider configuration (`provider`, `endpoint`, `model`, ...)"); +AIParamSpecs FunctionBaseAI::commonParams() +{ + return { + /// `credentials` is required, but falls back to the default-credentials setting (handled in resolveAIParams). + {"credentials", AIParamKind::String, std::nullopt}, + /// `model` is required, but is normally supplied by the named collection. + {"model", AIParamKind::String, std::nullopt, /*inherit_from_collection=*/ true}, + {"max_tokens", AIParamKind::UInt, Field(DEFAULT_AI_MAX_TOKENS), /*inherit_from_collection=*/ true}, + }; +} - context->checkAccess(AccessType::NAMED_COLLECTION, config.collection_name); +AIParamSpecs FunctionBaseAI::allParams() const +{ + auto spec = commonParams(); + auto extra = functionParams(); + spec.insert(spec.end(), extra.begin(), extra.end()); + return spec; +} - const auto & named_collection = NamedCollectionFactory::instance().get(config.collection_name); +namespace +{ - config.provider = named_collection->getOrDefault("provider", ""); - config.endpoint = named_collection->getOrDefault("endpoint", ""); - config.model = named_collection->getOrDefault("model", ""); - config.api_key = named_collection->getOrDefault("api_key", ""); - config.api_version = named_collection->getOrDefault("api_version", ""); +const Field & getResolvedAIParam(const AIParamValues & values, std::string_view key) +{ + auto it = values.find(key); + chassert(it != values.end()); + return it->second; +} - if (config.provider.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}' must have 'provider'", config.collection_name); - if (config.endpoint.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}' must have 'endpoint'", config.collection_name); - if (config.model.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}' must have 'model'", config.collection_name); +} - context->getRemoteHostFilter().checkURL(Poco::URI(config.endpoint)); +String FunctionBaseAI::AIParams::getString(std::string_view key) const +{ + return getResolvedAIParam(values, key).safeGet(); +} - return config; +Float64 FunctionBaseAI::AIParams::getFloat(std::string_view key) const +{ + return getResolvedAIParam(values, key).safeGet(); } -UInt64 FunctionBaseAI::computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt) +UInt64 FunctionBaseAI::AIParams::getUInt(std::string_view key) const { - constexpr UInt64 max_retry_delay_ms = 60'000; - UInt64 delay_ms = std::min(initial_delay_ms, max_retry_delay_ms); - for (UInt64 i = 0; i < attempt && delay_ms < max_retry_delay_ms; ++i) - delay_ms = std::min(delay_ms * 2, max_retry_delay_ms); - return delay_ms; + return getResolvedAIParam(values, key).safeGet(); } -FunctionBaseAI::ResolvedConfig FunctionBaseAI::resolveConfig() const +FunctionBaseAI::AIParams FunctionBaseAI::resolveAIParams( + const ContextPtr & context, + const ColumnsWithTypeAndName & arguments, + const AIParamSpecs & spec, + const String & default_credentials) { - auto base = resolveAINamedCollection(getContext(), credentials_collection_name); + /// The parameter map, when present, is the last argument (validated as a const Map(String, String) + /// by getReturnTypeImpl). Read it into a plain string->string map. + std::map> map_values; // STYLE_CHECK_ALLOW_STD_CONTAINERS + if (!arguments.empty() && isStringToStringMap(*arguments.back().type)) + { + const auto * map_const = typeid_cast(arguments.back().column.get()); + if (!map_const) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI function parameter map must be a constant"); + + const Map & map = (*map_const->getDataColumnPtr())[0].safeGet(); + for (const auto & element : map) + { + const Tuple & kv = element.safeGet(); + const String & key = kv[0].safeGet(); + if (!map_values.emplace(key, kv[1].safeGet()).second) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Duplicate AI function parameter '{}' in the parameter map", key); + } + } + + /// Reject unknown keys so typos surface immediately instead of being silently ignored. + for (const auto & [key, _] : map_values) + { + bool known = std::any_of(spec.begin(), spec.end(), [&](const AIParamSpec & p) { return p.name == key; }); + if (!known) + { + if (key == "model") + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "This function does not accept 'model' in the parameter map; pass 'model' to the function directly"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown AI function parameter '{}'", key); + } + } - ResolvedConfig config; - config.provider = std::move(base.provider); - config.endpoint = std::move(base.endpoint); - config.model = std::move(base.model); - config.api_key = std::move(base.api_key); - config.api_version = std::move(base.api_version); - config.temperature = defaultTemperature(); + /// Resolve credentials first: they name the collection everything else is read from. + String credentials; + if (auto it = map_values.find("credentials"); it != map_values.end()) + credentials = it->second; + else + credentials = default_credentials; - const auto & named_collection = NamedCollectionFactory::instance().get(base.collection_name); - config.max_tokens = named_collection->getOrDefault("max_tokens", DEFAULT_AI_MAX_TOKENS); + if (credentials.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "AI function requires credentials: pass 'credentials' in the parameter map or set the default-credentials setting"); - /// Poco JSON does not support UInt64, so providers cast max_tokens to Int64 for serialization. - if (config.max_tokens > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}': max_tokens exceeds maximum ({})", - base.collection_name, std::numeric_limits::max()); + context->checkAccess(AccessType::NAMED_COLLECTION, credentials); + const auto & collection = NamedCollectionFactory::instance().get(credentials); - return config; -} + AIParams params; + params.collection.collection_name = credentials; + params.collection.provider = collection->getOrDefault("provider", ""); + params.collection.endpoint = collection->getOrDefault("endpoint", ""); + params.collection.api_key = collection->getOrDefault("api_key", ""); + params.collection.api_version = collection->getOrDefault("api_version", ""); -float FunctionBaseAI::resolveTemperature(const ColumnsWithTypeAndName & arguments, const ResolvedConfig & config) const -{ - size_t temp_idx = temperatureArgumentIndex(); - if (temp_idx < arguments.size() && isNumber(arguments[temp_idx].type)) + if (params.collection.provider.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}' must have 'provider'", credentials); + if (params.collection.endpoint.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "AI named collection '{}' must have 'endpoint'", credentials); + + context->getRemoteHostFilter().checkURL(Poco::URI(params.collection.endpoint)); + + /// A function that does not declare `model` (i.e. `aiEmbed`, which takes it as an argument) must + /// not silently ignore a `model` defined in the named collection: reject it instead. + const bool declares_model = std::any_of(spec.begin(), spec.end(), [](const AIParamSpec & p) { return p.name == "model"; }); + if (!declares_model && collection->has("model")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "AI named collection '{}' defines 'model', which this function does not read from the named collection; " + "remove it from the collection and pass 'model' to the function directly", credentials); + + /// Resolve every declared parameter: map override -> named collection (if inherited) -> default. + for (const auto & p : spec) { - const auto * col_const = typeid_cast(arguments[temp_idx].column.get()); - if (col_const) - return static_cast(col_const->getFloat64(0)); + if (p.name == "credentials") + continue; + + if (auto it = map_values.find(p.name); it != map_values.end()) + params.values.emplace(String(p.name), parseAIParamValue(p.kind, it->second, p.name)); + else if (p.inherit_from_collection && collection->has(String(p.name))) + params.values.emplace(String(p.name), readAIParamFromCollection(p.kind, collection, p.name)); + else if (p.default_value) + params.values.emplace(String(p.name), *p.default_value); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "AI named collection '{}' must have '{}', or it must be passed in the parameter map", credentials, p.name); } - return config.temperature; + return params; +} + +UInt64 FunctionBaseAI::computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt) +{ + constexpr UInt64 max_retry_delay_ms = 60'000; + UInt64 delay_ms = std::min(initial_delay_ms, max_retry_delay_ms); + for (UInt64 i = 0; i < attempt && delay_ms < max_retry_delay_ms; ++i) + delay_ms = std::min(delay_ms * 2, max_retry_delay_ms); + return delay_ms; } ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { - auto config = resolveConfig(); + const auto & settings = getContext()->getSettingsRef(); + auto params = resolveAIParams(getContext(), arguments, allParams(), settings[Setting::ai_function_text_default_credentials]); + + String model = params.getString("model"); + UInt64 max_tokens = params.getUInt("max_tokens"); + float temperature = static_cast(params.getFloat("temperature")); /// Row-independent validation must run before the zero-row fast path so malformed constant /// arguments fail consistently regardless of source size. checkSanityBeforeExecuteImpl(arguments, result_type, input_rows_count); - String system_prompt = sanitizeTextForAI(buildSystemPrompt(arguments)); + String system_prompt = sanitizeTextForAI(buildSystemPrompt(arguments, params)); auto response_format = buildResponseFormat(arguments); - auto provider = createAIProvider(config.provider, config.endpoint, config.api_key, config.api_version); + auto provider = createAIProvider(params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (input_rows_count == 0) return result_type->createColumn(); /// A Nullable prompt can arrive as `ColumnNullable` or as `ColumnConst(ColumnNullable)` (e.g. `NULL::Nullable(String)`). /// `convertToFullColumnIfConst` unwraps the latter into the former, so a single null-map path handles both. - size_t prompt_idx = promptArgumentIndex(); ColumnPtr prompt_column; const ColumnNullable * prompt_nullable = nullptr; - if (prompt_idx < arguments.size() && arguments[prompt_idx].type->isNullable()) + if (arguments[0].type->isNullable()) { - prompt_column = arguments[prompt_idx].column->convertToFullColumnIfConst(); + prompt_column = arguments[0].column->convertToFullColumnIfConst(); prompt_nullable = typeid_cast(prompt_column.get()); } - float temperature = resolveTemperature(arguments, config); - - const auto & settings = getContext()->getSettingsRef(); UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; UInt64 max_retries = settings[Setting::ai_function_max_retries].value; UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; @@ -236,9 +379,9 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, ai_request.system_prompt = system_prompt; ai_request.user_message = user_message; ai_request.response_format = response_format; - ai_request.model = config.model; + ai_request.model = model; ai_request.temperature = temperature; - ai_request.max_tokens = config.max_tokens; + ai_request.max_tokens = max_tokens; /// update api_calls/quotas before call so failed calls are still added to total ++total_api_calls; diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index ab6bf6da1ff5..daaeda01b403 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -6,12 +6,48 @@ #include #include #include +#include + +#include +#include +#include +#include +#include namespace DB { static constexpr UInt64 DEFAULT_AI_MAX_TOKENS = 1024; +/// Logical type of an AI-function parameter. The parameter map is `Map(String, String)`, so every +/// value arrives as a `String`; the kind tells the resolver how to parse each one into its real type +/// (and how to read the same key from a named collection). +enum class AIParamKind +{ + String, + Float, + UInt, +}; + +/// Declarative description of one parameter an AI function accepts in its trailing +/// `Map(String, String)` argument. Each function declares its own params; `FunctionBaseAI` +/// contributes the common ones (`credentials`, `model`, `max_tokens`). +struct AIParamSpec +{ + std::string_view name; + AIParamKind kind; + /// `std::nullopt` => required: the resolver throws if the key is absent everywhere. + std::optional default_value; + /// When true, a missing map key falls back to the same-named field of the named collection + /// before falling back to `default_value` (e.g. `model`, `max_tokens`). + bool inherit_from_collection = false; +}; + +/// Small, short-lived config containers (a handful of entries, not user-data-scaled), so the +/// default allocator is fine and a memory-tracking variant would add nothing. +using AIParamSpecs = std::vector; // STYLE_CHECK_ALLOW_STD_CONTAINERS +using AIParamValues = std::map>; // STYLE_CHECK_ALLOW_STD_CONTAINERS + class FunctionBaseAI : public IFunction { public: @@ -47,23 +83,49 @@ class FunctionBaseAI : public IFunction return inner; } - /// Fields read from the named collection that every AI function needs. Function-specific knobs - /// (max_tokens, temperature, dimensions, …) are layered on top by individual callers. + /// Connection fields read from the named collection. `model` is resolved separately as a + /// parameter (it may be overridden via the map), so it is not part of the connection config. struct AINamedCollectionConfig { String collection_name; String provider; String endpoint; - String model; String api_key; String api_version; }; - /// Resolve the named collection named by `collection_name` (the value of the `ai_function_credentials` - /// setting, read once at construction): error if it is empty, run the `NAMED_COLLECTION` access check, - /// fetch from `NamedCollectionFactory`, and validate that the required fields (`provider`, `endpoint`, - /// `model`) are non-empty. `api_key` is optional. - static AINamedCollectionConfig resolveAINamedCollection(const ContextPtr & context, const String & collection_name); + /// Resolved parameters for one AI-function call: the named-collection connection config plus the + /// per-parameter values (map override -> named collection -> declared default). Only keys that + /// resolved to a value are present; `has` distinguishes "absent" from "set to empty string". + struct AIParams + { + AINamedCollectionConfig collection; + AIParamValues values; + + bool has(std::string_view key) const { return values.contains(key); } + + String getString(std::string_view key) const; + Float64 getFloat(std::string_view key) const; + UInt64 getUInt(std::string_view key) const; + }; + + /// Type validator for the optional trailing parameter argument: a `Map(String, String)`. + static bool isStringToStringMap(const IDataType & type); + + /// Resolve the trailing `Map(String, String)` argument (if any) against `spec`: + /// - reject any map key not declared in `spec`; + /// - resolve `credentials` (map key -> `default_credentials` -> throw), then load and + /// access-check the named collection and validate `provider`/`endpoint`; + /// - resolve every other spec entry: map override -> (if `inherit_from_collection`) named + /// collection field -> `default_value` -> throw when required and absent. + static AIParams resolveAIParams( + const ContextPtr & context, + const ColumnsWithTypeAndName & arguments, + const AIParamSpecs & spec, + const String & default_credentials); + + /// Parameters common to every AI function. Function-specific params are appended by `functionParams`. + static AIParamSpecs commonParams(); /// Exponential backoff delay capped at one minute, so adversarial values of /// `ai_function_retry_initial_delay_ms` or `ai_function_max_retries` cannot produce a multi-hour @@ -74,21 +136,17 @@ class FunctionBaseAI : public IFunction ContextPtr context; ContextPtr getContext() const { return context; } - /// Value of the `ai_function_credentials` setting, read once at construction (it is constant for the query). - String credentials_collection_name; - virtual String functionName() const = 0; - /// Temperature controls the randomness or noise of the response. The accepted values depend on the AI provider - /// (0.0 - 2.0 for OpenAI, 0.0 - 1.0 for Anthropic). Lower is better for more deterministic tasks, while - /// a higher value is useful for creative tasks, such as chatting or text generation. - virtual float defaultTemperature() const = 0; + /// Function-specific parameters accepted in the trailing `Map(String, String)` argument, on top + /// of `commonParams`. Each entry carries its own default (or is required). Default: none. + virtual AIParamSpecs functionParams() const { return {}; } /// Performs additional validation of the input arguments. virtual void checkSanityBeforeExecuteImpl(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const {} /// A system prompt applies to each request. AI funcs will probably want to provide a default on a per-function basis. - virtual String buildSystemPrompt(const ColumnsWithTypeAndName & arguments) const = 0; + virtual String buildSystemPrompt(const ColumnsWithTypeAndName & arguments, const AIParams & params) const = 0; /// The user prompt is appended to the system prompt, this is usually what is contained in each row. virtual String buildUserMessage(const ColumnsWithTypeAndName & arguments, size_t row) const = 0; @@ -99,26 +157,9 @@ class FunctionBaseAI : public IFunction virtual String postProcessResponse(const String & raw_response) const { return raw_response; } - /// Index of the per-row text column in the arguments list. - virtual size_t promptArgumentIndex() const = 0; - - /// Index of the temperature argument. Return 0 if the function doesn't accept temperature. - virtual size_t temperatureArgumentIndex() const = 0; - private: - struct ResolvedConfig - { - String provider; - String endpoint; - String model; - String api_key; - String api_version; - float temperature = 0; - UInt64 max_tokens = 0; - }; - - ResolvedConfig resolveConfig() const; - float resolveTemperature(const ColumnsWithTypeAndName & arguments, const ResolvedConfig & config) const; + /// Full parameter spec for this function: `commonParams` followed by `functionParams`. + AIParamSpecs allParams() const; }; } diff --git a/src/Functions/aiClassify.cpp b/src/Functions/aiClassify.cpp index c8b2240ef94a..f54a3405894e 100644 --- a/src/Functions/aiClassify.cpp +++ b/src/Functions/aiClassify.cpp @@ -51,24 +51,23 @@ class FunctionAiClassify final : public FunctionBaseAI {"categories", static_cast(&isArrayOfStrings), &isColumnConst, "const Array(String)"}, }; FunctionArgumentDescriptors optional_args{ - {"temperature", static_cast(&isNumber), &isColumnConst, "const Number"}, + {"params", static_cast(&FunctionBaseAI::isStringToStringMap), &isColumnConst, "const Map(String, String)"}, }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); - return wrapReturnTypeForNullablePrompt(arguments, prompt_arg_index, std::make_shared()); + return wrapReturnTypeForNullablePrompt(arguments, 0, std::make_shared()); } private: static constexpr float default_temp = 0.0f; - static constexpr size_t prompt_arg_index = 0; static constexpr size_t categories_arg_index = 1; - static constexpr size_t temp_arg_idx = 2; String functionName() const override { return name; } - float defaultTemperature() const override { return default_temp; } - size_t promptArgumentIndex() const override { return prompt_arg_index; } - size_t temperatureArgumentIndex() const override { return temp_arg_idx; } + AIParamSpecs functionParams() const override + { + return {{"temperature", AIParamKind::Float, Field(static_cast(default_temp))}}; + } void checkSanityBeforeExecuteImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override { @@ -80,7 +79,7 @@ class FunctionAiClassify final : public FunctionBaseAI throw Exception(ErrorCodes::BAD_ARGUMENTS, "aiClassify: 'categories' must contain at least one label"); } - String buildSystemPrompt(const ColumnsWithTypeAndName & arguments) const override + String buildSystemPrompt(const ColumnsWithTypeAndName & arguments, const AIParams &) const override { const auto & col_categories = assert_cast(*arguments[categories_arg_index].column); auto categories = (*col_categories.getDataColumnPtr())[0].safeGet(); @@ -102,7 +101,7 @@ class FunctionAiClassify final : public FunctionBaseAI String buildUserMessage(const ColumnsWithTypeAndName & arguments, size_t row) const override { - return String(arguments[prompt_arg_index].column->getDataAt(row)); + return String(arguments[0].column->getDataAt(row)); } /// Builds the OpenAI `response_format` schema object constraining the model to output one of the @@ -191,18 +190,20 @@ The function sends the text together with a fixed classification prompt and a JS constraining the model to return exactly one of the supplied labels. When the response is returned as a JSON object of the form `{"category": "..."}`, the label is unwrapped and the label string is returned. -Provider credentials and configuration are taken from the named collection specified by the `ai_function_credentials` setting. +Credentials (a named collection specifying the provider, model, endpoint, and optionally an API key) +are taken from the `credentials` key of the optional parameter map, or from the +`ai_function_text_default_credentials` setting when the map omits it. )", - .syntax = "aiClassify(text, categories[, temperature])", + .syntax = "aiClassify(text, categories[, params])", .arguments = { {"text", "Text to classify.", {"String"}}, {"categories", "Constant list of candidate category labels.", {"Array(String)"}}, - {"temperature", "Sampling temperature controlling randomness. Default: `0.0`.", {"Float64"}}, + {"params", "Optional constant `Map(String, String)` of parameters. Function-specific keys: `temperature` (sampling temperature controlling randomness; default `0.0`), `max_tokens` (maximum output tokens per call; default `1024`). The common parameters `credentials` and `model` also apply (see [AI Functions](/sql-reference/functions/ai-functions)).", {"Map(String, String)"}}, }, .returned_value = {"One of the provided category labels, or the default value for the column type (empty string) if the request failed and `ai_function_throw_on_error` is disabled.", {"String"}}, .examples = { - {"Classify sentiment", "SELECT aiClassify('I love this product!', ['positive', 'negative', 'neutral']) SETTINGS ai_function_credentials = 'my_ai_credentials'", "positive"}, - {"Classify a column", "SELECT body, aiClassify(body, ['bug', 'question', 'feature']) AS kind FROM issues LIMIT 5", ""}, + {"Classify sentiment", "SELECT aiClassify('I love this product!', ['positive', 'negative', 'neutral'])", "positive"}, + {"Classify a column with explicit credentials", "SELECT body, aiClassify(body, ['bug', 'question', 'feature'], map('credentials', 'ai_text_credentials')) AS kind FROM issues LIMIT 5", ""}, }, .introduced_in = {26, 4}, .category = FunctionDocumentation::Category::AI}); diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index b2b0630dd78e..f3e404cbbc71 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -40,7 +40,6 @@ namespace DB namespace Setting { extern const SettingsBool allow_experimental_ai_functions; - extern const SettingsString ai_function_credentials; extern const SettingsUInt64 ai_function_request_timeout_sec; extern const SettingsUInt64 ai_function_max_retries; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; @@ -50,11 +49,11 @@ namespace Setting extern const SettingsUInt64 ai_function_max_api_calls_per_query; extern const SettingsBool ai_function_throw_on_quota_exceeded; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; + extern const SettingsString ai_function_embedding_default_credentials; } namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int SUPPORT_IS_DISABLED; @@ -72,12 +71,9 @@ class FunctionAiEmbed final : public IFunction explicit FunctionAiEmbed(ContextPtr context_) : context(context_) { - const auto & settings = getContext()->getSettingsRef(); - if (!settings[Setting::allow_experimental_ai_functions]) + if (!getContext()->getSettingsRef()[Setting::allow_experimental_ai_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AI functions are experimental. Set `allow_experimental_ai_functions` setting to enable it"); - - credentials_collection_name = settings[Setting::ai_function_credentials]; } String getName() const override { return name; } @@ -103,42 +99,46 @@ class FunctionAiEmbed final : public IFunction { FunctionArgumentDescriptors mandatory_args{ {"text", static_cast(&FunctionBaseAI::isStringOrNullableString), nullptr, "String or Nullable(String)"}, + /// `model` must be a plain (non-nullable) `String`; constness is enforced by the column validator. + {"model", static_cast(&isString), &isColumnConst, "const String"}, }; FunctionArgumentDescriptors optional_args{ - {"dimensions", static_cast(&isNativeUInt), &isColumnConst, "const UInt"}, + {"params", static_cast(&FunctionBaseAI::isStringToStringMap), &isColumnConst, "const Map(String, String)"}, }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); return std::make_shared(std::make_shared()); } + /// Parameters accepted in the optional trailing `Map(String, String)` argument. `aiEmbed` does not + /// inherit `FunctionBaseAI`, so it declares its own spec (no `max_tokens`, which embeddings do not + /// use; no `model`, which is a required positional argument for `aiEmbed`). + static AIParamSpecs embeddingParams() + { + return { + {"credentials", AIParamKind::String, std::nullopt}, + {"dimensions", AIParamKind::UInt, Field(UInt64(0))}, + }; + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - auto nc = FunctionBaseAI::resolveAINamedCollection(getContext(), credentials_collection_name); + const auto & settings = getContext()->getSettingsRef(); + auto params = FunctionBaseAI::resolveAIParams( + getContext(), arguments, embeddingParams(), settings[Setting::ai_function_embedding_default_credentials]); - UInt64 dimensions = 0; - if (arguments.size() > 1) - { - const auto * dim_const = typeid_cast(arguments[1].column.get()); - chassert(dim_const, "dimensions must be a constant UInt (validated by getReturnTypeImpl)"); - dimensions = dim_const->getUInt(0); - - /// Providers serialize `dimensions` as Int64 (Poco JSON does not support UInt64). - /// Reject values that would silently become negative after the cast. - if (dimensions > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "aiEmbed: 'dimensions' exceeds maximum ({})", - std::numeric_limits::max()); - } + UInt64 dimensions = params.getUInt("dimensions"); + String model(arguments[model_arg_index].column->getDataAt(0)); - auto provider = createAIProvider(nc.provider, nc.endpoint, nc.api_key, nc.api_version); + auto provider = createAIProvider( + params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "AI provider '{}' does not support embeddings", nc.provider); + "AI provider '{}' does not support embeddings", params.collection.provider); if (input_rows_count == 0) return result_type->createColumn(); - const auto & settings = getContext()->getSettingsRef(); UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; UInt64 max_retries = settings[Setting::ai_function_max_retries].value; UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; @@ -209,7 +209,7 @@ class FunctionAiEmbed final : public IFunction size_t batch_end = std::min(batch_start + max_batch_size, live_rows.size()); AIEmbeddingRequest ai_embedding_request; - ai_embedding_request.model = nc.model; + ai_embedding_request.model = model; ai_embedding_request.dimensions = dimensions; ai_embedding_request.inputs.reserve(batch_end - batch_start); @@ -292,12 +292,10 @@ class FunctionAiEmbed final : public IFunction private: static constexpr size_t text_arg_index = 0; + static constexpr size_t model_arg_index = 1; ContextPtr context; ContextPtr getContext() const { return context; } - - /// Value of the `ai_function_credentials` setting, read once at construction (it is constant for the query). - String credentials_collection_name; }; } @@ -313,19 +311,29 @@ Within a single block of rows, inputs are grouped into batches of up to [`ai_function_embedding_max_batch_size`](/operations/settings/settings#ai_function_embedding_max_batch_size) entries per HTTP request to reduce per-call overhead. -Provider credentials and configuration are taken from the named collection specified by the `ai_function_credentials` setting. -The optional `dimensions` argument, when supported by the model (e.g. OpenAI's `text-embedding-3-*`), +Credentials (a named collection specifying the provider, endpoint, and optionally an API key) +are taken from the `credentials` key of the parameter map, or from the +`ai_function_embedding_default_credentials` setting when the map omits it. Note that `aiEmbed` uses a +separate default-credentials setting from the text functions, since an embeddings endpoint differs +from a chat one. + +The `model` is a required positional argument (a constant `String`). Unlike the text functions, +`aiEmbed` does not read `model` from the named collection or the parameter map. A named collection +that defines `model` is rejected rather than silently ignored. + +The optional `dimensions` parameter, when supported by the model (e.g. OpenAI's `text-embedding-3-*`), requests a vector of the given size; otherwise the model's native size is returned. )", - .syntax = "aiEmbed(text[, dimensions])", + .syntax = "aiEmbed(text, model[, params])", .arguments = {{"text", "Text to embed.", {"String"}}, - {"dimensions", "Optional target dimensionality for the output vector. `0` or omitted means the model's native size.", {"UInt64"}}}, + {"model", "Embedding model name.", {"const String"}}, + {"params", "Optional constant `Map(String, String)` of parameters. Function-specific key: `dimensions` (target dimensionality of the output vector; `0` or omitted means the model's native size). The common parameter `credentials` also applies (see [AI Functions](/sql-reference/functions/ai-functions)).", {"Map(String, String)"}}}, .returned_value = {"The embedding vector, or an empty array if the input is NULL or empty, the request failed and `ai_function_throw_on_error` is disabled, or a quota was exceeded with `ai_function_throw_on_quota_exceeded` disabled.", {"Array(Float32)"}}, .examples - = {{"Embed a single string", "SELECT aiEmbed('Hello world') SETTINGS ai_function_credentials = 'my_ai_credentials'", ""}, - {"With explicit dimensions", "SELECT aiEmbed('Hello world', 256) SETTINGS ai_function_credentials = 'my_ai_credentials'", ""}, - {"Embed a column of texts", "SELECT aiEmbed(title, 256) FROM articles LIMIT 10", ""}}, + = {{"Embed a single string (`credentials` can be omitted if the `ai_function_embedding_default_credentials` setting is set)", "SELECT aiEmbed('Hello world', 'text-embedding-3-small', map('credentials', 'ai_embedding_credentials'))", ""}, + {"With explicit dimensions", "SELECT aiEmbed('Hello world', 'text-embedding-3-small', map('credentials', 'ai_embedding_credentials', 'dimensions', '256'))", ""}, + {"Embed a column of texts", "SELECT aiEmbed(title, 'text-embedding-3-small', map('credentials', 'ai_embedding_credentials', 'dimensions', '256')) FROM articles LIMIT 10", ""}}, .introduced_in = {26, 6}, .category = FunctionDocumentation::Category::AI}); } diff --git a/src/Functions/aiExtract.cpp b/src/Functions/aiExtract.cpp index 86cf0b4a4840..6941aa9744e2 100644 --- a/src/Functions/aiExtract.cpp +++ b/src/Functions/aiExtract.cpp @@ -39,24 +39,23 @@ class FunctionAiExtract final : public FunctionBaseAI {"instruction_or_schema", static_cast(&isString), &isColumnConst, "const String"}, }; FunctionArgumentDescriptors optional_args{ - {"temperature", static_cast(&isNumber), &isColumnConst, "const Number"}, + {"params", static_cast(&FunctionBaseAI::isStringToStringMap), &isColumnConst, "const Map(String, String)"}, }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); - return wrapReturnTypeForNullablePrompt(arguments, prompt_arg_index, std::make_shared()); + return wrapReturnTypeForNullablePrompt(arguments, 0, std::make_shared()); } private: static constexpr float default_temp = 0.0f; - static constexpr size_t prompt_arg_index = 0; static constexpr size_t instruction_arg_index = 1; - static constexpr size_t temp_arg_idx = 2; String functionName() const override { return name; } - float defaultTemperature() const override { return default_temp; } - size_t promptArgumentIndex() const override { return prompt_arg_index; } - size_t temperatureArgumentIndex() const override { return temp_arg_idx; } + AIParamSpecs functionParams() const override + { + return {{"temperature", AIParamKind::Float, Field(static_cast(default_temp))}}; + } static bool isJSONSchema(const String & instruction) { @@ -69,7 +68,7 @@ class FunctionAiExtract final : public FunctionBaseAI return String(arguments[instruction_arg_index].column->getDataAt(0)); } - String buildSystemPrompt(const ColumnsWithTypeAndName & arguments) const override + String buildSystemPrompt(const ColumnsWithTypeAndName & arguments, const AIParams &) const override { auto instruction = getInstruction(arguments); if (isJSONSchema(instruction)) @@ -82,7 +81,7 @@ class FunctionAiExtract final : public FunctionBaseAI String buildUserMessage(const ColumnsWithTypeAndName & arguments, size_t row) const override { - return String(arguments[prompt_arg_index].column->getDataAt(row)); + return String(arguments[0].column->getDataAt(row)); } /// Builds the OpenAI `response_format` schema object. Two shapes depending on `instruction_or_schema`: @@ -225,23 +224,25 @@ REGISTER_FUNCTION(AiExtract) .description = R"( Extracts structured information from unstructured text using an LLM provider. -The second argument may be either a free-form natural-language instruction (e.g. `'the main complaint'`) or a +The third argument may be either a free-form natural-language instruction (e.g. `'the main complaint'`) or a JSON-encoded schema of the form `'{"field_a": "description of field a", "field_b": "description of field b"}'`. In instruction mode, the function returns the extracted value as a plain string, or an empty string if nothing was found. In schema mode, the function returns a JSON object string whose keys match the requested schema; missing fields are `null`. -Provider credentials and configuration are taken from the named collection specified by the `ai_function_credentials` setting. +Credentials (a named collection specifying the provider, model, endpoint, and optionally an API key) +are taken from the `credentials` key of the optional parameter map, or from the +`ai_function_text_default_credentials` setting when the map omits it. )", - .syntax = "aiExtract(text, instruction_or_schema[, temperature])", + .syntax = "aiExtract(text, instruction_or_schema[, params])", .arguments = { {"text", "Text to extract information from.", {"String"}}, {"instruction_or_schema", "Free-form extraction instruction, or a constant JSON object describing the fields to extract.", {"const String"}}, - {"temperature", "Sampling temperature controlling randomness. Default: `0.0`.", {"const Float64"}}, + {"params", "Optional constant `Map(String, String)` of parameters. Function-specific keys: `temperature` (sampling temperature controlling randomness; default `0.0`), `max_tokens` (maximum output tokens per call; default `1024`). The common parameters `credentials` and `model` also apply (see [AI Functions](/sql-reference/functions/ai-functions)).", {"Map(String, String)"}}, }, .returned_value = {"A single extracted value (instruction mode) or a JSON object string (schema mode). Returns the default value for the column type (empty string) if the request failed and `ai_function_throw_on_error` is disabled.", {"String"}}, .examples = { - {"Free-form instruction", "SELECT aiExtract('The package arrived late and was damaged.', 'the main complaint') SETTINGS ai_function_credentials = 'my_ai_credentials'", "late and damaged package"}, + {"Free-form instruction", "SELECT aiExtract('The package arrived late and was damaged.', 'the main complaint')", "late and damaged package"}, {"Schema extraction", R"(SELECT aiExtract(review, '{"sentiment": "positive, negative or neutral", "topic": "main topic of the review"}') FROM reviews LIMIT 5)", ""}, }, .introduced_in = {26, 4}, diff --git a/src/Functions/aiGenerate.cpp b/src/Functions/aiGenerate.cpp index ba1849549d23..1ab75410da1e 100644 --- a/src/Functions/aiGenerate.cpp +++ b/src/Functions/aiGenerate.cpp @@ -34,41 +34,34 @@ class FunctionAiGenerate final : public FunctionBaseAI {"prompt", static_cast(&FunctionBaseAI::isStringOrNullableString), nullptr, "String or Nullable(String)"}, }; FunctionArgumentDescriptors optional_args{ - {"system_prompt", static_cast(&isString), &isColumnConst, "const String"}, - {"temperature", static_cast(&isNumber), &isColumnConst, "const Number"}, + {"params", static_cast(&FunctionBaseAI::isStringToStringMap), &isColumnConst, "const Map(String, String)"}, }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); - return wrapReturnTypeForNullablePrompt(arguments, prompt_arg_index, std::make_shared()); + return wrapReturnTypeForNullablePrompt(arguments, 0, std::make_shared()); } private: static constexpr float default_temp = 0.7f; - static constexpr size_t prompt_arg_index = 0; - static constexpr size_t system_prompt_arg_idx = 1; - static constexpr size_t temp_arg_idx = 2; String functionName() const override { return name; } - float defaultTemperature() const override { return default_temp; } - size_t promptArgumentIndex() const override { return prompt_arg_index; } - size_t temperatureArgumentIndex() const override { return temp_arg_idx; } + AIParamSpecs functionParams() const override + { + return { + {"temperature", AIParamKind::Float, Field(static_cast(default_temp))}, + {"system_prompt", AIParamKind::String, Field(String(default_system_prompt))}, + }; + } - String buildSystemPrompt(const ColumnsWithTypeAndName & arguments) const override + String buildSystemPrompt(const ColumnsWithTypeAndName &, const AIParams & params) const override { - if (arguments.size() > system_prompt_arg_idx) - { - String system_prompt(arguments[system_prompt_arg_idx].column->getDataAt(0)); - if (!system_prompt.empty()) - return system_prompt; - } - - return default_system_prompt; + return params.getString("system_prompt"); } String buildUserMessage(const ColumnsWithTypeAndName & arguments, size_t row) const override { - return String(arguments[prompt_arg_index].column->getDataAt(row)); + return String(arguments[0].column->getDataAt(row)); } }; @@ -81,20 +74,23 @@ REGISTER_FUNCTION(AiGenerate) Generates free-form text content from a prompt using an LLM provider. The function sends the prompt to the configured AI provider and returns the generated text. -An optional system prompt can be provided to guide the model's behavior (e.g. tone, format, role). -If no system prompt is given, the default system prompt is: `)" + String(default_system_prompt) + R"(` -Provider credentials and configuration are taken from the named collection specified by the `ai_function_credentials` setting. +Credentials (a named collection specifying the provider, model, endpoint, and optionally an API key) +are taken from the `credentials` key of the optional parameter map, or from the +`ai_function_text_default_credentials` setting when the map omits it. + +The optional parameter map may also set `system_prompt` (an instruction that guides the model's +behavior, e.g. tone, format, role), `temperature`, `max_tokens`, and `model`. If `system_prompt` is +not set, the default is: `)" + String(default_system_prompt) + R"(` )", - .syntax = "aiGenerate(prompt[, system_prompt[, temperature]])", + .syntax = "aiGenerate(prompt[, params])", .arguments = {{"prompt", "The user prompt or question to send to the model.", {"String"}}, - {"system_prompt", "Optional constant system-level instruction that guides the model's behavior (e.g. persona, output format), sent along with each prompt.", {"String"}}, - {"temperature", "Sampling temperature controlling randomness. Default: `0.7`.", {"Float64"}}}, + {"params", "Optional constant `Map(String, String)` of parameters. Function-specific keys: `temperature` (sampling temperature controlling randomness; default `0.7`), `max_tokens` (maximum output tokens per call; default `1024`), `system_prompt` (constant system-level instruction guiding the model's behavior; default a generic assistant prompt). The common parameters `credentials` and `model` also apply (see [AI Functions](/sql-reference/functions/ai-functions)).", {"Map(String, String)"}}}, .returned_value = {"The generated text response, or the default value for the column type (empty string) if the request failed and `ai_function_throw_on_error` is disabled.", {"String"}}, .examples - = {{"Simple question", "SELECT aiGenerate('What is 2 + 2? Reply with just the number.') SETTINGS ai_function_credentials = 'my_ai_credentials'", "4"}, - {"With system prompt", "SELECT aiGenerate('Explain ClickHouse', 'You are a database expert. Be concise.') SETTINGS ai_function_credentials = 'my_ai_credentials'", ""}, + = {{"Simple question", "SELECT aiGenerate('What is 2 + 2? Reply with just the number.')", "4"}, + {"With explicit credentials and system prompt", "SELECT aiGenerate('Explain ClickHouse', map('credentials', 'ai_text_credentials', 'system_prompt', 'You are a database expert. Be concise.'))", ""}, {"Summarize column values", "SELECT article_title, aiGenerate(concat('Summarize in one sentence: ', article_body)) AS summary FROM articles LIMIT 5", ""}}, .introduced_in = {26, 4}, .category = FunctionDocumentation::Category::AI}); diff --git a/src/Functions/aiTranslate.cpp b/src/Functions/aiTranslate.cpp index 42b07c40b397..d36fa50dcfdd 100644 --- a/src/Functions/aiTranslate.cpp +++ b/src/Functions/aiTranslate.cpp @@ -32,26 +32,26 @@ class FunctionAiTranslate final : public FunctionBaseAI {"target_language", static_cast(&isString), &isColumnConst, "const String"}, }; FunctionArgumentDescriptors optional_args{ - {"instructions", static_cast(&isString), &isColumnConst, "const String"}, - {"temperature", static_cast(&isNumber), &isColumnConst, "const Number"}, + {"params", static_cast(&FunctionBaseAI::isStringToStringMap), &isColumnConst, "const Map(String, String)"}, }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); - return wrapReturnTypeForNullablePrompt(arguments, prompt_arg_index, std::make_shared()); + return wrapReturnTypeForNullablePrompt(arguments, 0, std::make_shared()); } private: static constexpr float default_temp = 0.3f; - static constexpr size_t prompt_arg_index = 0; static constexpr size_t target_language_arg_index = 1; - static constexpr size_t instructions_arg_index = 2; - static constexpr size_t temp_arg_idx = 3; String functionName() const override { return name; } - float defaultTemperature() const override { return default_temp; } - size_t promptArgumentIndex() const override { return prompt_arg_index; } - size_t temperatureArgumentIndex() const override { return temp_arg_idx; } + AIParamSpecs functionParams() const override + { + return { + {"temperature", AIParamKind::Float, Field(static_cast(default_temp))}, + {"instructions", AIParamKind::String, Field(String(""))}, + }; + } void checkSanityBeforeExecuteImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override { @@ -60,23 +60,20 @@ class FunctionAiTranslate final : public FunctionBaseAI throw Exception(ErrorCodes::BAD_ARGUMENTS, "aiTranslate: 'target_language' must not be empty"); } - String buildSystemPrompt(const ColumnsWithTypeAndName & arguments) const override + String buildSystemPrompt(const ColumnsWithTypeAndName & arguments, const AIParams & params) const override { auto target_language = String(arguments[target_language_arg_index].column->getDataAt(0)); auto prompt = "Translate the following text into " + target_language + ". Return only the translation, nothing else."; - if (arguments.size() > instructions_arg_index) - { - auto instructions = String(arguments[instructions_arg_index].column->getDataAt(0)); - if (!instructions.empty()) - prompt += " Additional instructions: " + instructions; - } + auto instructions = params.getString("instructions"); + if (!instructions.empty()) + prompt += " Additional instructions: " + instructions; return prompt; } String buildUserMessage(const ColumnsWithTypeAndName & arguments, size_t row) const override { - return String(arguments[prompt_arg_index].column->getDataAt(row)); + return String(arguments[0].column->getDataAt(row)); } }; @@ -86,21 +83,22 @@ REGISTER_FUNCTION(AiTranslate) .description = R"( Translates the given text into the specified target language using an LLM provider. -Additional style or dialect instructions may be passed as a third argument (e.g. `'keep technical terms untranslated'`). +Additional style or dialect instructions may be passed via the `instructions` key of the parameter map (e.g. `'keep technical terms untranslated'`). -Provider credentials and configuration are taken from the named collection specified by the `ai_function_credentials` setting. +Credentials (a named collection specifying the provider, model, endpoint, and optionally an API key) +are taken from the `credentials` key of the optional parameter map, or from the +`ai_function_text_default_credentials` setting when the map omits it. )", - .syntax = "aiTranslate(text, target_language[, instructions[, temperature]])", + .syntax = "aiTranslate(text, target_language[, params])", .arguments = { {"text", "Text to translate.", {"String"}}, {"target_language", "Target language name or BCP-47 code (e.g. `'French'`, `'es-MX'`).", {"String"}}, - {"instructions", "Optional constant additional instructions for the translator.", {"String"}}, - {"temperature", "Sampling temperature controlling randomness. Default: `0.3`.", {"Float64"}}, + {"params", "Optional constant `Map(String, String)` of parameters. Function-specific keys: `temperature` (sampling temperature controlling randomness; default `0.3`), `max_tokens` (maximum output tokens per call; default `1024`), `instructions` (additional style or dialect instructions for the translator). The common parameters `credentials` and `model` also apply (see [AI Functions](/sql-reference/functions/ai-functions)).", {"Map(String, String)"}}, }, .returned_value = {"The translated text, or the default value for the column type (empty string) if the request failed and `ai_function_throw_on_error` is disabled.", {"String"}}, .examples = { - {"Translate to French", "SELECT aiTranslate('Hello, world!', 'French') SETTINGS ai_function_credentials = 'my_ai_credentials'", "Bonjour le monde!"}, - {"Translate to Japanese with style instructions", "SELECT aiTranslate(body, 'Japanese', 'Use polite form (desu/masu)') FROM articles LIMIT 5", ""}, + {"Translate to French", "SELECT aiTranslate('Hello, world!', 'French')", "Bonjour le monde!"}, + {"Translate to Japanese with style instructions", "SELECT aiTranslate(body, 'Japanese', map('instructions', 'Use polite form (desu/masu)')) FROM articles LIMIT 5", ""}, }, .introduced_in = {26, 4}, .category = FunctionDocumentation::Category::AI}); diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index df03a1b5541a..d86fcb579bdc 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -113,28 +113,24 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"CREATE NAMED COLLECTION ai_embed AS " f"provider = 'openai', " f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings', " - f"model = 'test-embed-model', " f"api_key = 'test-key'" ) instance.query( f"CREATE NAMED COLLECTION ai_embed_error AS " f"provider = 'openai', " f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_error', " - f"model = 'test-embed-model', " f"api_key = 'test-key'" ) instance.query( f"CREATE NAMED COLLECTION ai_embed_dup_index AS " f"provider = 'openai', " f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_dup_index', " - f"model = 'test-embed-model', " f"api_key = 'test-key'" ) instance.query( f"CREATE NAMED COLLECTION ai_embed_wrong_count AS " f"provider = 'openai', " f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_wrong_count', " - f"model = 'test-embed-model', " f"api_key = 'test-key'" ) @@ -155,8 +151,8 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: def test_generate_content_basic(started_cluster): result = instance.query( - "SELECT aiGenerate('hello world')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiGenerate('hello world', map('credentials', 'ai_mock'))", + settings=AI_SETTINGS, ) assert result.strip() == "hello world" @@ -165,19 +161,32 @@ def test_generate_content_multiple_rows(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('row1'), ('row2'), ('row3')") result = instance.query( - "SELECT aiGenerate(x) FROM test_input ORDER BY x", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiGenerate(x, map('credentials', 'ai_mock')) FROM test_input ORDER BY x", + settings=AI_SETTINGS, ) assert result.strip().split("\n") == ["row1", "row2", "row3"] +def test_generate_uses_text_default_credentials(started_cluster): + """End-to-end default-credentials path: with no `credentials` in the call, a real (non-empty) + request must actually use `ai_function_text_default_credentials`, not just resolve it for the + zero-row fast path. The mock echoes the input back, so a wiring bug would show up here.""" + instance.query("TRUNCATE TABLE test_input") + instance.query("INSERT INTO test_input VALUES ('row1'), ('row2')") + result = instance.query( + "SELECT aiGenerate(x) FROM test_input ORDER BY x", + settings={**AI_SETTINGS, "ai_function_text_default_credentials": "ai_mock"}, + ) + assert result.strip().split("\n") == ["row1", "row2"] + + def test_generate_content_profile_events(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('a'), ('b'), ('c')") qid = unique_query_id("gen_content_events") instance.query( - "SELECT aiGenerate(x) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiGenerate(x, map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) events = get_profile_events(qid) @@ -194,8 +203,8 @@ def test_generate_content_null_input(started_cluster): "INSERT INTO test_input_nullable VALUES (NULL), ('hello'), (NULL)" ) result = instance.query( - "SELECT aiGenerate(x) FROM test_input_nullable", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiGenerate(x, map('credentials', 'ai_mock')) FROM test_input_nullable", + settings=AI_SETTINGS, ) lines = result.strip().split("\n") assert lines.count("\\N") == 2 @@ -204,16 +213,16 @@ def test_generate_content_null_input(started_cluster): def test_generate_content_error_throw(started_cluster): error = instance.query_and_get_error( - "SELECT aiGenerate('hello')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_error"}, + "SELECT aiGenerate('hello', map('credentials', 'ai_error'))", + settings=AI_SETTINGS, ) assert "RECEIVED_ERROR_FROM_REMOTE_IO_SERVER" in error def test_generate_content_error_graceful(started_cluster): result = instance.query( - "SELECT aiGenerate('hello')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_error", "ai_function_throw_on_error": 0}, + "SELECT aiGenerate('hello', map('credentials', 'ai_error'))", + settings={**AI_SETTINGS, "ai_function_throw_on_error": 0}, ) assert result.strip() == "" @@ -230,8 +239,8 @@ def test_generate_without_api_key(started_cluster): """A named collection that omits `api_key` resolves and runs end-to-end, and the provider sends no `Authorization` header (rather than an empty/dummy token).""" result = instance.query( - "SELECT aiGenerate('no key here')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_no_key"}, + "SELECT aiGenerate('no key here', map('credentials', 'ai_no_key'))", + settings=AI_SETTINGS, ) assert result.strip() == "no key here" assert "authorization" not in last_request()["headers"] @@ -240,13 +249,83 @@ def test_generate_without_api_key(started_cluster): def test_generate_with_api_key_sends_auth_header(started_cluster): """A keyed collection forwards the key as a `Bearer` `Authorization` header.""" result = instance.query( - "SELECT aiGenerate('with key')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiGenerate('with key', map('credentials', 'ai_mock'))", + settings=AI_SETTINGS, ) assert result.strip() == "with key" assert last_request()["headers"].get("authorization") == "Bearer test-key" +def test_generate_model_override_with_default_credentials(started_cluster): + """`map('model', ...)` overrides the collection's model on the actual request, even when the + collection itself is selected via `ai_function_text_default_credentials` rather than the map.""" + instance.query( + "SELECT aiGenerate('hi', map('model', 'override-model'))", + settings={**AI_SETTINGS, "ai_function_text_default_credentials": "ai_mock"}, + ) + assert json.loads(last_request()["body"])["model"] == "override-model" + + +def test_embed_model_override_with_default_credentials(started_cluster): + """Same for aiEmbed: the required positional `model` argument sets the embedding model on the + request, with the collection selected via `ai_function_embedding_default_credentials`.""" + instance.query( + "SELECT aiEmbed('hi', 'override-embed-model')", + settings={**AI_SETTINGS, "ai_function_embedding_default_credentials": "ai_embed"}, + ) + assert json.loads(last_request()["body"])["model"] == "override-embed-model" + + +def test_generate_empty_model_override_with_default_credentials(started_cluster): + """An explicitly empty `model` in the params map overrides the collection's model: the resolver + honors presence, not content, so `map('model', '')` sends an empty model (letting an endpoint + pick one), even though the collection selected via the default setting defines `test-model`.""" + instance.query( + "SELECT aiGenerate('hi', map('model', ''))", + settings={**AI_SETTINGS, "ai_function_text_default_credentials": "ai_mock"}, + ) + assert json.loads(last_request()["body"])["model"] == "" + + +# Setting every credential/config key in the params map at once. `ai_mock` carries an api_key, +# `ai_no_key` does not, so the auth header proves which collection was actually contacted. +_ALL_PARAMS_QUERY = ( + "SELECT aiGenerate('hi', map(" + "'credentials', 'ai_mock', 'model', 'map-model', 'max_tokens', '7', " + "'temperature', '0.9', 'system_prompt', 'be terse'))" +) + + +def _assert_all_params_applied(): + req = last_request() + body = json.loads(req["body"]) + # `credentials` picked `ai_mock` (keyed) — proves the map won over any default setting. + assert req["headers"].get("authorization") == "Bearer test-key" + assert body["model"] == "map-model" + assert body["max_tokens"] == 7 + assert abs(body["temperature"] - 0.9) < 1e-4 + assert body["messages"][0]["role"] == "system" + assert body["messages"][0]["content"] == "be terse" + + +def test_generate_all_map_params_override_setting(started_cluster): + """Every param passed in the map overrides a default-credentials setting that points at a + different collection: `credentials` (proven via the auth header) plus `model` / `max_tokens` / + `temperature` / `system_prompt` all take effect.""" + instance.query( + _ALL_PARAMS_QUERY, + settings={**AI_SETTINGS, "ai_function_text_default_credentials": "ai_no_key"}, + ) + _assert_all_params_applied() + + +def test_generate_all_map_params_without_setting(started_cluster): + """The same map, with no default-credentials setting at all: the map supplies everything, + including `credentials`, and all keys take effect.""" + instance.query(_ALL_PARAMS_QUERY, settings=AI_SETTINGS) + _assert_all_params_applied() + + # --------------------------------------------------------------------------- # aiClassify # --------------------------------------------------------------------------- @@ -258,8 +337,8 @@ def test_classify_basic(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('I love this product!')") result = instance.query( - "SELECT aiClassify(x, ['positive', 'negative', 'neutral']) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiClassify(x, ['positive', 'negative', 'neutral'], map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, ) # Mock returns first enum value; postProcessResponse extracts "category" from JSON assert result.strip() == "positive" @@ -271,8 +350,8 @@ def test_classify_multiple_rows(started_cluster): "INSERT INTO test_input VALUES ('great'), ('terrible'), ('okay')" ) result = instance.query( - "SELECT aiClassify(x, ['positive', 'negative', 'neutral']) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiClassify(x, ['positive', 'negative', 'neutral'], map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, ) lines = result.strip().split("\n") # All rows get the first enum value from the mock @@ -285,8 +364,8 @@ def test_classify_profile_events(started_cluster): instance.query("INSERT INTO test_input VALUES ('a'), ('b')") qid = unique_query_id("classify_events") instance.query( - "SELECT aiClassify(x, ['cat_a', 'cat_b']) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiClassify(x, ['cat_a', 'cat_b'], map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) events = get_profile_events(qid) @@ -298,8 +377,8 @@ def test_classify_null_input(started_cluster): instance.query("TRUNCATE TABLE test_input_nullable") instance.query("INSERT INTO test_input_nullable VALUES (NULL), ('text')") result = instance.query( - "SELECT aiClassify(x, ['a', 'b']) FROM test_input_nullable", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiClassify(x, ['a', 'b'], map('credentials', 'ai_mock')) FROM test_input_nullable", + settings=AI_SETTINGS, ) lines = result.strip().split("\n") assert len(lines) == 2 @@ -318,8 +397,8 @@ def test_extract_simple_instruction(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('The price is $42.99')") result = instance.query( - "SELECT aiExtract(x, 'the price') FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiExtract(x, 'the price', map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, ) # Mock returns {"result": ""}, postProcess extracts the value assert result.strip() == "The price is $42.99" @@ -331,8 +410,8 @@ def test_extract_json_schema(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('John is 30 years old')") result = instance.query( - """SELECT aiExtract(x, '{"name": "person name", "age": "person age"}') FROM test_input""", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + """SELECT aiExtract(x, '{"name": "person name", "age": "person age"}', map('credentials', 'ai_mock')) FROM test_input""", + settings=AI_SETTINGS, ) # Mock returns {"name": "", "age": ""} # postProcessResponse returns raw JSON since there's no single "result" field @@ -346,8 +425,8 @@ def test_extract_multiple_rows(started_cluster): instance.query("INSERT INTO test_input VALUES ('text1'), ('text2'), ('text3')") qid = unique_query_id("extract_events") instance.query( - "SELECT aiExtract(x, 'main topic') FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiExtract(x, 'main topic', map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) events = get_profile_events(qid) @@ -359,8 +438,8 @@ def test_extract_null_input(started_cluster): instance.query("TRUNCATE TABLE test_input_nullable") instance.query("INSERT INTO test_input_nullable VALUES (NULL), ('some text')") result = instance.query( - "SELECT aiExtract(x, 'key info') FROM test_input_nullable", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiExtract(x, 'key info', map('credentials', 'ai_mock')) FROM test_input_nullable", + settings=AI_SETTINGS, ) lines = result.strip().split("\n") assert "\\N" in lines @@ -377,8 +456,8 @@ def test_translate_basic(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('Hello world')") result = instance.query( - "SELECT aiTranslate(x, 'French') FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiTranslate(x, 'French', map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, ) assert result.strip() == "Hello world" @@ -387,8 +466,8 @@ def test_translate_multiple_rows(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('one'), ('two'), ('three')") result = instance.query( - "SELECT aiTranslate(x, 'Spanish') FROM test_input ORDER BY x", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiTranslate(x, 'Spanish', map('credentials', 'ai_mock')) FROM test_input ORDER BY x", + settings=AI_SETTINGS, ) assert result.strip().split("\n") == ["one", "three", "two"] @@ -397,8 +476,8 @@ def test_translate_with_instructions(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('Hello')") result = instance.query( - "SELECT aiTranslate(x, 'German', 'Use formal tone') FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiTranslate(x, 'German', map('credentials', 'ai_mock', 'instructions', 'Use formal tone')) FROM test_input", + settings=AI_SETTINGS, ) assert result.strip() == "Hello" @@ -418,8 +497,8 @@ def test_translate_profile_events(started_cluster): instance.query("INSERT INTO test_input VALUES ('a'), ('b')") qid = unique_query_id("translate_events") instance.query( - "SELECT aiTranslate(x, 'Japanese') FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiTranslate(x, 'Japanese', map('credentials', 'ai_mock')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) events = get_profile_events(qid) @@ -431,8 +510,8 @@ def test_translate_null_input(started_cluster): instance.query("TRUNCATE TABLE test_input_nullable") instance.query("INSERT INTO test_input_nullable VALUES (NULL), ('hello')") result = instance.query( - "SELECT aiTranslate(x, 'French') FROM test_input_nullable", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_mock"}, + "SELECT aiTranslate(x, 'French', map('credentials', 'ai_mock')) FROM test_input_nullable", + settings=AI_SETTINGS, ) lines = result.strip().split("\n") assert "\\N" in lines @@ -455,8 +534,33 @@ def parse_embedding(s): def test_embed_basic(started_cluster): """Single-row aiEmbed returns an `Array(Float32)` of the model's native size.""" result = instance.query( - "SELECT aiEmbed('hello')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed('hello', 'test-embed-model', map('credentials', 'ai_embed'))", + settings=AI_SETTINGS, + ) + vec = parse_embedding(result) + assert len(vec) == 4 # DEFAULT_EMBED_DIM in mock server + assert any(v != 0.0 for v in vec) + + +def test_embed_rejects_model_in_named_collection(started_cluster): + """aiEmbed takes `model` as a positional argument and never reads it from the named collection. + A collection that defines `model` (e.g. the text collection `ai_mock`) is rejected rather than + silently ignored.""" + error = instance.query_and_get_error( + "SELECT aiEmbed('hello', 'test-embed-model', map('credentials', 'ai_mock'))", + settings=AI_SETTINGS, + ) + assert "BAD_ARGUMENTS" in error + assert "defines 'model'" in error + + +def test_embed_uses_embedding_default_credentials(started_cluster): + """End-to-end default-credentials path for embeddings: with no `credentials` in the call, a real + (non-empty) request must actually use `ai_function_embedding_default_credentials`. Confirms the + embedding default is applied on the request path, not only for the zero-row fast path.""" + result = instance.query( + "SELECT aiEmbed('hello', 'test-embed-model')", + settings={**AI_SETTINGS, "ai_function_embedding_default_credentials": "ai_embed"}, ) vec = parse_embedding(result) assert len(vec) == 4 # DEFAULT_EMBED_DIM in mock server @@ -468,8 +572,8 @@ def test_embed_multiple_rows(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('alpha'), ('beta'), ('gamma')") result = instance.query( - "SELECT aiEmbed(x) FROM test_input ORDER BY x", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input ORDER BY x", + settings=AI_SETTINGS, ) rows = [parse_embedding(line) for line in result.strip().split("\n")] assert len(rows) == 3 @@ -481,8 +585,8 @@ def test_embed_multiple_rows(started_cluster): def test_embed_with_dimensions(started_cluster): """The `dimensions` argument is forwarded to the provider and honored in the response.""" result = instance.query( - "SELECT aiEmbed('hello world', 16)", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed('hello world', 'test-embed-model', map('credentials', 'ai_embed', 'dimensions', '16'))", + settings=AI_SETTINGS, ) vec = parse_embedding(result) assert len(vec) == 16 @@ -496,8 +600,8 @@ def test_embed_null_and_empty_input(started_cluster): ) qid = unique_query_id("embed_null_empty") result = instance.query( - "SELECT aiEmbed(x) FROM test_input_nullable ORDER BY x NULLS FIRST", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input_nullable ORDER BY x NULLS FIRST", + settings=AI_SETTINGS, query_id=qid, ) rows = [parse_embedding(line) for line in result.strip().split("\n")] @@ -522,8 +626,8 @@ def test_embed_profile_events_token_accounting(started_cluster): instance.query("INSERT INTO test_input VALUES ('abc'), ('de'), ('fghi')") qid = unique_query_id("embed_tokens") instance.query( - "SELECT aiEmbed(x) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) events = get_profile_events(qid) @@ -542,8 +646,8 @@ def test_embed_batching(started_cluster): ) qid = unique_query_id("embed_batch") instance.query( - "SELECT aiEmbed(x) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed", "ai_function_embedding_max_batch_size": 2}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", + settings={**AI_SETTINGS, "ai_function_embedding_max_batch_size": 2}, query_id=qid, ) events = get_profile_events(qid) @@ -556,8 +660,8 @@ def test_embed_batching(started_cluster): def test_embed_error_throw(started_cluster): """By default, provider errors propagate as `RECEIVED_ERROR_FROM_REMOTE_IO_SERVER`.""" error = instance.query_and_get_error( - "SELECT aiEmbed('hello')", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed_error"}, + "SELECT aiEmbed('hello', 'test-embed-model', map('credentials', 'ai_embed_error'))", + settings=AI_SETTINGS, ) assert "RECEIVED_ERROR_FROM_REMOTE_IO_SERVER" in error @@ -567,9 +671,9 @@ def test_embed_error_graceful(started_cluster): instance.query("TRUNCATE TABLE test_input") instance.query("INSERT INTO test_input VALUES ('a'), ('b')") result = instance.query( - "SELECT aiEmbed(x) FROM test_input", + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed_error')) FROM test_input", settings={ - **AI_SETTINGS, "ai_function_credentials": "ai_embed_error", + **AI_SETTINGS, "ai_function_throw_on_error": 0, "ai_function_max_retries": 0, }, @@ -581,8 +685,8 @@ def test_embed_error_graceful(started_cluster): def test_embed_duplicate_index_rejected(started_cluster): """`OpenAIProvider::embed` rejects responses with duplicate `index` values.""" error = instance.query_and_get_error( - "SELECT aiEmbed(x) FROM (SELECT arrayJoin(['a', 'b']) AS x)", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed_dup_index", "ai_function_max_retries": 0}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed_dup_index')) FROM (SELECT arrayJoin(['a', 'b']) AS x)", + settings={**AI_SETTINGS, "ai_function_max_retries": 0}, ) assert "MALFORMED_AI_PROVIDER_RESPONSE" in error assert "duplicates" in error or "duplicate" in error.lower() @@ -591,8 +695,8 @@ def test_embed_duplicate_index_rejected(started_cluster): def test_embed_wrong_count_rejected(started_cluster): """`OpenAIProvider::embed` rejects responses whose `data` size != number of inputs.""" error = instance.query_and_get_error( - "SELECT aiEmbed(x) FROM (SELECT arrayJoin(['a', 'b']) AS x)", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed_wrong_count", "ai_function_max_retries": 0}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed_wrong_count')) FROM (SELECT arrayJoin(['a', 'b']) AS x)", + settings={**AI_SETTINGS, "ai_function_max_retries": 0}, ) assert "MALFORMED_AI_PROVIDER_RESPONSE" in error @@ -602,8 +706,8 @@ def test_embed_empty_input_table(started_cluster): instance.query("TRUNCATE TABLE test_input") qid = unique_query_id("embed_zero_rows") result = instance.query( - "SELECT aiEmbed(x) FROM test_input", - settings={**AI_SETTINGS, "ai_function_credentials": "ai_embed"}, + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", + settings=AI_SETTINGS, query_id=qid, ) assert result.strip() == "" @@ -622,9 +726,9 @@ def test_embed_quota_input_tokens_exceeded(started_cluster): # Each batch costs `sum(len(text))` input tokens. With batch_size=1 and rows # of length 5 ("row_0".."row_3"), the second batch pushes us over a 5-token cap. result = instance.query( - "SELECT aiEmbed(x) FROM test_input", + "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", settings={ - **AI_SETTINGS, "ai_function_credentials": "ai_embed", + **AI_SETTINGS, "ai_function_embedding_max_batch_size": 1, "ai_function_max_input_tokens_per_query": 5, "ai_function_throw_on_quota_exceeded": 0, diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 00ed5168be98..e9e8de287c69 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -1,12 +1,14 @@ -- Disabled by default -- Enabled after setting aiGenerate --- ai_function_credentials unset errors -- aiGenerate: too few arguments -- aiGenerate: too many arguments +-- Missing credentials (no default, no map) -- Named collection missing provider -- Named collection missing endpoint --- Named collection missing model +-- Named collection missing model (and none in map) +-- Model supplied via the parameter map resolves +0 -- Named collection without api_key resolves 0 -- Named collection without api_key reaches HTTP path @@ -23,25 +25,25 @@ result String -- Anthropic provider resolves 0 -- aiEmbed rejects anthropic provider --- Custom system prompt accepted -0 --- Temperature: Float32 -0 --- Temperature: Float64 -0 --- Temperature: zero -0 --- Temperature: integer literal -0 --- Temperature without system prompt --- Temperature without system prompt (integer) --- Non-constant system prompt --- Non-constant temperature --- Wrong type for system prompt (number instead of string) --- Wrong type for temperature (string instead of number) --- ai_function_credentials setting default -1 +-- Custom system prompt via map +0 +-- Temperature via map +0 +-- max_tokens and model via map +0 +-- Unknown parameter key rejected +-- Non-numeric temperature rejected +-- Non-integer max_tokens rejected +-- Negative max_tokens rejected +-- Out-of-range max_tokens rejected (exceeds Int64) +-- Overflowing max_tokens rejected (exceeds UInt64, must not wrap) +-- Duplicate map key rejected +-- Non-constant parameter map rejected +-- Wrong type for parameter argument (not a map) +-- Wrong map value type (Map(String, Float) not accepted) +-- Map in the prompt position rejected -- Setting defaults +ai_function_embedding_default_credentials ai_function_embedding_max_batch_size 100 ai_function_max_api_calls_per_query 0 ai_function_max_input_tokens_per_query 1000000 @@ -49,6 +51,7 @@ ai_function_max_output_tokens_per_query 500000 ai_function_max_retries 0 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 +ai_function_text_default_credentials ai_function_throw_on_error 1 ai_function_throw_on_quota_exceeded 1 allow_experimental_ai_functions 0 @@ -95,9 +98,15 @@ result String aiEmbed -- aiEmbed: too few arguments -- aiEmbed: too many arguments --- aiEmbed: non-constant dimensions --- aiEmbed: wrong type for dimensions (signed integer) --- aiEmbed: wrong type for dimensions (string) +-- aiEmbed: non-constant parameter map +-- aiEmbed: wrong type for parameter argument (not a map) +-- aiEmbed: wrong type for model argument (not a string) +-- aiEmbed: non-constant model argument +-- aiEmbed: model is a required positional argument +-- aiEmbed: model in the parameter map is rejected +-- aiEmbed: model in the named collection is rejected +-- aiEmbed: model supplied as a positional argument resolves +0 -- aiEmbed: return type result Array(Float32) -- aiEmbed: return type with dimensions @@ -114,14 +123,14 @@ result Array(Float32) result Array(Float32) -- aiEmbed: NULL input → [] 0 --- aiEmbed: DEFAULT survives INSERT (no server crash) +-- aiEmbed: DEFAULT survives INSERT (no exception) 1 0 --- aiGenerate: DEFAULT survives INSERT (no server crash) +-- aiGenerate: DEFAULT survives INSERT (no exception) 1 0 --- aiClassify: DEFAULT survives INSERT (no server crash) +-- aiClassify: DEFAULT survives INSERT (no exception) 1 0 --- aiExtract: DEFAULT survives INSERT (no server crash) +-- aiExtract: DEFAULT survives INSERT (no exception) 1 0 --- aiTranslate: DEFAULT survives INSERT (no server crash) +-- aiTranslate: DEFAULT survives INSERT (no exception) 1 0 -- Re-disabled blocks function diff --git a/tests/queries/0_stateless/03300_ai_functions.sql b/tests/queries/0_stateless/03300_ai_functions.sql index ef41fc03ad3f..8fcd77bde25c 100644 --- a/tests/queries/0_stateless/03300_ai_functions.sql +++ b/tests/queries/0_stateless/03300_ai_functions.sql @@ -5,10 +5,14 @@ -- ============================================================================= -- AI Functions Test Suite -- Tests argument validation, error handling, return types, settings behavior, --- and named collection resolution for `aiGenerate`. --- Credentials are resolved from the `ai_function_credentials` setting (the named collection --- name), not from a function argument. +-- and named collection resolution for the AI functions. -- All tests run without a real AI provider or API key. +-- +-- Signature: each function takes the per-row text first, then any +-- function-specific mandatory arguments, then an optional trailing +-- Map(String, String) of parameters (credentials, model, temperature, …). +-- Credentials come from the map's `credentials` key or, when absent, from +-- `ai_function_text_default_credentials` / `ai_function_embedding_default_credentials`. -- ============================================================================= -- Helper table: a String column with zero rows, used to test function behavior @@ -29,11 +33,6 @@ SET allow_experimental_ai_functions = 1; SELECT '-- Enabled after setting'; SELECT name FROM system.functions WHERE name = 'aiGenerate'; --- `ai_function_credentials` is unset by default: AI functions must raise a clear error --- rather than make an implicit outbound call. -SELECT '-- ai_function_credentials unset errors'; -SELECT aiGenerate('hello'); -- { serverError BAD_ARGUMENTS } - -- ============================================================================= -- 2. Argument count validation -- ============================================================================= @@ -42,10 +41,17 @@ SELECT '-- aiGenerate: too few arguments'; SELECT aiGenerate(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiGenerate: too many arguments'; -SELECT aiGenerate('a', 'b', 0.7, 'x'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT aiGenerate('a', map('credentials', 'c'), 'x'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -- ============================================================================= --- 3. Named collection: missing required fields +-- 3. Missing credentials +-- ============================================================================= + +SELECT '-- Missing credentials (no default, no map)'; +SELECT aiGenerate('hi'); -- { serverError BAD_ARGUMENTS } + +-- ============================================================================= +-- 4. Named collection: missing required fields -- ============================================================================= DROP NAMED COLLECTION IF EXISTS ai_no_provider; @@ -55,7 +61,7 @@ CREATE NAMED COLLECTION ai_no_provider AS api_key = 'fake-key'; SELECT '-- Named collection missing provider'; -SELECT aiGenerate('hi') SETTINGS ai_function_credentials = 'ai_no_provider'; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate('hi', map('credentials', 'ai_no_provider')); -- { serverError BAD_ARGUMENTS } DROP NAMED COLLECTION ai_no_provider; @@ -66,7 +72,7 @@ CREATE NAMED COLLECTION ai_no_endpoint AS api_key = 'fake-key'; SELECT '-- Named collection missing endpoint'; -SELECT aiGenerate('hi') SETTINGS ai_function_credentials = 'ai_no_endpoint'; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate('hi', map('credentials', 'ai_no_endpoint')); -- { serverError BAD_ARGUMENTS } DROP NAMED COLLECTION ai_no_endpoint; @@ -76,8 +82,11 @@ CREATE NAMED COLLECTION ai_no_model AS endpoint = 'http://localhost:1/v1/chat/completions', api_key = 'fake-key'; -SELECT '-- Named collection missing model'; -SELECT aiGenerate('hi') SETTINGS ai_function_credentials = 'ai_no_model'; -- { serverError BAD_ARGUMENTS } +SELECT '-- Named collection missing model (and none in map)'; +SELECT aiGenerate('hi', map('credentials', 'ai_no_model')); -- { serverError BAD_ARGUMENTS } + +SELECT '-- Model supplied via the parameter map resolves'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_no_model', 'model', 'test-model')) AS result FROM tab); DROP NAMED COLLECTION ai_no_model; @@ -91,7 +100,7 @@ CREATE NAMED COLLECTION ai_no_api_key AS model = 'test-model'; SELECT '-- Named collection without api_key resolves'; -SELECT count() FROM (SELECT aiGenerate(x) AS result FROM tab) SETTINGS ai_function_credentials = 'ai_no_api_key'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_no_api_key')) AS result FROM tab); -- Force the no-key path through provider construction and an actual HTTP request: -- `localhost:1` refuses the connection, `ai_function_throw_on_error = 0` swallows it, @@ -101,26 +110,24 @@ SELECT '-- Named collection without api_key reaches HTTP path'; DROP TABLE IF EXISTS _03300_no_api_key_in; CREATE TABLE _03300_no_api_key_in (x String) ENGINE = Memory; INSERT INTO _03300_no_api_key_in VALUES ('hello'); -SET ai_function_credentials = 'ai_no_api_key'; SET ai_function_throw_on_error = 0; SET ai_function_request_timeout_sec = 3; -SELECT length(aiGenerate(x)) FROM _03300_no_api_key_in; +SELECT length(aiGenerate(x, map('credentials', 'ai_no_api_key'))) FROM _03300_no_api_key_in; SET ai_function_throw_on_error = 1; SET ai_function_request_timeout_sec = 60; -SET ai_function_credentials = ''; DROP TABLE _03300_no_api_key_in; DROP NAMED COLLECTION ai_no_api_key; -- ============================================================================= --- 4. Named collection: nonexistent collection +-- 5. Named collection: nonexistent collection -- ============================================================================= SELECT '-- Nonexistent named collection'; -SELECT aiGenerate('hello') SETTINGS ai_function_credentials = 'nonexistent_collection_xyz'; -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +SELECT aiGenerate('hello', map('credentials', 'nonexistent_collection_xyz')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } -- ============================================================================= --- 5. Test collection for remaining tests +-- 6. Test collection + default credentials for remaining tests -- ============================================================================= DROP NAMED COLLECTION IF EXISTS ai_credentials; @@ -130,11 +137,18 @@ CREATE NAMED COLLECTION ai_credentials AS model = 'test-model', api_key = 'fake-key'; --- From here on, resolve credentials from this collection by default. -SET ai_function_credentials = 'ai_credentials'; +-- aiEmbed takes `model` as a positional argument, so its named collection must not define `model`. +DROP NAMED COLLECTION IF EXISTS ai_embed_credentials; +CREATE NAMED COLLECTION ai_embed_credentials AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/embeddings', + api_key = 'fake-key'; + +SET ai_function_text_default_credentials = 'ai_credentials'; +SET ai_function_embedding_default_credentials = 'ai_embed_credentials'; -- ============================================================================= --- 6. Return type verification +-- 7. Return type verification -- ============================================================================= SELECT '-- aiGenerate return type'; @@ -146,7 +160,7 @@ SELECT name, type FROM system.columns DROP TABLE IF EXISTS _03300_ret_content; -- ============================================================================= --- 7. NULL input propagation +-- 8. NULL input propagation -- ============================================================================= DROP TABLE IF EXISTS _03300_null_input; @@ -162,14 +176,14 @@ DROP TABLE IF EXISTS _03300_null_result; DROP TABLE IF EXISTS _03300_null_input; -- ============================================================================= --- 8. Empty string input: zero rows, should not error +-- 9. Empty string input: zero rows, should not error -- ============================================================================= SELECT '-- Empty string input accepted'; SELECT count() FROM (SELECT aiGenerate(x) AS result FROM tab); -- ============================================================================= --- 9. Unknown provider name +-- 10. Unknown provider name -- ============================================================================= DROP NAMED COLLECTION IF EXISTS ai_bad_provider; @@ -180,16 +194,25 @@ CREATE NAMED COLLECTION ai_bad_provider AS api_key = 'fake-key'; SELECT '-- Unknown provider name'; -SELECT aiGenerate('hi') SETTINGS ai_function_credentials = 'ai_bad_provider'; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate('hi', map('credentials', 'ai_bad_provider')); -- { serverError BAD_ARGUMENTS } SELECT '-- Unknown provider name on empty input'; -SELECT aiGenerate(x) FROM (SELECT '' AS x WHERE 0) SETTINGS ai_function_credentials = 'ai_bad_provider'; -- { serverError BAD_ARGUMENTS } -SELECT aiEmbed(x) FROM (SELECT '' AS x WHERE 0) SETTINGS ai_function_credentials = 'ai_bad_provider'; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate(x, map('credentials', 'ai_bad_provider')) FROM (SELECT '' AS x WHERE 0); -- { serverError BAD_ARGUMENTS } + +-- aiEmbed needs a `model`-free collection so the unknown-provider error (not the collection-model +-- check) is what rejects the call. +DROP NAMED COLLECTION IF EXISTS ai_bad_provider_embed; +CREATE NAMED COLLECTION ai_bad_provider_embed AS + provider = 'unknown_provider', + endpoint = 'http://localhost:1/v1/embeddings', + api_key = 'fake-key'; +SELECT aiEmbed(x, 'test-model', map('credentials', 'ai_bad_provider_embed')) FROM (SELECT '' AS x WHERE 0); -- { serverError BAD_ARGUMENTS } DROP NAMED COLLECTION ai_bad_provider; +DROP NAMED COLLECTION ai_bad_provider_embed; -- ============================================================================= --- 10. Provider name: anthropic +-- 11. Provider name: anthropic -- ============================================================================= DROP NAMED COLLECTION IF EXISTS ai_anthropic; @@ -200,62 +223,72 @@ CREATE NAMED COLLECTION ai_anthropic AS api_key = 'fake-key'; SELECT '-- Anthropic provider resolves'; -SELECT count() FROM (SELECT aiGenerate(x) AS result FROM tab) SETTINGS ai_function_credentials = 'ai_anthropic'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_anthropic')) AS result FROM tab); + +-- aiEmbed needs a `model`-free collection (it takes `model` as a positional argument). +DROP NAMED COLLECTION IF EXISTS ai_anthropic_embed; +CREATE NAMED COLLECTION ai_anthropic_embed AS + provider = 'anthropic', + endpoint = 'http://localhost:1/v1/messages', + api_key = 'fake-key'; SELECT '-- aiEmbed rejects anthropic provider'; -SELECT aiEmbed('hi') SETTINGS ai_function_credentials = 'ai_anthropic'; -- { serverError NOT_IMPLEMENTED } -SELECT aiEmbed(x) FROM (SELECT '' AS x WHERE 0) SETTINGS ai_function_credentials = 'ai_anthropic'; -- { serverError NOT_IMPLEMENTED } +SELECT aiEmbed('hi', 'claude-test', map('credentials', 'ai_anthropic_embed')); -- { serverError NOT_IMPLEMENTED } +SELECT aiEmbed(x, 'claude-test', map('credentials', 'ai_anthropic_embed')) FROM (SELECT '' AS x WHERE 0); -- { serverError NOT_IMPLEMENTED } DROP NAMED COLLECTION ai_anthropic; +DROP NAMED COLLECTION ai_anthropic_embed; -- ============================================================================= --- 11. Custom system prompt argument +-- 12. Parameter map: keys and validation -- ============================================================================= -SELECT '-- Custom system prompt accepted'; -SELECT count() FROM (SELECT aiGenerate(x, 'You are a pirate') AS result FROM tab); +SELECT '-- Custom system prompt via map'; +SELECT count() FROM (SELECT aiGenerate(x, map('system_prompt', 'You are a pirate')) AS result FROM tab); --- ============================================================================= --- 12. Temperature argument --- ============================================================================= +SELECT '-- Temperature via map'; +SELECT count() FROM (SELECT aiGenerate(x, map('temperature', '0.5')) AS result FROM tab); + +SELECT '-- max_tokens and model via map'; +SELECT count() FROM (SELECT aiGenerate(x, map('max_tokens', '128', 'model', 'other-model')) AS result FROM tab); + +SELECT '-- Unknown parameter key rejected'; +SELECT aiGenerate('hi', map('bogus', '1')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature: Float32'; -SELECT count() FROM (SELECT aiGenerate(x, 'system', toFloat32(0.5)) AS result FROM tab); +SELECT '-- Non-numeric temperature rejected'; +SELECT aiGenerate('hi', map('temperature', 'hot')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature: Float64'; -SELECT count() FROM (SELECT aiGenerate(x, 'system', 0.5) AS result FROM tab); +SELECT '-- Non-integer max_tokens rejected'; +SELECT aiGenerate('hi', map('max_tokens', '3.5')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature: zero'; -SELECT count() FROM (SELECT aiGenerate(x, 'system', toFloat32(0.0)) AS result FROM tab); +SELECT '-- Negative max_tokens rejected'; +SELECT aiGenerate('hi', map('max_tokens', '-1')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature: integer literal'; -SELECT count() FROM (SELECT aiGenerate(x, 'system', 1) AS result FROM tab); +SELECT '-- Out-of-range max_tokens rejected (exceeds Int64)'; +SELECT aiGenerate('hi', map('max_tokens', '18446744073709551615')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature without system prompt'; -SELECT aiGenerate(x, toFloat32(0.5)) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '-- Overflowing max_tokens rejected (exceeds UInt64, must not wrap)'; +SELECT aiGenerate('hi', map('max_tokens', '18446744073709551616')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Temperature without system prompt (integer)'; -SELECT aiGenerate(x, 1) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '-- Duplicate map key rejected'; +SELECT aiGenerate('hi', map('temperature', '0.1', 'temperature', '0.2')); -- { serverError BAD_ARGUMENTS } -SELECT '-- Non-constant system prompt'; -SELECT aiGenerate(x, x) FROM tab; -- { serverError ILLEGAL_COLUMN } +SELECT '-- Non-constant parameter map rejected'; +SELECT aiGenerate(x, map('credentials', x)) FROM tab; -- { serverError ILLEGAL_COLUMN } -SELECT '-- Non-constant temperature'; -SELECT aiGenerate(x, 'system', toFloat32(number)) FROM (SELECT x, 0 AS number FROM tab); -- { serverError ILLEGAL_COLUMN } +SELECT '-- Wrong type for parameter argument (not a map)'; +SELECT aiGenerate(x, 'notamap') FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT '-- Wrong type for system prompt (number instead of string)'; -SELECT aiGenerate(x, 42) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '-- Wrong map value type (Map(String, Float) not accepted)'; +SELECT aiGenerate(x, map('temperature', 0.5)) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT '-- Wrong type for temperature (string instead of number)'; -SELECT aiGenerate(x, 'system', 'hot') FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '-- Map in the prompt position rejected'; +SELECT aiGenerate(map('credentials', 'ai_credentials')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -- ============================================================================= -- 13. Setting types and defaults -- ============================================================================= -SELECT '-- ai_function_credentials setting default'; -SELECT default = '' FROM system.settings WHERE name = 'ai_function_credentials'; - SELECT '-- Setting defaults'; SELECT name, @@ -271,7 +304,9 @@ WHERE name IN ( 'ai_function_max_output_tokens_per_query', 'ai_function_max_api_calls_per_query', 'ai_function_throw_on_quota_exceeded', - 'ai_function_embedding_max_batch_size' + 'ai_function_embedding_max_batch_size', + 'ai_function_text_default_credentials', + 'ai_function_embedding_default_credentials' ) ORDER BY name; @@ -283,10 +318,11 @@ SELECT '-- aiClassify: registered'; SELECT name FROM system.functions WHERE name = 'aiClassify'; SELECT '-- aiClassify: too few arguments'; +SELECT aiClassify(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT aiClassify('hello'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiClassify: too many arguments'; -SELECT aiClassify('x', ['a', 'b'], 0.0, 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT aiClassify('x', ['a', 'b'], map('temperature', '0.0'), 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiClassify: non-constant categories'; SELECT aiClassify(x, [x]) FROM tab; -- { serverError ILLEGAL_COLUMN } @@ -312,7 +348,7 @@ SELECT '-- aiClassify: empty input executes'; SELECT count() FROM (SELECT aiClassify(x, ['a', 'b']) AS result FROM tab); SELECT '-- aiClassify: with temperature'; -SELECT count() FROM (SELECT aiClassify(x, ['a', 'b'], 0.0) AS result FROM tab); +SELECT count() FROM (SELECT aiClassify(x, ['a', 'b'], map('temperature', '0.0')) AS result FROM tab); -- ============================================================================= -- 15. aiExtract @@ -322,10 +358,11 @@ SELECT '-- aiExtract: registered'; SELECT name FROM system.functions WHERE name = 'aiExtract'; SELECT '-- aiExtract: too few arguments'; +SELECT aiExtract(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT aiExtract('hello'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiExtract: too many arguments'; -SELECT aiExtract('x', 'instr', 0.0, 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT aiExtract('x', 'instr', map('temperature', '0.0'), 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiExtract: non-constant instruction'; SELECT aiExtract(x, x) FROM tab; -- { serverError ILLEGAL_COLUMN } @@ -359,7 +396,7 @@ SELECT aiExtract('hi', ' {invalid'); -- { serverError BAD_ARGUMENTS } SELECT aiExtract('hi', '\n\t {invalid'); -- { serverError BAD_ARGUMENTS } SELECT '-- aiExtract: with temperature'; -SELECT count() FROM (SELECT aiExtract(x, 'main topic', 0.0) AS result FROM tab); +SELECT count() FROM (SELECT aiExtract(x, 'main topic', map('temperature', '0.0')) AS result FROM tab); -- ============================================================================= -- 16. aiTranslate @@ -369,10 +406,11 @@ SELECT '-- aiTranslate: registered'; SELECT name FROM system.functions WHERE name = 'aiTranslate'; SELECT '-- aiTranslate: too few arguments'; +SELECT aiTranslate(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT aiTranslate('hello'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiTranslate: too many arguments'; -SELECT aiTranslate('x', 'French', 'instr', 0.3, 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT aiTranslate('x', 'French', map('temperature', '0.3'), 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiTranslate: non-constant target language'; SELECT aiTranslate(x, x) FROM tab; -- { serverError ILLEGAL_COLUMN } @@ -390,7 +428,7 @@ SELECT name, type FROM system.columns DROP TABLE IF EXISTS _03300_ret_translate; SELECT '-- aiTranslate: with instructions and temperature'; -SELECT count() FROM (SELECT aiTranslate(x, 'French', 'keep proper nouns', 0.3) AS result FROM tab); +SELECT count() FROM (SELECT aiTranslate(x, 'French', map('instructions', 'keep proper nouns', 'temperature', '0.3')) AS result FROM tab); -- ============================================================================= -- 17. aiEmbed @@ -403,21 +441,41 @@ SELECT '-- aiEmbed: too few arguments'; SELECT aiEmbed(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT '-- aiEmbed: too many arguments'; -SELECT aiEmbed('x', 256, 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT aiEmbed('x', 'test-model', map('dimensions', '256'), 'extra'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT '-- aiEmbed: non-constant parameter map'; +SELECT aiEmbed(x, 'test-model', map('dimensions', toString(number))) FROM (SELECT x, 0 AS number FROM tab); -- { serverError ILLEGAL_COLUMN } + +SELECT '-- aiEmbed: wrong type for parameter argument (not a map)'; +SELECT aiEmbed(x, 'test-model', 256) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT '-- aiEmbed: wrong type for model argument (not a string)'; +SELECT aiEmbed(x, 256) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT '-- aiEmbed: non-constant model argument'; +SELECT aiEmbed(x, x) FROM tab; -- { serverError ILLEGAL_COLUMN } + +-- `model` is a required positional argument for aiEmbed (unlike the text functions, which read it +-- from the parameter map or the named collection). +SELECT '-- aiEmbed: model is a required positional argument'; +SELECT aiEmbed('hi'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT '-- aiEmbed: non-constant dimensions'; -SELECT aiEmbed(x, toUInt64(number)) FROM (SELECT x, 0 AS number FROM tab); -- { serverError ILLEGAL_COLUMN } +-- `model` in the parameter map is rejected: it is not a known map key for aiEmbed. +SELECT '-- aiEmbed: model in the parameter map is rejected'; +SELECT aiEmbed('hi', 'test-model', map('credentials', 'ai_embed_credentials', 'model', 'other-model')); -- { serverError BAD_ARGUMENTS } -SELECT '-- aiEmbed: wrong type for dimensions (signed integer)'; -SELECT aiEmbed(x, -1) FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +-- `model` defined in the named collection is rejected rather than silently ignored: aiEmbed never +-- reads `model` from the collection (ai_credentials defines `model`). +SELECT '-- aiEmbed: model in the named collection is rejected'; +SELECT aiEmbed('hi', 'test-model', map('credentials', 'ai_credentials')); -- { serverError BAD_ARGUMENTS } -SELECT '-- aiEmbed: wrong type for dimensions (string)'; -SELECT aiEmbed(x, '256') FROM tab; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '-- aiEmbed: model supplied as a positional argument resolves'; +SELECT count() FROM (SELECT aiEmbed(x, 'test-model', map('credentials', 'ai_embed_credentials')) AS result FROM tab); SELECT '-- aiEmbed: return type'; DROP TABLE IF EXISTS _03300_ret_embed; CREATE TABLE _03300_ret_embed ENGINE = Memory AS - SELECT aiEmbed(x) AS result FROM tab; + SELECT aiEmbed(x, 'test-model') AS result FROM tab; SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = '_03300_ret_embed'; DROP TABLE IF EXISTS _03300_ret_embed; @@ -425,24 +483,24 @@ DROP TABLE IF EXISTS _03300_ret_embed; SELECT '-- aiEmbed: return type with dimensions'; DROP TABLE IF EXISTS _03300_ret_embed_dim; CREATE TABLE _03300_ret_embed_dim ENGINE = Memory AS - SELECT aiEmbed(x, 256) AS result FROM tab; + SELECT aiEmbed(x, 'test-model', map('dimensions', '256')) AS result FROM tab; SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = '_03300_ret_embed_dim'; DROP TABLE IF EXISTS _03300_ret_embed_dim; SELECT '-- aiEmbed: empty input executes'; -SELECT count() FROM (SELECT aiEmbed(x) AS result FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'test-model') AS result FROM tab); SELECT '-- aiEmbed: empty input with dimensions'; -SELECT count() FROM (SELECT aiEmbed(x, 128) AS result FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'test-model', map('dimensions', '128')) AS result FROM tab); -- `dimensions` is a row-independent constant, so an out-of-range value must fail -- the query even when the source has zero rows. SELECT '-- aiEmbed: out-of-range dimensions on empty input'; -SELECT aiEmbed(x, 18446744073709551615) FROM (SELECT '' AS x WHERE 0); -- { serverError BAD_ARGUMENTS } +SELECT aiEmbed(x, 'test-model', map('dimensions', '18446744073709551615')) FROM (SELECT '' AS x WHERE 0); -- { serverError BAD_ARGUMENTS } SELECT '-- aiEmbed: nonexistent named collection'; -SELECT aiEmbed('hello') SETTINGS ai_function_credentials = 'nonexistent_collection_xyz'; -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +SELECT aiEmbed('hello', 'test-model', map('credentials', 'nonexistent_collection_xyz')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } SELECT '-- aiEmbed: batch size setting default'; SELECT default FROM system.settings WHERE name = 'ai_function_embedding_max_batch_size'; @@ -456,7 +514,7 @@ DROP TABLE IF EXISTS _03300_embed_null_out; CREATE TABLE _03300_embed_null_in (x Nullable(String)) ENGINE = Memory; INSERT INTO _03300_embed_null_in VALUES (NULL); CREATE TABLE _03300_embed_null_out ENGINE = Memory AS - SELECT aiEmbed(x) AS result FROM _03300_embed_null_in; + SELECT aiEmbed(x, 'test-model') AS result FROM _03300_embed_null_in; SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = '_03300_embed_null_out'; @@ -475,19 +533,19 @@ DROP TABLE IF EXISTS _03300_embed_null_in; SET ai_function_throw_on_error = 0; SET ai_function_request_timeout_sec = 3; -SELECT '-- aiEmbed: DEFAULT survives INSERT (no server crash)'; +SELECT '-- aiEmbed: DEFAULT survives INSERT (no exception)'; DROP TABLE IF EXISTS _03300_embed_default; CREATE TABLE _03300_embed_default ( id UInt32, doc String, - vector Array(Float32) DEFAULT aiEmbed(doc) + vector Array(Float32) DEFAULT aiEmbed(doc, 'test-model') ) ENGINE = MergeTree ORDER BY id; INSERT INTO _03300_embed_default (id, doc) VALUES (1, 'hello world'); SELECT id, length(vector) FROM _03300_embed_default; DROP TABLE _03300_embed_default; -SELECT '-- aiGenerate: DEFAULT survives INSERT (no server crash)'; +SELECT '-- aiGenerate: DEFAULT survives INSERT (no exception)'; DROP TABLE IF EXISTS _03300_generate_default; CREATE TABLE _03300_generate_default ( @@ -499,7 +557,7 @@ INSERT INTO _03300_generate_default (id, doc) VALUES (1, 'hello world'); SELECT id, length(summary) FROM _03300_generate_default; DROP TABLE _03300_generate_default; -SELECT '-- aiClassify: DEFAULT survives INSERT (no server crash)'; +SELECT '-- aiClassify: DEFAULT survives INSERT (no exception)'; DROP TABLE IF EXISTS _03300_classify_default; CREATE TABLE _03300_classify_default ( @@ -511,7 +569,7 @@ INSERT INTO _03300_classify_default (id, doc) VALUES (1, 'hello world'); SELECT id, length(label) FROM _03300_classify_default; DROP TABLE _03300_classify_default; -SELECT '-- aiExtract: DEFAULT survives INSERT (no server crash)'; +SELECT '-- aiExtract: DEFAULT survives INSERT (no exception)'; DROP TABLE IF EXISTS _03300_extract_default; CREATE TABLE _03300_extract_default ( @@ -523,7 +581,7 @@ INSERT INTO _03300_extract_default (id, doc) VALUES (1, 'hello world'); SELECT id, length(extracted) FROM _03300_extract_default; DROP TABLE _03300_extract_default; -SELECT '-- aiTranslate: DEFAULT survives INSERT (no server crash)'; +SELECT '-- aiTranslate: DEFAULT survives INSERT (no exception)'; DROP TABLE IF EXISTS _03300_translate_default; CREATE TABLE _03300_translate_default ( @@ -552,5 +610,8 @@ SET allow_experimental_ai_functions = 1; -- Cleanup -- ============================================================================= +SET ai_function_text_default_credentials = ''; +SET ai_function_embedding_default_credentials = ''; DROP TABLE IF EXISTS tab; DROP NAMED COLLECTION ai_credentials; +DROP NAMED COLLECTION ai_embed_credentials; diff --git a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.reference b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.reference index 07ef7826272b..587a6b9dec2d 100644 --- a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.reference +++ b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.reference @@ -2,6 +2,14 @@ ACCESS_DENIED ACCESS_DENIED ACCESS_DENIED ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +OK +OK +OK +OK OK OK OK diff --git a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh index da0eae5be3b2..85cd63785129 100755 --- a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh +++ b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh @@ -32,14 +32,37 @@ function check_access_both() { $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " SET allow_experimental_ai_functions = 1; - SET ai_function_credentials = '$collection_name'; + SELECT aiGenerate('hi', map('credentials', '$collection_name')) FORMAT Null; + SELECT 'SEP'; + SELECT aiEmbed('hi', 'test-model', map('credentials', '$collection_name')) FORMAT Null; + SELECT 'SEP'; + SELECT aiGenerate(x, map('credentials', '$collection_name')) FROM (SELECT '' AS x WHERE 0) FORMAT Null; + SELECT 'SEP'; + SELECT aiEmbed(x, 'test-model', map('credentials', '$collection_name')) FROM (SELECT '' AS x WHERE 0) FORMAT Null; + " 2>&1 | awk ' + /ACCESS_DENIED/ { denied = 1; next } + /^SEP$/ { print (denied ? "ACCESS_DENIED" : "OK"); denied = 0; next } + END { print (denied ? "ACCESS_DENIED" : "OK") } + ' +} + +# Same checks, but credentials are selected via the default-credentials settings instead of the +# `credentials` map key. This settings-based indirection is a separate, security-sensitive path +# (`ai_function_text_default_credentials` for text functions, `ai_function_embedding_default_credentials` +# for aiEmbed) that must enforce the same NAMED_COLLECTION grant. +function check_access_both_default() +{ + $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " + SET allow_experimental_ai_functions = 1; + SET ai_function_text_default_credentials = '$collection_name'; + SET ai_function_embedding_default_credentials = '$collection_name'; SELECT aiGenerate('hi') FORMAT Null; SELECT 'SEP'; - SELECT aiEmbed('hi') FORMAT Null; + SELECT aiEmbed('hi', 'test-model') FORMAT Null; SELECT 'SEP'; SELECT aiGenerate(x) FROM (SELECT '' AS x WHERE 0) FORMAT Null; SELECT 'SEP'; - SELECT aiEmbed(x) FROM (SELECT '' AS x WHERE 0) FORMAT Null; + SELECT aiEmbed(x, 'test-model') FROM (SELECT '' AS x WHERE 0) FORMAT Null; " 2>&1 | awk ' /ACCESS_DENIED/ { denied = 1; next } /^SEP$/ { print (denied ? "ACCESS_DENIED" : "OK"); denied = 0; next } @@ -49,13 +72,16 @@ function check_access_both() # Without NAMED COLLECTION grant: must fail with ACCESS_DENIED before any network call, # even on the zero-row queries (the access check must precede the empty-input fast path). +# Both credential-selection paths (explicit map key and default settings) must deny. check_access_both +check_access_both_default $CLICKHOUSE_CLIENT -q "GRANT NAMED COLLECTION ON $collection_name TO $user_name" -# With the grant: access check passes. The 1-row calls still fail (unreachable host), -# but the failure must not be ACCESS_DENIED. The 0-row calls now succeed cleanly. +# With the grant: access check passes for both paths. The 1-row calls still fail (unreachable +# host), but the failure must not be ACCESS_DENIED. The 0-row calls now succeed cleanly. check_access_both +check_access_both_default $CLICKHOUSE_CLIENT -q " DROP USER IF EXISTS $user_name; diff --git a/tests/queries/0_stateless/04492_ai_functions_default_credentials.reference b/tests/queries/0_stateless/04492_ai_functions_default_credentials.reference new file mode 100644 index 000000000000..b47308ea8539 --- /dev/null +++ b/tests/queries/0_stateless/04492_ai_functions_default_credentials.reference @@ -0,0 +1,12 @@ +-- No defaults: text function fails +-- No defaults: aiEmbed fails +-- Text default set: aiGenerate resolves via default +0 +-- Text default does not leak into aiEmbed +-- Embedding default set: aiEmbed resolves via default +0 +-- Embedding default does not leak into text functions +-- Map credentials override with no text default +0 +-- Map credentials override wins over default +0 diff --git a/tests/queries/0_stateless/04492_ai_functions_default_credentials.sql b/tests/queries/0_stateless/04492_ai_functions_default_credentials.sql new file mode 100644 index 000000000000..40771ff31993 --- /dev/null +++ b/tests/queries/0_stateless/04492_ai_functions_default_credentials.sql @@ -0,0 +1,78 @@ +-- Tags: no-parallel, no-replicated-database +-- no-parallel: creates and drops global named collections +-- no-replicated-database: named collections are server-global, not database-scoped + +-- ============================================================================= +-- Default-credentials resolution for AI functions. +-- The text functions (aiGenerate/aiClassify/aiExtract/aiTranslate) and aiEmbed +-- use separate default-credentials settings, because a chat-completions endpoint +-- differs from an embeddings one. A per-call `credentials` map key +-- overrides the default. All tests run without a real AI provider. +-- ============================================================================= + +SET allow_experimental_ai_functions = 1; + +DROP TABLE IF EXISTS tab; +CREATE TABLE tab (x String) ENGINE = Memory; + +DROP NAMED COLLECTION IF EXISTS ai_text_nc; +DROP NAMED COLLECTION IF EXISTS ai_embed_nc; +CREATE NAMED COLLECTION ai_text_nc AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/chat/completions', + model = 'chat-model', + api_key = 'fake-key'; +-- aiEmbed takes `model` as a positional argument, so its collection must not define `model`. +CREATE NAMED COLLECTION ai_embed_nc AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/embeddings', + api_key = 'fake-key'; + +-- Start with no defaults set: bare calls must fail with a clear error. +SET ai_function_text_default_credentials = ''; +SET ai_function_embedding_default_credentials = ''; + +SELECT '-- No defaults: text function fails'; +SELECT aiGenerate('hi'); -- { serverError BAD_ARGUMENTS } +SELECT '-- No defaults: aiEmbed fails'; +SELECT aiEmbed('hi', 'embed-model'); -- { serverError BAD_ARGUMENTS } + +-- Set only the text default. aiGenerate resolves; aiEmbed still has no default. +SET ai_function_text_default_credentials = 'ai_text_nc'; + +SELECT '-- Text default set: aiGenerate resolves via default'; +SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab); + +SELECT '-- Text default does not leak into aiEmbed'; +SELECT aiEmbed('hi', 'embed-model'); -- { serverError BAD_ARGUMENTS } + +-- Set only the embedding default (clear the text one). aiEmbed resolves; text fails. +SET ai_function_text_default_credentials = ''; +SET ai_function_embedding_default_credentials = 'ai_embed_nc'; + +-- aiEmbed requires `model` as a positional argument; credentials still come from the default setting. +SELECT '-- Embedding default set: aiEmbed resolves via default'; +SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab); + +SELECT '-- Embedding default does not leak into text functions'; +SELECT aiGenerate('hi'); -- { serverError BAD_ARGUMENTS } + +-- The per-call `credentials` map key overrides the default (and works with no default set). +SELECT '-- Map credentials override with no text default'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_text_nc')) AS r FROM tab); + +-- Map credentials override wins over a set default. `ai_embed_nc` has no `model`, so aiGenerate +-- passes it in the map. +SET ai_function_text_default_credentials = 'ai_text_nc'; +SELECT '-- Map credentials override wins over default'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_embed_nc', 'model', 'embed-model')) AS r FROM tab); + +-- ============================================================================= +-- Cleanup +-- ============================================================================= + +SET ai_function_text_default_credentials = ''; +SET ai_function_embedding_default_credentials = ''; +DROP NAMED COLLECTION ai_text_nc; +DROP NAMED COLLECTION ai_embed_nc; +DROP TABLE tab; From 6f47d1d8daa51acae810cdd8d461283dc8164a09 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 1 Aug 2026 00:07:41 +0000 Subject: [PATCH 73/86] Backport #108977 to 26.6: Fix out-of-bounds crash in ObjectStorageQueueSource::FileIterator::next --- src/Common/FailPoint.cpp | 1 + .../ObjectStorageQueueIFileMetadata.cpp | 14 ++- .../ObjectStorageQueueSource.cpp | 9 +- .../test_parallel_inserts.py | 119 ++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 71b6b731d871..9a9faf255860 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -70,6 +70,7 @@ static struct InitFiu ONCE(distributed_cache_fail_request_in_the_middle_of_request) \ ONCE(object_storage_queue_fail_commit_once) \ ONCE(object_storage_queue_fail_commit_after_success) \ + ONCE(object_storage_queue_skip_one_file_in_batch) \ ONCE(object_storage_queue_cancel_in_generate) \ ONCE(object_storage_queue_sleep_in_generate) \ ONCE(distributed_cache_fail_continue_request) \ diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp index de9c71c11cd4..5e8637c39390 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,11 @@ namespace ProfileEvents namespace DB { +namespace FailPoints +{ + extern const char object_storage_queue_skip_one_file_in_batch[]; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -321,7 +327,13 @@ std::optional ObjectStorageQueueIFileMetadata::prepareSetProcessingRequests(Coordination::Requests & requests, const std::string & processing_id) { std::unique_lock processing_lock(file_status->processing_lock, std::defer_lock); - if (!processing_lock.try_lock()) + bool processing_lock_acquired = processing_lock.try_lock(); + + /// Test-only: simulate the file being grabbed by another consumer (a processing-lock conflict). + /// ONCE, so it skips the first file after being enabled, exercising the batch compaction path. + fiu_do_on(FailPoints::object_storage_queue_skip_one_file_in_batch, { processing_lock_acquired = false; }); + + if (!processing_lock_acquired) { /// This is possible in case on the same server /// there are more than one S3(Azure)Queue table processing the same keeper path. diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index adcb519416d6..9e1fe34ce171 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -393,6 +393,9 @@ ObjectStorageQueueSource::FileIterator::next() if (num_successful_objects != new_batch.size()) { + /// file_metadatas is empty when the keeper tryMulti above failed and + /// cleared it (see the chassert below); only compact it when populated. + const bool compact_file_metadatas = !file_metadatas.empty(); size_t batch_i = 0; for (size_t i = 0; i < num_successful_objects; ++i, ++batch_i) { @@ -408,10 +411,12 @@ ObjectStorageQueueSource::FileIterator::next() } new_batch[i] = new_batch[batch_i]; - file_metadatas[i] = file_metadatas[batch_i]; + if (compact_file_metadatas) + file_metadatas[i] = file_metadatas[batch_i]; } new_batch.resize(num_successful_objects); - file_metadatas.resize(num_successful_objects); + if (compact_file_metadatas) + file_metadatas.resize(num_successful_objects); } chassert(file_metadatas.empty() || new_batch.size() == file_metadatas.size()); diff --git a/tests/integration/test_storage_s3_queue/test_parallel_inserts.py b/tests/integration/test_storage_s3_queue/test_parallel_inserts.py index aebb5ae66da8..13550042c54d 100644 --- a/tests/integration/test_storage_s3_queue/test_parallel_inserts.py +++ b/tests/integration/test_storage_s3_queue/test_parallel_inserts.py @@ -258,3 +258,122 @@ def get_new_parts_in_dst(): DROP TABLE {table_name}; """ ) + + +def test_batch_set_processing_failure_does_not_crash(started_cluster): + """Regression for the out-of-bounds crash in FileIterator::next. + + The unordered hash-ring batch path aborts the server when two things happen for one + batch: + 1) at least one file is non-processable, so num_successful_objects < new_batch.size() + and the compaction block runs; + 2) the keeper multi that sets the batch as processing fails, so file_metadatas is + cleared before that compaction runs. + The compaction then subscripted the now-empty file_metadatas. + + Both conditions are the ones that happen in production when several consumers share a + keeper path: another consumer grabs one file first (making it non-processable) and has + already created a processing node for another file in the batch (making this consumer's + multi fail). Here condition 2 is reproduced by creating a real processing node in keeper + for one of the batch files, so the engine's own keeper multi fails against it (no faked + keeper response). Condition 1 is reproduced with a small failpoint that marks the first + file of the batch non-processable through the same std::nullopt path the engine takes + when a file is already being processed elsewhere. + """ + node = started_cluster.instances["instance"] + + table_name = f"test_batch_set_processing_failure_{generate_random_string()}" + dst_table_name = f"{table_name}_dst" + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + + # A handful of files, all listed in a single batch. File 0 is forced non-processable by + # the failpoint; a different file gets a pre-created processing node so the multi fails. + files_to_generate = 10 + generate_random_files( + started_cluster, files_path, files_to_generate, start_ind=0, row_num=1 + ) + + create_table( + started_cluster, + node, + table_name, + "unordered", + files_path, + additional_settings={ + "keeper_path": keeper_path, + "enable_hash_ring_filtering": 1, + "s3queue_processing_threads_num": 1, + "s3queue_loading_retries": 100, + # Both conditions must land in the SAME batch, so pin the listing batch size instead + # of relying on the engine default (1000) happening to exceed files_to_generate. + "list_objects_batch_size": files_to_generate, + }, + ) + + # Pre-create a real processing node for one file (not file 0, which the failpoint skips) + # so the engine's keeper multi fails against it exactly as it would if another consumer + # had set that file as processing first. The node name is the SipHash64 of the file path, + # which is what ObjectStorageQueueIFileMetadata::getNodeName uses. + conflict_file = f"{files_path}/test_1.csv" + conflict_node = node.query(f"SELECT sipHash64('{conflict_file}')").strip() + zk = started_cluster.get_kazoo_client("zoo1") + zk.ensure_path(f"{keeper_path}/processing") + zk.create(f"{keeper_path}/processing/{conflict_node}", b"conflict") + + def batch_set_processing_failures(): + node.query("SELECT 1") # fails loudly if the server aborted + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + "SELECT value FROM system.events" + " WHERE event = 'ObjectStorageQueueFailedToBatchSetProcessing'" + ).strip() + or 0 + ) + + # system.events counts for the lifetime of the server process, which is shared with the + # other tests of this module, so snapshot it before anything can consume from this table + # (the MV below is what starts the streaming task) and later wait for an increase. An + # absolute "> 0" would already be satisfied by an earlier increment and the test would + # then delete the conflict node without ever exercising the failing batch. + failures_before = batch_set_processing_failures() + + node.query( + "SYSTEM ENABLE FAILPOINT object_storage_queue_skip_one_file_in_batch" + ) + try: + create_mv(node, table_name, dst_table_name) + + # Wait until the batch that hits both conditions has actually been attempted: the failed + # keeper multi against the pre-created processing node bumps this profile event. This is + # the batch where the server aborted without the fix, so observing it guarantees the + # fixed path was exercised (a delayed CI worker cannot skip it). The server must stay + # alive while we wait. + run_with_retry( + lambda x: x > failures_before, batch_set_processing_failures, retries=120 + ) + + # Remove the artificial conflict and confirm the queue keeps making progress after the + # failed batch (the iterator recovered rather than getting stuck or having crashed). + zk.delete(f"{keeper_path}/processing/{conflict_node}") + + def get_count(): + return int(node.query(f"SELECT count() FROM {dst_table_name}")) + + # All files except the one left in an in-memory Processing state by the aborted + # multi are processed; the point is that the server survived and the queue drains. + run_with_retry(lambda x: x >= files_to_generate - 1, get_count) + + # The server must still be alive and responsive. + assert node.query("SELECT 1").strip() == "1" + finally: + node.query( + "SYSTEM DISABLE FAILPOINT object_storage_queue_skip_one_file_in_batch" + ) + node.query( + f""" + DROP TABLE IF EXISTS {dst_table_name}; + DROP TABLE IF EXISTS {table_name}; + """ + ) From 6342a38b49678220baf111b2c1da9d95e4142f5f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 1 Aug 2026 03:08:35 +0000 Subject: [PATCH 74/86] Backport #109107 to 26.6: Stream .backup metadata with SAX instead of building a DOM tree --- src/Backups/BackupImpl.cpp | 261 +++++++++++------ src/Backups/BackupMetadataHandler.cpp | 100 +++++++ src/Backups/BackupMetadataHandler.h | 67 +++++ .../tests/gtest_backup_metadata_handler.cpp | 275 ++++++++++++++++++ ...04494_backup_metadata_round_trip.reference | 6 + .../04494_backup_metadata_round_trip.sh | 64 ++++ ...backup_metadata_version_overflow.reference | 0 .../04495_backup_metadata_version_overflow.sh | 29 ++ 8 files changed, 717 insertions(+), 85 deletions(-) create mode 100644 src/Backups/BackupMetadataHandler.cpp create mode 100644 src/Backups/BackupMetadataHandler.h create mode 100644 src/Backups/tests/gtest_backup_metadata_handler.cpp create mode 100644 tests/queries/0_stateless/04494_backup_metadata_round_trip.reference create mode 100755 tests/queries/0_stateless/04494_backup_metadata_round_trip.sh create mode 100644 tests/queries/0_stateless/04495_backup_metadata_version_overflow.reference create mode 100755 tests/queries/0_stateless/04495_backup_metadata_version_overflow.sh diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 3d1791a9126c..62016fafd5f5 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -27,8 +27,10 @@ #include #include #include -#include +#include +#include +#include #include @@ -530,8 +532,6 @@ void BackupImpl::readBackupMetadata() LOG_TRACE(log, "Backup {}: Reading metadata", backup_name_for_logging); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::BackupReadMetadataMicroseconds); - using namespace XMLUtils; - std::unique_ptr in; if (use_archive) { @@ -549,111 +549,202 @@ void BackupImpl::readBackupMetadata() String str; readStringUntilEOF(str, *in); - Poco::XML::DOMParser dom_parser; - Poco::AutoPtr config = dom_parser.parseMemory(str.data(), str.size()); - const Poco::XML::Node * config_root = getRootNode(config); - version = getInt(config_root, "version"); - if ((version < INITIAL_BACKUP_VERSION) || (version > CURRENT_BACKUP_VERSION)) + + num_files = 0; + total_size = 0; + num_entries = 0; + size_of_entries = 0; + + bool contents_seen = false; + + /// Strict parsers: reject trailing garbage / unknown boolean text (fail closed with BACKUP_DAMAGED) + /// instead of DB::parse's lenient truncation (e.g. 12x34 read as 12). + auto to_uint64 = [&](const String & value, const String & key) -> UInt64 + { + UInt64 result = 0; + const char * begin = value.data(); + const char * end = begin + value.size(); + auto [ptr, ec] = std::from_chars(begin, end, result); + if (ec != std::errc{} || ptr != end) + throw Exception( + ErrorCodes::BACKUP_DAMAGED, "Backup {}: Cannot parse <{}> value {}", backup_name_for_logging, key, quoteString(value)); + return result; + }; + auto to_bool = [&](const String & value, const String & key) -> bool + { + if (value == "true" || value == "1") + return true; + if (value == "false" || value == "0") + return false; throw Exception( - ErrorCodes::BACKUP_VERSION_NOT_SUPPORTED, "Backup {}: Version {} is not supported", backup_name_for_logging, version); + ErrorCodes::BACKUP_DAMAGED, "Backup {}: Cannot parse <{}> boolean value {}", backup_name_for_logging, key, quoteString(value)); + }; - timestamp = parse<::LocalDateTime>(getString(config_root, "timestamp")).to_time_t(); - uuid = parse(getString(config_root, "uuid")); + BackupMetadataHandler handler; - if (config_root->getNodeByPath("base_backup") && !base_backup_info) + handler.on_header = [&](const BackupMetadataHandler::Fields & h) { - base_backup_info = BackupInfo::fromString(getString(config_root, "base_backup")); + auto req = [&](const String & key) -> const String & + { + auto it = h.find(key); + if (it == h.end()) + throw Exception( + ErrorCodes::BACKUP_DAMAGED, "Backup {}: Cannot read <{}> from metadata", backup_name_for_logging, key); + return it->second; + }; - /// The marker is honored only when the base backup locator itself comes from the metadata: - /// if the locator was overridden with the `base_backup` setting, the override is used as is. - base_backup_copy_s3_credentials_from_backup = getBool(config_root, BASE_BACKUP_COPY_S3_CREDENTIALS_FROM_BACKUP, false); - } + /// Range-check the parsed UInt64 before narrowing to int: a value that fits in UInt64 but not in + /// int would otherwise wrap (e.g. 4294967298 -> 2) and pass the supported-range check. + const auto version_value = to_uint64(req("version"), "version"); + if ((version_value < INITIAL_BACKUP_VERSION) || (version_value > CURRENT_BACKUP_VERSION)) + throw Exception( + ErrorCodes::BACKUP_VERSION_NOT_SUPPORTED, "Backup {}: Version {} is not supported", backup_name_for_logging, version_value); + version = static_cast(version_value); - if (config_root->getNodeByPath("base_backup_uuid")) - base_backup_uuid = parse(getString(config_root, "base_backup_uuid")); + timestamp = parse<::LocalDateTime>(req("timestamp")).to_time_t(); + uuid = parse(req("uuid")); - if (config_root->getNodeByPath("original_endpoint")) - original_endpoint = getString(config_root, "original_endpoint"); - if (config_root->getNodeByPath("original_namespace")) - original_namespace = getString(config_root, "original_namespace"); + if (h.contains("base_backup") && !base_backup_info) + { + base_backup_info = BackupInfo::fromString(req("base_backup")); - num_files = 0; - total_size = 0; - num_entries = 0; - size_of_entries = 0; + /// The marker is honored only when the base backup locator itself comes from the metadata: + /// if the locator was overridden with the `base_backup` setting, the override is used as is. + auto it = h.find(BASE_BACKUP_COPY_S3_CREDENTIALS_FROM_BACKUP); + base_backup_copy_s3_credentials_from_backup + = (it != h.end()) && to_bool(it->second, BASE_BACKUP_COPY_S3_CREDENTIALS_FROM_BACKUP); + } + + if (h.contains("base_backup_uuid")) + base_backup_uuid = parse(req("base_backup_uuid")); + + if (h.contains("original_endpoint")) + original_endpoint = req("original_endpoint"); + if (h.contains("original_namespace")) + original_namespace = req("original_namespace"); - const auto * contents = config_root->getNodeByPath("contents"); - for (const Poco::XML::Node * child = contents->firstChild(); child; child = child->nextSibling()) + contents_seen = true; + }; + + /// `readBackupMetadata` runs under `mutex` (TSA_REQUIRES), and `on_file` is invoked synchronously from + /// `parseMemoryNP` below while that lock is held, so the guarded members are safe to touch here. TSA cannot + /// see through the lambda boundary, hence the explicit suppression. + handler.on_file = [&](const BackupMetadataHandler::Fields & f) TSA_NO_THREAD_SAFETY_ANALYSIS { - if (child->nodeName() == "file") + auto req = [&](const String & key) -> const String & { - const Poco::XML::Node * file_config = child; - BackupFileInfo info; - info.file_name = getString(file_config, "name"); - validateFileNameFromBackup(info.file_name, "name", backup_name_for_logging); - info.object_key = getString(file_config, "object_key", ""); - info.size = getUInt64(file_config, "size"); - if (info.size) - { - info.checksum = unhexChecksum(getString(file_config, "checksum")); - - bool use_base = getBool(file_config, "use_base", false); - info.base_size = getUInt64(file_config, "base_size", use_base ? info.size : 0); - if (info.base_size) - use_base = true; + auto it = f.find(key); + if (it == f.end()) + throw Exception( + ErrorCodes::BACKUP_DAMAGED, "Backup {}: Cannot read <{}> of a file from metadata", backup_name_for_logging, key); + return it->second; + }; + auto opt = [&](const String & key, const String & def) -> String + { + auto it = f.find(key); + return it == f.end() ? def : it->second; + }; + auto get_bool = [&](const String & key, bool def) + { + auto it = f.find(key); + return it == f.end() ? def : to_bool(it->second, key); + }; - if (info.base_size > info.size) - { - throw Exception( - ErrorCodes::BACKUP_DAMAGED, - "Backup {}: Base size must not be greater than the size of entry {}", - backup_name_for_logging, - quoteString(info.file_name)); - } + BackupFileInfo info; + info.file_name = req("name"); + validateFileNameFromBackup(info.file_name, "name", backup_name_for_logging); + info.object_key = opt("object_key", ""); + info.size = to_uint64(req("size"), "size"); + if (info.size) + { + info.checksum = unhexChecksum(req("checksum")); - if (use_base) - { - if (info.base_size == info.size) - info.base_checksum = info.checksum; - else - info.base_checksum = unhexChecksum(getString(file_config, "base_checksum")); - } + bool use_base = get_bool("use_base", false); + auto base_size_it = f.find("base_size"); + info.base_size = (base_size_it != f.end()) ? to_uint64(base_size_it->second, "base_size") : (use_base ? info.size : 0); + if (info.base_size) + use_base = true; - if (info.size > info.base_size) - { - info.data_file_name = getString(file_config, "data_file", info.file_name); - if (info.data_file_name != info.file_name) - validateFileNameFromBackup(info.data_file_name, "data_file", backup_name_for_logging); - } - info.encrypted_by_disk = getBool(file_config, "encrypted_by_disk", false); + if (info.base_size > info.size) + { + throw Exception( + ErrorCodes::BACKUP_DAMAGED, + "Backup {}: Base size must not be greater than the size of entry {}", + backup_name_for_logging, + quoteString(info.file_name)); } - file_names.emplace(info.file_name, std::pair{info.size, info.checksum}); - if (!info.object_key.empty()) + if (use_base) { - if (original_endpoint.empty() || original_namespace.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "In lightweight snapshot backup, the endpoint or namespace should be not empty. We cannot restore this file."); - - if (open_mode == OpenMode::READ) - lightweight_snapshot_reader = lightweight_snapshot_reader_creator(original_endpoint, original_namespace); - - file_object_keys.emplace(info.file_name, info.object_key); - lightweight_snapshot_file_infos.try_emplace(info.object_key, info); + if (info.base_size == info.size) + info.base_checksum = info.checksum; + else + info.base_checksum = unhexChecksum(req("base_checksum")); } - else if (info.size) - file_infos.try_emplace(std::pair{info.size, info.checksum}, info); - ++num_files; - total_size += info.size; - bool has_entry = !params.deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || info.data_file_name == info.file_name)); - if (has_entry) + if (info.size > info.base_size) { - ++num_entries; - size_of_entries += info.size - info.base_size; + info.data_file_name = opt("data_file", info.file_name); + if (info.data_file_name != info.file_name) + validateFileNameFromBackup(info.data_file_name, "data_file", backup_name_for_logging); } + info.encrypted_by_disk = get_bool("encrypted_by_disk", false); + } + + file_names.emplace(info.file_name, std::pair{info.size, info.checksum}); + if (!info.object_key.empty()) + { + if (original_endpoint.empty() || original_namespace.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "In lightweight snapshot backup, the endpoint or namespace should be not empty. We cannot restore this file."); + + if (open_mode == OpenMode::READ) + lightweight_snapshot_reader = lightweight_snapshot_reader_creator(original_endpoint, original_namespace); + + file_object_keys.emplace(info.file_name, info.object_key); + lightweight_snapshot_file_infos.try_emplace(info.object_key, info); } + else if (info.size) + file_infos.try_emplace(std::pair{info.size, info.checksum}, info); + + ++num_files; + total_size += info.size; + bool has_entry = !params.deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || info.data_file_name == info.file_name)); + if (has_entry) + { + ++num_entries; + size_of_entries += info.size - info.base_size; + } + }; + + Poco::XML::SAXParser xml_parser; + xml_parser.setContentHandler(&handler); + /// Keep the namespace prefix in the element name (the old DOM parser enabled this too). Without it a + /// prefixed element like arrives as local name "contents" and would be accepted as an + /// ordinary element; with it the handler sees "x:contents" and ignores it (writeBackupMetadata never + /// emits namespaces, so this only rejects hand-crafted manifests). + xml_parser.setFeature(Poco::XML::XMLReader::FEATURE_NAMESPACE_PREFIXES, true); + try + { + xml_parser.parseMemoryNP(str.data(), str.size()); + } + catch (...) + { + /// A callback exception captured earlier is the root cause; prefer it over a secondary XML parse + /// error that a callback failure may have led to. + if (handler.saved_exception) + std::rethrow_exception(handler.saved_exception); + throw; } + /// Callbacks must not throw through expat; a captured exception is rethrown here. + if (handler.saved_exception) + std::rethrow_exception(handler.saved_exception); + + /// A well-formed but incomplete manifest (no ) leaves the header unapplied - version/uuid + /// unset - and must be rejected instead of being treated as an empty backup. + if (!contents_seen) + throw Exception(ErrorCodes::BACKUP_DAMAGED, "Backup {}: Metadata has no ", backup_name_for_logging); + uncompressed_size = size_of_entries + str.size(); compressed_size = uncompressed_size; if (!use_archive) diff --git a/src/Backups/BackupMetadataHandler.cpp b/src/Backups/BackupMetadataHandler.cpp new file mode 100644 index 000000000000..e108b32c36a9 --- /dev/null +++ b/src/Backups/BackupMetadataHandler.cpp @@ -0,0 +1,100 @@ +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BACKUP_DAMAGED; +} + +void BackupMetadataHandler::startElement( + const Poco::XML::XMLString &, + const Poco::XML::XMLString & local_name, + const Poco::XML::XMLString & qname, + const Poco::XML::Attributes &) +{ + if (saved_exception) + return; + try + { + /// A scalar leaf (header leaf under the root, file leaf under /) must be text-only: + /// reject mixed content, which the SAX path would otherwise collapse to the last text run (turning a + /// damaged value into a valid one). + if ((path.size() == 2 && path[1] != "contents") + || (path.size() == 4 && path[1] == "contents" && path[2] == "file")) + throw Exception( + ErrorCodes::BACKUP_DAMAGED, "Backup metadata has a child element inside scalar field <{}>", path.back()); + + current_text.clear(); + const String & name = qname.empty() ? local_name : qname; + /// Gate callbacks by exact position so a misplaced / is ignored. `path` holds the + /// ancestors; directly under the root fires on_header (all header leaves collected by then). + if (name == "contents" && path.size() == 1) + { + /// A well-formed manifest has exactly one top-level . Reject a second one instead of + /// re-applying the header and appending another file list (writeBackupMetadata never emits two). + if (root_contents_seen) + throw Exception(ErrorCodes::BACKUP_DAMAGED, "Backup metadata has more than one top-level "); + root_contents_seen = true; + if (on_header) + on_header(header_fields); + } + else if (name == "file" && path.size() == 2 && path[1] == "contents") + file_fields.clear(); + path.push_back(name); + } + catch (...) + { + saved_exception = std::current_exception(); + } +} + +void BackupMetadataHandler::endElement( + const Poco::XML::XMLString &, + const Poco::XML::XMLString & local_name, + const Poco::XML::XMLString & qname) +{ + if (saved_exception) + return; + try + { + const String & name = qname.empty() ? local_name : qname; + /// On a closing tag `path.back() == name`. Gate by exact position (see startElement). + if (name == "file" && path.size() == 3 && path[1] == "contents") + { + if (on_file) + on_file(file_fields); + } + else if (path.size() == 2 && name != "contents") /// header leaf: / + header_fields.try_emplace(name, current_text); /// keep the first value, like the old DOM getNodeByPath + else if (path.size() == 4 && path[1] == "contents" && path[2] == "file") /// file leaf + file_fields.try_emplace(name, current_text); + current_text.clear(); + if (!path.empty()) + path.pop_back(); + } + catch (...) + { + saved_exception = std::current_exception(); + } +} + +void BackupMetadataHandler::characters(const Poco::XML::XMLChar ch[], int start, int length) +{ + if (saved_exception) + return; + try + { + current_text.append(ch + start, static_cast(length)); + } + catch (...) + { + saved_exception = std::current_exception(); + } +} + +} diff --git a/src/Backups/BackupMetadataHandler.h b/src/Backups/BackupMetadataHandler.h new file mode 100644 index 000000000000..28c83a9c369f --- /dev/null +++ b/src/Backups/BackupMetadataHandler.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ + +/// Streaming SAX handler that reads the `.backup` metadata document without materializing a DOM tree. +/// +/// The document has a fixed, shallow shape: +/// +/// ... other header elements ... +/// +/// ... +/// ... +/// +/// +/// +/// All header elements precede ``, so `on_header` is fired once `` opens - before any +/// `` is seen. Each `` fires `on_file` as soon as it closes, so files are processed one at a time +/// instead of being retained in a DOM tree. Callbacks may throw; the exception is captured into `saved_exception` +/// and must be rethrown by the caller after parsing finishes, because exceptions must not propagate through the +/// underlying expat callbacks. +class BackupMetadataHandler : public Poco::XML::DefaultHandler +{ +public: + using Fields = std::map; + + /// Fired once, when `` starts, with all top-level header elements collected. + std::function on_header; + /// Fired once per ``, when it closes, with the file's leaf elements. + std::function on_file; + + /// The first exception thrown by a callback, if any. The caller must rethrow it after parsing. + std::exception_ptr saved_exception; + + void startElement( + const Poco::XML::XMLString & uri, + const Poco::XML::XMLString & local_name, + const Poco::XML::XMLString & qname, + const Poco::XML::Attributes & attributes) override; + + void endElement( + const Poco::XML::XMLString & uri, + const Poco::XML::XMLString & local_name, + const Poco::XML::XMLString & qname) override; + + void characters(const Poco::XML::XMLChar ch[], int start, int length) override; + +private: + std::vector path; + String current_text; + Fields header_fields; + Fields file_fields; + bool root_contents_seen = false; +}; + +} diff --git a/src/Backups/tests/gtest_backup_metadata_handler.cpp b/src/Backups/tests/gtest_backup_metadata_handler.cpp new file mode 100644 index 000000000000..f43631f4914b --- /dev/null +++ b/src/Backups/tests/gtest_backup_metadata_handler.cpp @@ -0,0 +1,275 @@ +#include + +#include +#include +#include + +#include + +#include +#include + + +using namespace DB; + +namespace +{ + struct ParseResult + { + BackupMetadataHandler::Fields header; + std::vector files; + bool header_seen = false; + bool file_seen_before_header = false; + std::exception_ptr saved_exception; + }; + + /// Drives the handler over `xml` exactly like `BackupImpl::readBackupMetadata` does (default `SAXParser`, + /// `parseMemoryNP`), recording the header and per-file field maps. + ParseResult parse(const std::string & xml) + { + ParseResult result; + BackupMetadataHandler handler; + handler.on_header = [&](const BackupMetadataHandler::Fields & h) + { + result.header = h; + result.header_seen = true; + }; + handler.on_file = [&](const BackupMetadataHandler::Fields & f) + { + if (!result.header_seen) + result.file_seen_before_header = true; + result.files.push_back(f); + }; + + Poco::XML::SAXParser parser; + parser.setContentHandler(&handler); + /// Mirror BackupImpl::readBackupMetadata so prefixed elements keep their prefix in the name. + parser.setFeature(Poco::XML::XMLReader::FEATURE_NAMESPACE_PREFIXES, true); + parser.parseMemoryNP(xml.data(), xml.size()); + result.saved_exception = handler.saved_exception; + return result; + } + + /// A manifest with two files, mirroring the whitespace-free output of `writeBackupMetadata`. + const std::string two_files_xml = + "" + "2" + "2020-01-01 00:00:00" + "00000000-0000-0000-0000-000000000001" + "Disk('backups', 'base')" + "00000000-0000-0000-0000-000000000002" + "" + "" + "data/db/tbl/full.bin" + "100" + "0123456789abcdef0123456789abcdef" + "" + "" + "data/db/tbl/incremental.bin" + "200" + "fedcba9876543210fedcba9876543210" + "true" + "150" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "data/db/tbl/other.bin" + "true" + "" + "" + ""; +} + + +TEST(BackupMetadataHandler, ParsesHeaderFields) +{ + auto result = parse(two_files_xml); + + EXPECT_TRUE(result.header_seen); + EXPECT_EQ(result.header.at("version"), "2"); + EXPECT_EQ(result.header.at("timestamp"), "2020-01-01 00:00:00"); + EXPECT_EQ(result.header.at("uuid"), "00000000-0000-0000-0000-000000000001"); + EXPECT_EQ(result.header.at("base_backup"), "Disk('backups', 'base')"); + EXPECT_EQ(result.header.at("base_backup_uuid"), "00000000-0000-0000-0000-000000000002"); + /// `` is not a header leaf and must not leak into the header map. + EXPECT_EQ(result.header.count("contents"), 0u); +} + +TEST(BackupMetadataHandler, ParsesFilesInOrderWithAllLeafFields) +{ + auto result = parse(two_files_xml); + + ASSERT_EQ(result.files.size(), 2u); + + const auto & f0 = result.files[0]; + EXPECT_EQ(f0.at("name"), "data/db/tbl/full.bin"); + EXPECT_EQ(f0.at("size"), "100"); + EXPECT_EQ(f0.at("checksum"), "0123456789abcdef0123456789abcdef"); + /// Optional fields that were not present must be absent (not empty strings). + EXPECT_EQ(f0.count("use_base"), 0u); + EXPECT_EQ(f0.count("base_size"), 0u); + EXPECT_EQ(f0.count("data_file"), 0u); + + const auto & f1 = result.files[1]; + EXPECT_EQ(f1.at("name"), "data/db/tbl/incremental.bin"); + EXPECT_EQ(f1.at("size"), "200"); + EXPECT_EQ(f1.at("use_base"), "true"); + EXPECT_EQ(f1.at("base_size"), "150"); + EXPECT_EQ(f1.at("base_checksum"), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + EXPECT_EQ(f1.at("data_file"), "data/db/tbl/other.bin"); + EXPECT_EQ(f1.at("encrypted_by_disk"), "true"); +} + +TEST(BackupMetadataHandler, HeaderIsAppliedBeforeAnyFile) +{ + auto result = parse(two_files_xml); + EXPECT_FALSE(result.file_seen_before_header); +} + +TEST(BackupMetadataHandler, EmptyContentsStillAppliesHeader) +{ + auto result = parse("1"); + + EXPECT_TRUE(result.header_seen); + EXPECT_EQ(result.header.at("version"), "1"); + EXPECT_TRUE(result.files.empty()); +} + +TEST(BackupMetadataHandler, FieldMapIsResetBetweenFiles) +{ + /// The second file omits `checksum`; it must not inherit the first file's value. + auto result = parse( + "2" + "a1c1" + "b0" + ""); + + ASSERT_EQ(result.files.size(), 2u); + EXPECT_EQ(result.files[1].at("name"), "b"); + EXPECT_EQ(result.files[1].count("checksum"), 0u); +} + +TEST(BackupMetadataHandler, CapturesFileCallbackExceptionAndShortCircuits) +{ + BackupMetadataHandler handler; + int file_calls = 0; + handler.on_file = [&](const BackupMetadataHandler::Fields &) + { + ++file_calls; + throw std::runtime_error("boom"); + }; + + Poco::XML::SAXParser parser; + parser.setContentHandler(&handler); + /// The exception must NOT propagate through the expat-based parser. + EXPECT_NO_THROW(parser.parseMemoryNP(two_files_xml.data(), two_files_xml.size())); + + /// The first file threw; the second must be short-circuited. + EXPECT_EQ(file_calls, 1); + ASSERT_TRUE(handler.saved_exception); + EXPECT_THROW(std::rethrow_exception(handler.saved_exception), std::runtime_error); +} + +TEST(BackupMetadataHandler, HeaderCallbackExceptionShortCircuitsFiles) +{ + BackupMetadataHandler handler; + int file_calls = 0; + handler.on_header = [&](const BackupMetadataHandler::Fields &) { throw std::runtime_error("bad header"); }; + handler.on_file = [&](const BackupMetadataHandler::Fields &) { ++file_calls; }; + + Poco::XML::SAXParser parser; + parser.setContentHandler(&handler); + EXPECT_NO_THROW(parser.parseMemoryNP(two_files_xml.data(), two_files_xml.size())); + + EXPECT_EQ(file_calls, 0); + ASSERT_TRUE(handler.saved_exception); + EXPECT_THROW(std::rethrow_exception(handler.saved_exception), std::runtime_error); +} + +TEST(BackupMetadataHandler, MalformedXmlThrowsFromParser) +{ + BackupMetadataHandler handler; + Poco::XML::SAXParser parser; + parser.setContentHandler(&handler); + + /// A parse error (mismatched tags) is reported by the parser itself, not captured in saved_exception. + const std::string bad = "2"; + EXPECT_ANY_THROW(parser.parseMemoryNP(bad.data(), bad.size())); +} + +TEST(BackupMetadataHandler, MissingContentsDoesNotApplyHeader) +{ + /// Without a top-level , on_header must not fire (BackupImpl rejects this afterwards). + auto result = parse("100000000-0000-0000-0000-000000000001"); + EXPECT_FALSE(result.header_seen); + EXPECT_TRUE(result.files.empty()); + EXPECT_FALSE(result.saved_exception); +} + +TEST(BackupMetadataHandler, FileOutsideContentsIsIgnored) +{ + /// A that is not directly under must not be reported (callbacks are gated by path). + auto result = parse("n0"); + EXPECT_FALSE(result.header_seen); + EXPECT_TRUE(result.files.empty()); +} + +TEST(BackupMetadataHandler, DuplicateTopLevelContentsIsRejected) +{ + /// A second top-level must be rejected rather than re-applying the header / appending files. + auto result = parse( + "1"); + ASSERT_TRUE(result.saved_exception); + EXPECT_THROW(std::rethrow_exception(result.saved_exception), DB::Exception); +} + +TEST(BackupMetadataHandler, DuplicateHeaderFieldKeepsFirstValue) +{ + /// Duplicate scalar header fields keep the first value (matches the old DOM getNodeByPath behavior). + auto result = parse( + "9992" + "00000000-0000-0000-0000-000000000001"); + ASSERT_TRUE(result.header_seen); + EXPECT_EQ(result.header.at("version"), "999"); +} + +TEST(BackupMetadataHandler, DuplicateFileFieldKeepsFirstValue) +{ + auto result = parse( + "2" + "a12" + ""); + ASSERT_EQ(result.files.size(), 1u); + EXPECT_EQ(result.files[0].at("size"), "1"); +} + +TEST(BackupMetadataHandler, NamespacePrefixedContentsIsNotTreatedAsContents) +{ + /// With namespace prefixes enabled (as in readBackupMetadata) a prefixed keeps its + /// prefix in the element name, so it does not match "contents": the header is never applied and no + /// files are collected (readBackupMetadata then rejects the manifest as having no ). + auto result = parse( + "1" + "a1"); + EXPECT_FALSE(result.header_seen); + EXPECT_TRUE(result.files.empty()); +} + +TEST(BackupMetadataHandler, ChildElementInsideHeaderScalarIsRejected) +{ + /// Mixed content in a header scalar (42949672982) must be rejected, not collapse to "2". + auto result = parse( + "42949672982"); + ASSERT_TRUE(result.saved_exception); + EXPECT_THROW(std::rethrow_exception(result.saved_exception), DB::Exception); +} + +TEST(BackupMetadataHandler, ChildElementInsideFileScalarIsRejected) +{ + /// Mixed content in a file scalar (truefalse) must be rejected, not collapse to "false". + auto result = parse( + "2" + "a1" + "truefalse" + ""); + ASSERT_TRUE(result.saved_exception); + EXPECT_THROW(std::rethrow_exception(result.saved_exception), DB::Exception); +} diff --git a/tests/queries/0_stateless/04494_backup_metadata_round_trip.reference b/tests/queries/0_stateless/04494_backup_metadata_round_trip.reference new file mode 100644 index 000000000000..6ef8a9fe9585 --- /dev/null +++ b/tests/queries/0_stateless/04494_backup_metadata_round_trip.reference @@ -0,0 +1,6 @@ +BACKUP_CREATED +BACKUP_CREATED +RESTORED +RESTORED +full disk: OK +incremental: OK diff --git a/tests/queries/0_stateless/04494_backup_metadata_round_trip.sh b/tests/queries/0_stateless/04494_backup_metadata_round_trip.sh new file mode 100755 index 000000000000..71ec54e31d01 --- /dev/null +++ b/tests/queries/0_stateless/04494_backup_metadata_round_trip.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Round-trip test for backup metadata: the `.backup` manifest is written by +# `writeBackupMetadata` and read back by `readBackupMetadata`. A mis-parse of the +# manifest (file names, sizes, checksums, base-backup dedup fields) would either +# fail the RESTORE or restore wrong data, so restoring and comparing the data is a +# direct check of the read path. Covers a full backup and an incremental backup on +# top of it (base-backup dedup fields). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +full_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_full +incr_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_incr + +full_backup="Disk('backups', '$full_id')" +incr_backup="Disk('backups', '$incr_id')" + +# Deterministic data (no now()/rand()), several inserts so the backup lists many files across parts. +${CLICKHOUSE_CLIENT} -m --query " +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS expected_full; +DROP TABLE IF EXISTS src_from_full; +DROP TABLE IF EXISTS src_from_incr; + +CREATE TABLE src (k UInt64, s String, t DateTime) ENGINE = MergeTree ORDER BY k; +INSERT INTO src SELECT number, repeat('x', number % 17), toDateTime('2020-01-01 00:00:00') + number FROM numbers(0, 1000); +INSERT INTO src SELECT number, repeat('y', number % 13), toDateTime('2020-01-01 00:00:00') + number FROM numbers(1000, 500); + +-- Snapshot of the table as it is captured by the full backup. +CREATE TABLE expected_full ENGINE = MergeTree ORDER BY k AS SELECT * FROM src; +" + +# Full backup. +${CLICKHOUSE_CLIENT} --query "BACKUP TABLE ${CLICKHOUSE_DATABASE}.src TO $full_backup SETTINGS id='$full_id'" | grep -o "BACKUP_CREATED" + +# Add more data, then an incremental backup on top of the full one. +${CLICKHOUSE_CLIENT} --query "INSERT INTO src SELECT number, repeat('z', number % 11), toDateTime('2020-01-01 00:00:00') + number FROM numbers(2000, 300)" +${CLICKHOUSE_CLIENT} --query "BACKUP TABLE ${CLICKHOUSE_DATABASE}.src TO $incr_backup SETTINGS id='$incr_id', base_backup=$full_backup" | grep -o "BACKUP_CREATED" + +# Restore each backup into a separate table (this is what exercises readBackupMetadata). +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE ${CLICKHOUSE_DATABASE}.src AS ${CLICKHOUSE_DATABASE}.src_from_full FROM $full_backup" | grep -o "RESTORED" +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE ${CLICKHOUSE_DATABASE}.src AS ${CLICKHOUSE_DATABASE}.src_from_incr FROM $incr_backup" | grep -o "RESTORED" + +# Compare restored data against the expected state. `label: OK` iff the sets are equal. +compare() { + ${CLICKHOUSE_CLIENT} --query " + SELECT '$3: ' || if( + (SELECT count() FROM (SELECT * FROM $1 EXCEPT SELECT * FROM $2)) = 0 + AND (SELECT count() FROM (SELECT * FROM $2 EXCEPT SELECT * FROM $1)) = 0 + AND (SELECT count() FROM $1) = (SELECT count() FROM $2), + 'OK', 'MISMATCH')" +} + +# The full backup captured expected_full; the incremental captured the live src (with the extra rows). +compare "${CLICKHOUSE_DATABASE}.src_from_full" "${CLICKHOUSE_DATABASE}.expected_full" "full disk" +compare "${CLICKHOUSE_DATABASE}.src_from_incr" "${CLICKHOUSE_DATABASE}.src" "incremental" + +${CLICKHOUSE_CLIENT} -m --query " +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS expected_full; +DROP TABLE IF EXISTS src_from_full; +DROP TABLE IF EXISTS src_from_incr; +" diff --git a/tests/queries/0_stateless/04495_backup_metadata_version_overflow.reference b/tests/queries/0_stateless/04495_backup_metadata_version_overflow.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04495_backup_metadata_version_overflow.sh b/tests/queries/0_stateless/04495_backup_metadata_version_overflow.sh new file mode 100755 index 000000000000..5a8025adb3e1 --- /dev/null +++ b/tests/queries/0_stateless/04495_backup_metadata_version_overflow.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# A .backup manifest whose fits in UInt64 but not in int must be rejected, not silently +# narrowed past the supported-version check (e.g. 4294967298 must not wrap to 2 and be accepted). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tbl_ver_overflow" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE tbl_ver_overflow (id UInt64) ENGINE = MergeTree ORDER BY id" +${CLICKHOUSE_CLIENT} --query "INSERT INTO tbl_ver_overflow VALUES (1)" + +backups_disk_root=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.disks WHERE name='backups'" 2>/dev/null) +if [ -z "${backups_disk_root}" ]; then + echo "backups disk is not configured, skipping test" + exit 0 +fi + +bname="${CLICKHOUSE_TEST_UNIQUE_NAME}_ver" +${CLICKHOUSE_CLIENT} --query "BACKUP TABLE tbl_ver_overflow TO Disk('backups', '${bname}')" > /dev/null 2>&1 + +# 4294967298 = 2^32 + 2: fits in UInt64 but narrows to 2 as int, which would pass the range check. +sed -i "s|[0-9]*|4294967298|" "${backups_disk_root}/${bname}/.backup" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE tbl_ver_overflow" +${CLICKHOUSE_CLIENT} -m -q "RESTORE TABLE tbl_ver_overflow FROM Disk('backups', '${bname}'); -- { serverError BACKUP_VERSION_NOT_SUPPORTED }" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tbl_ver_overflow" +rm -rf "${backups_disk_root:?}/${bname}" 2>/dev/null || true From 75d69342fe014b699cfcb9ead26394095e29af6b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 1 Aug 2026 16:06:26 +0000 Subject: [PATCH 75/86] Backport #112839 to 26.6: Fix segfault and silent data corruption when appending a non-appendable format to a file --- src/Storages/StorageFile.cpp | 11 ++- ...04668_file_sink_no_append_format.reference | 25 +++++++ .../04668_file_sink_no_append_format.sh | 74 +++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04668_file_sink_no_append_format.reference create mode 100755 tests/queries/0_stateless/04668_file_sink_no_append_format.sh diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 643bef2f593e..55525120ed1b 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2131,6 +2131,15 @@ class StorageFileSink final : public SinkToStorage, WithContext /// In case of formats with prefixes if file is not empty we have already written prefix. bool do_not_write_prefix = naked_buffer->size(); const auto & settings = getContext()->getSettingsRef(); + + /// The size is re-checked here, per sink: `StorageFile::write` checks it once at query + /// start, and not at all when writing through a file descriptor or a partitioned path. + if (do_not_write_prefix + && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, getContext(), format_settings)) + throw Exception( + ErrorCodes::CANNOT_APPEND_TO_FILE, + "Data cannot be appended to {} because the {} format doesn't support appends", + use_table_fd ? "the given file descriptor" : ("file " + path), format_name); write_buf = wrapWriteBufferWithCompressionMethod( std::move(naked_buffer), compression_method, @@ -2636,7 +2645,7 @@ In [clickhouse-local](../../../operations/utilities/clickhouse-local.md) File en - Multiple `SELECT` queries can be performed concurrently, but `INSERT` queries will wait each other. - Supported creating new file by `INSERT` query. -- If file exists, `INSERT` would append new values in it. +- If file exists, `INSERT` would append new values in it, but only for formats that support appending. Formats that do not support it, such as `Avro`, `Arrow`, `JSON`, `Npy`, `ORC` and `Parquet`, reject an `INSERT` into a non-empty file with `CANNOT_APPEND_TO_FILE`. For a plain file path, use the `engine_file_truncate_on_insert` or `engine_file_allow_create_multiple_files` settings listed below instead; neither applies when writing through a file descriptor, where the caller owns the descriptor. - Not supported: - `ALTER` - `SELECT ... SAMPLE` diff --git a/tests/queries/0_stateless/04668_file_sink_no_append_format.reference b/tests/queries/0_stateless/04668_file_sink_no_append_format.reference new file mode 100644 index 000000000000..5fd7a12e2b0b --- /dev/null +++ b/tests/queries/0_stateless/04668_file_sink_no_append_format.reference @@ -0,0 +1,25 @@ +-- 1. no-append format, non-empty target, zero rows: rejected, target untouched +CANNOT_APPEND_TO_FILE +8 +-- 2. no-append format, non-empty target, with rows: rejected, target untouched +CANNOT_APPEND_TO_FILE +8 +-- 3. no-append format, empty target: still written, and readable back +7 +-- 4. append-supporting format, non-empty target: still appends +1 +9 +-- 5. partitioned write into an existing partition: rejected, first partition intact +CANNOT_APPEND_TO_FILE +3 +-- 6. Parquet, non-empty target: rejected too, the guard is not Avro-specific +CANNOT_APPEND_TO_FILE +-- 7. engine_file_truncate_on_insert still replaces the file +2 +-- 8. engine_file_allow_create_multiple_files still creates a new file +1 +2 +-- 9. settings-dependent checker is consulted: appendable config accepted, non-appendable rejected +2 +CANNOT_APPEND_TO_FILE +2 diff --git a/tests/queries/0_stateless/04668_file_sink_no_append_format.sh b/tests/queries/0_stateless/04668_file_sink_no_append_format.sh new file mode 100755 index 000000000000..445e5b8d175a --- /dev/null +++ b/tests/queries/0_stateless/04668_file_sink_no_append_format.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the Avro output format needs ENABLE_AVRO, which follows ENABLE_LIBRARIES=0 in the Fast test build. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR="${CLICKHOUSE_TMP:?}/04668_${CLICKHOUSE_DATABASE:?}" +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" + +# The fd route needs clickhouse-local: TableFunctionFile only accepts a numeric first +# argument when getApplicationType() == LOCAL. +local_query() { ${CLICKHOUSE_LOCAL} --query "$1"; } + +# Reports the error name when the insert is rejected, so a future unrelated failure +# cannot make the test pass vacuously. +insert_via_fd() { + local target="$1" format="$2" query="$3" + ${CLICKHOUSE_LOCAL} --query "INSERT INTO FUNCTION file(3, '${format}', 'c0 UInt8') ${query}" \ + 3>>"${target}" 2>&1 >/dev/null | grep -oF 'CANNOT_APPEND_TO_FILE' | head -n 1 +} + +echo '-- 1. no-append format, non-empty target, zero rows: rejected, target untouched' +printf 'PREBYTES' > "${WORK_DIR}/zero_rows.avro" +insert_via_fd "${WORK_DIR}/zero_rows.avro" Avro 'SELECT 1 WHERE 0' +stat --format=%s "${WORK_DIR}/zero_rows.avro" + +echo '-- 2. no-append format, non-empty target, with rows: rejected, target untouched' +printf 'PREBYTES' > "${WORK_DIR}/with_rows.avro" +insert_via_fd "${WORK_DIR}/with_rows.avro" Avro 'SELECT 1' +stat --format=%s "${WORK_DIR}/with_rows.avro" + +echo '-- 3. no-append format, empty target: still written, and readable back' +: > "${WORK_DIR}/empty.avro" +insert_via_fd "${WORK_DIR}/empty.avro" Avro 'SELECT 7' +local_query "SELECT c0 FROM file('${WORK_DIR}/empty.avro', 'Avro')" + +echo '-- 4. append-supporting format, non-empty target: still appends' +printf '{"c0":9}\n' > "${WORK_DIR}/append.jsonl" +insert_via_fd "${WORK_DIR}/append.jsonl" JSONEachRow 'SELECT 1' +local_query "SELECT c0 FROM file('${WORK_DIR}/append.jsonl', 'JSONEachRow') ORDER BY c0" + +echo '-- 5. partitioned write into an existing partition: rejected, first partition intact' +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/part_{_partition_id}.avro', 'Avro', 'c0 UInt8') PARTITION BY c0 SELECT 3" 2>&1 \ + | grep -oF 'CANNOT_APPEND_TO_FILE' | head -n 1 +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/part_{_partition_id}.avro', 'Avro', 'c0 UInt8') PARTITION BY c0 SELECT 3" 2>&1 \ + | grep -oF 'CANNOT_APPEND_TO_FILE' | head -n 1 +local_query "SELECT c0 FROM file('${WORK_DIR}/part_3.avro', 'Avro')" + +echo '-- 6. Parquet, non-empty target: rejected too, the guard is not Avro-specific' +printf 'PREBYTES' > "${WORK_DIR}/pq.parquet" +insert_via_fd "${WORK_DIR}/pq.parquet" Parquet 'SELECT 1' + +echo '-- 7. engine_file_truncate_on_insert still replaces the file' +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/truncate.avro', 'Avro', 'c0 UInt8') SELECT 1" +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/truncate.avro', 'Avro', 'c0 UInt8') SELECT 2 SETTINGS engine_file_truncate_on_insert = 1" +local_query "SELECT c0 FROM file('${WORK_DIR}/truncate.avro', 'Avro')" + +echo '-- 8. engine_file_allow_create_multiple_files still creates a new file' +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/multi.avro', 'Avro', 'c0 UInt8') SELECT 1" +local_query "INSERT INTO FUNCTION file('${WORK_DIR}/multi.avro', 'Avro', 'c0 UInt8') SELECT 2 SETTINGS engine_file_allow_create_multiple_files = 1" +local_query "SELECT c0 FROM file('${WORK_DIR}/multi*.avro', 'Avro') ORDER BY c0" + +echo '-- 9. settings-dependent checker is consulted: appendable config accepted, non-appendable rejected' +printf 'x\n' > "${WORK_DIR}/custom.txt" +insert_via_fd "${WORK_DIR}/custom.txt" CustomSeparated "SELECT 1 SETTINGS format_custom_result_after_delimiter = ''" +local_query "SELECT count() FROM file('${WORK_DIR}/custom.txt', 'LineAsString')" +printf 'x\n' > "${WORK_DIR}/custom_reject.txt" +insert_via_fd "${WORK_DIR}/custom_reject.txt" CustomSeparated "SELECT 1 SETTINGS format_custom_result_after_delimiter = 'END'" +stat --format=%s "${WORK_DIR}/custom_reject.txt" + +rm -rf "${WORK_DIR}" From a9a3954ae20cfa93e983d713fa1ba74b2ba66188 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 1 Aug 2026 18:06:37 +0000 Subject: [PATCH 76/86] Backport #110449 to 26.6: AI functions: send X-ClickHouse-AI-Function header from OpenAI provider --- .../sql-reference/functions/ai-functions.md | 2 +- src/Functions/AI/IAIProvider.h | 8 +++++++ src/Functions/AI/OpenAIProvider.cpp | 4 ++++ src/Functions/FunctionBaseAI.cpp | 1 + src/Functions/FunctionBaseAI.h | 2 -- src/Functions/aiClassify.cpp | 2 -- src/Functions/aiEmbed.cpp | 8 +++---- src/Functions/aiExtract.cpp | 2 -- src/Functions/aiGenerate.cpp | 2 -- src/Functions/aiTranslate.cpp | 2 -- tests/integration/test_ai_functions/test.py | 22 +++++++++++++++++++ 11 files changed, 39 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/functions/ai-functions.md b/docs/en/sql-reference/functions/ai-functions.md index 7bfc8c041887..170385b5e14b 100644 --- a/docs/en/sql-reference/functions/ai-functions.md +++ b/docs/en/sql-reference/functions/ai-functions.md @@ -79,7 +79,7 @@ SELECT aiGenerate('Bonjour', map('credentials', 'other_credentials')); ### Parameter map {#parameter-map} -Each function accepts an optional trailing `Map(String, String)` of parameters. All values are strings (quote numbers, e.g. `'0.2'`). Unknown keys are rejected. A key that is present overrides the corresponding named-collection value; a key that is absent falls back to the named collection (for `model`/`max_tokens`) or the built-in default. The exception is `aiEmbed`, which takes `model` as a required positional argument (`aiEmbed(text, model[, params])`) and errors if it is instead set in the parameter map or named collection. +Each function accepts an optional trailing `Map(String, String)` of parameters. All values are strings (quote numbers, e.g. `'0.2'`). Unknown keys are rejected. A key that is present overrides the corresponding named-collection value; a key that is absent falls back to the named collection (for `model`/`max_tokens`) or the built-in default. The exception is `aiEmbed`, which takes `model` as a required positional argument (`aiEmbed(text, model[, params])`) and errors if it is instead set in the parameter map or named collection. This is in order to enforce reproducible embeddings. The following parameters are common to all the AI functions: diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index 0d30a033fd2b..84e93ed227f9 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -66,6 +66,10 @@ struct AIRequest /// Maximum number of tokens the model may generate in its response. This is a per-request limit, not a per-query limit. UInt64 max_tokens = 0; + + /// SQL name of the AI function that produced this request (e.g. "aiGenerate"). + /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. + String function_name; }; /// Response from a single AI chat completion request. Returned by IAIProvider::call after parsing the provider's HTTP response. @@ -101,6 +105,10 @@ struct AIEmbeddingRequest /// Optional target dimensionality for the output vectors. 0 means use the model's native size. /// Supported by OpenAI's `text-embedding-3-*` models; providers that ignore it return the native size. UInt64 dimensions = 0; + + /// SQL name of the AI function that produced this request (e.g. "aiEmbed"). + /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. + String function_name; }; /// Response from a single embedding request. `embeddings` is aligned 1:1 with `AIEmbeddingRequest::inputs`. diff --git a/src/Functions/AI/OpenAIProvider.cpp b/src/Functions/AI/OpenAIProvider.cpp index 17875d06806b..e88bb6d970d5 100644 --- a/src/Functions/AI/OpenAIProvider.cpp +++ b/src/Functions/AI/OpenAIProvider.cpp @@ -95,6 +95,8 @@ AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTi http_request.setContentType("application/json"); if (!api_key.empty()) /// not all providers need API key http_request.set("Authorization", "Bearer " + api_key); + chassert(!ai_request.function_name.empty()); + http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); @@ -178,6 +180,8 @@ AIEmbeddingResponse OpenAIProvider::embed(const AIEmbeddingRequest & ai_embeddin http_request.setContentType("application/json"); if (!api_key.empty()) /// not all providers need API key http_request.set("Authorization", "Bearer " + api_key); + chassert(!ai_embedding_request.function_name.empty()); + http_request.set("X-ClickHouse-AI-Function", ai_embedding_request.function_name); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 050a11eed898..bc972dea744a 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -431,6 +431,7 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, ai_request.model = model; ai_request.temperature = temperature; ai_request.max_tokens = max_tokens; + ai_request.function_name = getName(); /// update api_calls/quotas before call so failed calls are still added to total ++total_api_calls; diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index 8aa54f659e8d..de5ebb5dd07c 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -142,8 +142,6 @@ class FunctionBaseAI : public IFunction ContextPtr context; ContextPtr getContext() const { return context; } - virtual String functionName() const = 0; - /// Function-specific parameters accepted in the trailing `Map(String, String)` argument, on top /// of `commonParams`. Each entry carries its own default (or is required). Default: none. virtual AIParamSpecs functionParams() const { return {}; } diff --git a/src/Functions/aiClassify.cpp b/src/Functions/aiClassify.cpp index f54a3405894e..49eeeb5c7a57 100644 --- a/src/Functions/aiClassify.cpp +++ b/src/Functions/aiClassify.cpp @@ -62,8 +62,6 @@ class FunctionAiClassify final : public FunctionBaseAI static constexpr float default_temp = 0.0f; static constexpr size_t categories_arg_index = 1; - String functionName() const override { return name; } - AIParamSpecs functionParams() const override { return {{"temperature", AIParamKind::Float, Field(static_cast(default_temp))}}; diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index efe14972af98..5d54732639cb 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -99,7 +99,6 @@ class FunctionAiEmbed final : public IFunction { FunctionArgumentDescriptors mandatory_args{ {"text", static_cast(&FunctionBaseAI::isStringOrNullableString), nullptr, "String or Nullable(String)"}, - /// `model` must be a plain (non-nullable) `String`; constness is enforced by the column validator. {"model", static_cast(&isString), &isColumnConst, "const String"}, }; FunctionArgumentDescriptors optional_args{ @@ -110,9 +109,7 @@ class FunctionAiEmbed final : public IFunction return std::make_shared(std::make_shared()); } - /// Parameters accepted in the optional trailing `Map(String, String)` argument. `aiEmbed` does not - /// inherit `FunctionBaseAI`, so it declares its own spec (no `max_tokens`, which embeddings do not - /// use; no `model`, which is a required positional argument for `aiEmbed`). + /// Parameters accepted in the optional trailing `Map(String, String)` argument. static AIParamSpecs embeddingParams() { return { @@ -211,6 +208,7 @@ class FunctionAiEmbed final : public IFunction AIEmbeddingRequest ai_embedding_request; ai_embedding_request.model = model; ai_embedding_request.dimensions = dimensions; + ai_embedding_request.function_name = getName(); ai_embedding_request.inputs.reserve(batch_end - batch_start); for (size_t k = batch_start; k < batch_end; ++k) @@ -320,7 +318,7 @@ from a chat one. The `model` is a required positional argument (a constant `String`). Unlike the text functions, `aiEmbed` does not read `model` from the named collection or the parameter map. A named collection -that defines `model` is rejected rather than silently ignored. +that defines `model` is rejected. The optional `dimensions` parameter, when supported by the model (e.g. OpenAI's `text-embedding-3-*`), requests a vector of the given size; otherwise the model's native size is returned. diff --git a/src/Functions/aiExtract.cpp b/src/Functions/aiExtract.cpp index 6941aa9744e2..97a5e06f3dbe 100644 --- a/src/Functions/aiExtract.cpp +++ b/src/Functions/aiExtract.cpp @@ -50,8 +50,6 @@ class FunctionAiExtract final : public FunctionBaseAI static constexpr float default_temp = 0.0f; static constexpr size_t instruction_arg_index = 1; - String functionName() const override { return name; } - AIParamSpecs functionParams() const override { return {{"temperature", AIParamKind::Float, Field(static_cast(default_temp))}}; diff --git a/src/Functions/aiGenerate.cpp b/src/Functions/aiGenerate.cpp index 1ab75410da1e..7c82ae05fb26 100644 --- a/src/Functions/aiGenerate.cpp +++ b/src/Functions/aiGenerate.cpp @@ -44,8 +44,6 @@ class FunctionAiGenerate final : public FunctionBaseAI private: static constexpr float default_temp = 0.7f; - String functionName() const override { return name; } - AIParamSpecs functionParams() const override { return { diff --git a/src/Functions/aiTranslate.cpp b/src/Functions/aiTranslate.cpp index d36fa50dcfdd..9ff587a3c1a4 100644 --- a/src/Functions/aiTranslate.cpp +++ b/src/Functions/aiTranslate.cpp @@ -43,8 +43,6 @@ class FunctionAiTranslate final : public FunctionBaseAI static constexpr float default_temp = 0.3f; static constexpr size_t target_language_arg_index = 1; - String functionName() const override { return name; } - AIParamSpecs functionParams() const override { return { diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 8b1b5f633bdd..406407b8ab72 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -932,6 +932,28 @@ def test_generate_retry_respects_api_call_quota(started_cluster): assert int(events["rows_skipped"]) == 1 +def test_function_name_header(started_cluster): + """The OpenAI provider tags every request with an `X-ClickHouse-AI-Function` header carrying the + SQL name of the calling function, so the upstream endpoint can tell which function made the call. + Covers the chat path (aiGenerate/aiClassify/aiExtract/aiTranslate) and the embedding path (aiEmbed).""" + cases = [ + ("aiGenerate", "SELECT aiGenerate('hi', map('credentials', 'ai_mock'))"), + ( + "aiClassify", + "SELECT aiClassify('hi', ['a', 'b'], map('credentials', 'ai_mock'))", + ), + ("aiExtract", "SELECT aiExtract('hi', 'the price', map('credentials', 'ai_mock'))"), + ("aiTranslate", "SELECT aiTranslate('hi', 'French', map('credentials', 'ai_mock'))"), + ( + "aiEmbed", + "SELECT aiEmbed('hi', 'test-embed-model', map('credentials', 'ai_embed'))", + ), + ] + for name, query in cases: + instance.query(query, settings=AI_SETTINGS) + assert last_request()["headers"].get("x-clickhouse-ai-function") == name + + def test_embed_retry_respects_api_call_quota(started_cluster): """The embedding path enforces the same per-attempt API-call quota: a retriable HTTP 500 is not retried past `ai_function_max_api_calls_per_query`.""" From ca609929a6373b9f2d423341c91f5afec21070ab Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 1 Aug 2026 23:49:07 +0000 Subject: [PATCH 77/86] Backport #109178 to 26.6: Preserve original key order in bucketed Map serialization --- .../Serializations/ISerialization.cpp | 2 + src/DataTypes/Serializations/ISerialization.h | 15 + .../Serializations/SerializationMap.cpp | 247 ++++++++++++++++- .../Serializations/SerializationMap.h | 6 +- .../SerializationMapKeysOrValues.cpp | 148 +++++++++- .../MergeTree/MergeTreeReaderCompact.cpp | 31 ++- .../MergeTree/MergeTreeReaderCompact.h | 3 +- .../MergeTree/MergeTreeReaderWide.cpp | 7 + .../test_bucketed_map_order.py | 259 ++++++++++++++++++ ...993_map_subcolumns_small_compact.reference | 96 +++---- .../03994_map_subcolumns_small_wide.reference | 96 +++---- ..._serialization_version_and_merge.reference | 30 +- ...change_serialization_version_and_merge.sql | 2 + .../04000_map_subcolumns_prewhere.reference | 36 +-- ...p_buckets_key_order_preservation.reference | 32 +++ ...409_map_buckets_key_order_preservation.sql | 244 +++++++++++++++++ ..._buckets_order_by_and_comparison.reference | 30 ++ ...10_map_buckets_order_by_and_comparison.sql | 201 ++++++++++++++ ..._buckets_parts_splitter_row_drop.reference | 18 ++ ...11_map_buckets_parts_splitter_row_drop.sql | 205 ++++++++++++++ 20 files changed, 1560 insertions(+), 148 deletions(-) create mode 100644 tests/integration/test_backward_compatibility/test_bucketed_map_order.py create mode 100644 tests/queries/0_stateless/04409_map_buckets_key_order_preservation.reference create mode 100644 tests/queries/0_stateless/04409_map_buckets_key_order_preservation.sql create mode 100644 tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.reference create mode 100644 tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.sql create mode 100644 tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.reference create mode 100644 tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.sql diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 7fd205b6ae57..7082129870fc 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -369,6 +369,8 @@ String getNameForSubstreamPath( stream_name += "." + std::to_string(it->bucket); else if (it->type == SubstreamType::MapBucketsInfo) stream_name += ".buckets_info"; + else if (it->type == SubstreamType::MapBucketIndexes) + stream_name += ".bucket_indexes"; else if (it->type == SubstreamType::ObjectSharedDataStructure) stream_name += ".structure"; else if (it->type == SubstreamType::ObjectSharedDataStructurePrefix) diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 8286988f9b9e..8e7a539cad3c 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -261,6 +261,7 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro Bucket, MapBucketsInfo, + MapBucketIndexes, Regular, }; @@ -351,6 +352,13 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro /// Type of MergeTree data part we serialize/deserialize data from if any. MergeTreeDataPartType data_part_type = MergeTreeDataPartType::Unknown; + /// Callback to check whether a specific substream exists in the current data part. + /// Used during enumeration to skip substreams that were introduced after the part + /// was written (e.g. MapBucketIndexes in old bucketed Map parts). + /// When not set, all substreams are enumerated unconditionally. + using CheckStreamExistsCallback = std::function; + CheckStreamExistsCallback check_stream_exists_callback; + /// Current level of array. Needed to differentiate stream names of nested array offsets. size_t array_level = 0; }; @@ -486,6 +494,13 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro /// Callback used to mark a specific stream as unneeded indicating that it won't be used anymore. std::function release_stream_callback; + /// Callback to check whether a specific substream exists in the current data part. + /// Used during deserialization to handle backward compatibility: old parts written + /// before a new substream was introduced will not have it, and the getter may throw + /// (e.g. in compact parts) if called for a non-existent substream. + using CheckStreamExistsCallback = std::function; + CheckStreamExistsCallback check_stream_exists_callback; + /// Type of MergeTree data part we deserialize data from if any. /// Some serializations may differ from type part for more optimal deserialization. MergeTreeDataPartType data_part_type = MergeTreeDataPartType::Unknown; diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index 9abbd2da3484..6674e902cb8f 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -525,6 +525,11 @@ struct SerializeBinaryBulkStateMapWithBuckets : public ISerialization::Serialize /// Per-bucket nested serialization states. std::vector bucket_nested_states; + /// Bucket index stream state (used when buckets > 1 to preserve original key order). + DataTypePtr bucket_index_type; + SerializationPtr bucket_index_serialization; + ISerialization::SerializeBinaryBulkStatePtr bucket_index_state; + SerializeBinaryBulkStateMapWithBuckets() = default; }; @@ -544,6 +549,12 @@ struct DeserializeBinaryBulkStateMap : public ISerialization::DeserializeBinaryB /// Per-bucket nested deserialization states. std::vector bucket_nested_states; + /// Bucket index stream state (used to preserve original key order). + DataTypePtr bucket_index_type; + SerializationPtr bucket_index_serialization; + ISerialization::DeserializeBinaryBulkStatePtr bucket_index_state; + bool has_bucket_index = false; + ISerialization::DeserializeBinaryBulkStatePtr clone() const override { auto new_state = std::make_shared(*this); @@ -552,6 +563,7 @@ struct DeserializeBinaryBulkStateMap : public ISerialization::DeserializeBinaryB new_state->buckets_info_state = buckets_info_state ? buckets_info_state->clone() : nullptr; for (size_t bucket = 0; bucket != bucket_nested_states.size(); ++bucket) new_state->bucket_nested_states[bucket] = bucket_nested_states[bucket] ? bucket_nested_states[bucket]->clone() : nullptr; + new_state->bucket_index_state = bucket_index_state ? bucket_index_state->clone() : nullptr; return new_state; } }; @@ -657,6 +669,25 @@ void SerializationMap::enumerateStreams( else if (map_column) buckets = calculateNumberOfBuckets(map_column->getOrCalculateStatistics(), settings.max_buckets_in_map, settings.map_buckets_strategy, settings.map_buckets_coefficient, settings.map_buckets_min_avg_size); + /// Enumerate the bucket index stream (used to preserve original key order). + /// Only needed when there are multiple buckets — single-bucket serialization + /// preserves order trivially. When a check_stream_exists_callback is set (e.g. in + /// compact parts determining deserialization order), skip the stream if it does not + /// exist in the part — old parts written before the bucket index fix lack this stream. + if (buckets > 1) + { + settings.path.push_back(Substream::MapBucketIndexes); + bool enumerate_bucket_index = !settings.check_stream_exists_callback || settings.check_stream_exists_callback(settings.path); + if (enumerate_bucket_index) + { + auto bucket_index_serialization = getSmallestIndexesType(buckets)->getDefaultSerialization(); + auto bucket_index_data = SubstreamData(bucket_index_serialization) + .withDeserializeState(map_deserialize_state ? map_deserialize_state->bucket_index_state : nullptr); + bucket_index_serialization->enumerateStreams(settings, callback, bucket_index_data); + } + settings.path.pop_back(); + } + /// Enumerate a nested Array(Tuple(K, V)) stream for each bucket. for (size_t bucket = 0; bucket < buckets; ++bucket) { @@ -719,6 +750,19 @@ void SerializationMap::serializeBinaryBulkStatePrefix( map_state->recalculate_statistics = true; } + /// Initialize bucket index serialization state for multi-bucket parts. + /// Single-bucket parts don't need a bucket index stream (order is trivially preserved). + if (map_state->buckets > 1) + { + map_state->bucket_index_type = getSmallestIndexesType(map_state->buckets); + map_state->bucket_index_serialization = map_state->bucket_index_type->getDefaultSerialization(); + + settings.path.push_back(Substream::MapBucketIndexes); + map_state->bucket_index_serialization->serializeBinaryBulkStatePrefix( + *map_state->bucket_index_type->createColumn(), settings, map_state->bucket_index_state); + settings.path.pop_back(); + } + /// Initialize nested serialization state for each bucket sub-stream. map_state->bucket_nested_states.resize(buckets); for (size_t bucket = 0; bucket < buckets; ++bucket) @@ -853,6 +897,20 @@ void SerializationMap::deserializeBinaryBulkStatePrefix( map_state->buckets_info_state = deserializeBucketsInfoStatePrefix(settings, cache); const auto * buckets_info_state_concrete = checkAndGetState(map_state->buckets_info_state); + /// Initialize bucket index deserialization state. + /// Only needed for multi-bucket parts; single-bucket parts preserve order trivially. + if (buckets_info_state_concrete->buckets > 1) + { + map_state->bucket_index_type = getSmallestIndexesType(buckets_info_state_concrete->buckets); + map_state->bucket_index_serialization = map_state->bucket_index_type->getDefaultSerialization(); + + settings.path.push_back(Substream::MapBucketIndexes); + map_state->has_bucket_index = settings.check_stream_exists_callback && settings.check_stream_exists_callback(settings.path); + if (map_state->has_bucket_index) + map_state->bucket_index_serialization->deserializeBinaryBulkStatePrefix(settings, map_state->bucket_index_state, cache); + settings.path.pop_back(); + } + /// Initialize nested deserialization state for each bucket sub-stream. map_state->bucket_nested_states.resize(buckets_info_state_concrete->buckets); for (size_t bucket = 0; bucket < buckets_info_state_concrete->buckets; ++bucket) @@ -948,7 +1006,7 @@ namespace /// We use `static_cast` here because the dispatch macro guarantees the correct type, /// and `assert_cast` would fail in debug builds for the `IColumn` fallback case /// (it requires exact `typeid` match, but the runtime type is a concrete column). -template +template void splitMapToBucketsTyped( const IColumn & src_keys_col, const IColumn & src_values_col, @@ -956,6 +1014,7 @@ void splitMapToBucketsTyped( std::vector & dst_keys_raw, std::vector & dst_values_raw, std::vector & dst_offsets, + IndexColumn & bucket_index_col, size_t start, size_t end, size_t num_buckets) { const auto & src_keys = static_cast(src_keys_col); @@ -978,6 +1037,7 @@ void splitMapToBucketsTyped( size_t bucket = getBucketForKeyImpl(src_keys, j, num_buckets); dst_keys[bucket]->insertFrom(src_keys, j); dst_values[bucket]->insertFrom(src_values, j); + bucket_index_col.getData().push_back(static_cast(bucket)); } for (size_t bucket = 0; bucket < num_buckets; ++bucket) @@ -985,26 +1045,64 @@ void splitMapToBucketsTyped( } } -/// Second level of the two-level type dispatch for `splitMapToBuckets`. +/// Dispatch macro for bucket index column types (only unsigned integer types). +/// Unlike DISPATCH_MAP_COLUMN_TYPE, this only covers UInt8/16/32/64 since bucket +/// indexes are always unsigned integers chosen by `getSmallestIndexesType`. +// clang-format off +#define DISPATCH_BUCKET_INDEX_COLUMN_TYPE(type_index, CALL) \ + switch (type_index) \ + { \ + case TypeIndex::UInt8: { CALL(ColumnVector); break; } \ + case TypeIndex::UInt16: { CALL(ColumnVector); break; } \ + case TypeIndex::UInt32: { CALL(ColumnVector); break; } \ + case TypeIndex::UInt64: { CALL(ColumnVector); break; } \ + default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected bucket index column type"); \ + } +// clang-format on + +/// Third level of the three-level type dispatch for `splitMapToBuckets`. +/// Key and value column types are already fixed; this dispatches on the bucket index column type. +template +void splitMapToBucketsDispatchByIndex( + const IColumn & src_keys, const IColumn & src_values, + const ColumnArray::Offsets & src_offsets, + std::vector & dst_keys, std::vector & dst_values, + std::vector & dst_offsets, + IColumn & bucket_index_column, + size_t start, size_t end, size_t num_buckets) +{ +// NOLINTBEGIN(bugprone-macro-parentheses) -- IndexColumn is a type used in static_cast<> +#define CALL_SPLIT(IndexColumn) splitMapToBucketsTyped( \ + src_keys, src_values, src_offsets, dst_keys, dst_values, dst_offsets, \ + static_cast(bucket_index_column), start, end, num_buckets) +// NOLINTEND(bugprone-macro-parentheses) + DISPATCH_BUCKET_INDEX_COLUMN_TYPE(bucket_index_column.getDataType(), CALL_SPLIT) +#undef CALL_SPLIT +} + +/// Second level of the three-level type dispatch for `splitMapToBuckets`. /// The key column type is already fixed as `KeyColumn`; this function dispatches -/// on the value column `TypeIndex` and calls `splitMapToBucketsTyped`. +/// on the value column `TypeIndex`. template void splitMapToBucketsDispatchByValue( const IColumn & src_keys, const IColumn & src_values, const ColumnArray::Offsets & src_offsets, std::vector & dst_keys, std::vector & dst_values, std::vector & dst_offsets, + IColumn & bucket_index_column, size_t start, size_t end, size_t num_buckets) { -#define CALL_SPLIT(ValueColumn) splitMapToBucketsTyped( \ - src_keys, src_values, src_offsets, dst_keys, dst_values, dst_offsets, start, end, num_buckets) +// NOLINTNEXTLINE(bugprone-macro-parentheses) -- ValueColumn is a type used as a template argument +#define CALL_SPLIT(ValueColumn) splitMapToBucketsDispatchByIndex( \ + src_keys, src_values, src_offsets, dst_keys, dst_values, dst_offsets, bucket_index_column, start, end, num_buckets) DISPATCH_MAP_COLUMN_TYPE(src_values.getDataType(), CALL_SPLIT) #undef CALL_SPLIT } -/// Entry point of the two-level type dispatch for splitting a Map column into buckets. +/// Entry point of the three-level type dispatch for splitting a Map column into buckets. /// Dispatches on the key column `TypeIndex`, then delegates to -/// `splitMapToBucketsDispatchByValue` which dispatches on the value column type. +/// `splitMapToBucketsDispatchByValue` which dispatches on the value column type, +/// then to `splitMapToBucketsDispatchByIndex` for the bucket index column type. /// For recognized concrete types (integer/float `ColumnVector` variants, `ColumnString`, /// `ColumnFixedString`) the call resolves to a fully devirtualized `splitMapToBucketsTyped`; /// for any other type it falls back to the `IColumn` interface (virtual dispatch). @@ -1013,14 +1111,92 @@ void splitMapToBucketsDispatch( const ColumnArray::Offsets & src_offsets, std::vector & dst_keys, std::vector & dst_values, std::vector & dst_offsets, + IColumn & bucket_index_column, size_t start, size_t end, size_t num_buckets) { +// NOLINTNEXTLINE(bugprone-macro-parentheses) -- KeyColumn is a type used as a template argument #define CALL_KEY_SPLIT(KeyColumn) splitMapToBucketsDispatchByValue( \ - src_keys, src_values, src_offsets, dst_keys, dst_values, dst_offsets, start, end, num_buckets) + src_keys, src_values, src_offsets, dst_keys, dst_values, dst_offsets, bucket_index_column, start, end, num_buckets) DISPATCH_MAP_COLUMN_TYPE(src_keys.getDataType(), CALL_KEY_SPLIT) #undef CALL_KEY_SPLIT } +/// Devirtualized inner loop for collecting Map from buckets in original insertion order. +/// Uses the bucket index array to pull key-value pairs from the correct bucket +/// in the order they were originally inserted. +template +void collectMapFromBucketsWithOrderImpl( + const VectorWithMemoryTracking & map_buckets, + const IndexColumn & bucket_index_col, + IColumn & map_column) +{ + if (map_buckets.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty list of buckets provided"); + + VectorWithMemoryTracking map_keys_buckets(map_buckets.size()); + VectorWithMemoryTracking map_values_buckets(map_buckets.size()); + std::vector map_offsets_buckets(map_buckets.size()); + for (size_t bucket = 0; bucket != map_buckets.size(); ++bucket) + { + const auto & nested_column = assert_cast(*map_buckets[bucket]).getNestedColumn(); + const auto & nested_data = assert_cast(nested_column.getData()); + map_offsets_buckets[bucket] = &nested_column.getOffsets(); + map_keys_buckets[bucket] = nested_data.getColumnPtr(0); + map_values_buckets[bucket] = nested_data.getColumnPtr(1); + } + + auto & nested_column = assert_cast(map_column).getNestedColumn(); + auto & nested_data = assert_cast(nested_column.getData()); + auto & map_keys_column = nested_data.getColumn(0); + auto & map_values_column = nested_data.getColumn(1); + auto & map_offsets = nested_column.getOffsets(); + size_t num_rows = map_buckets[0]->size(); + map_offsets.reserve(map_offsets.size() + num_rows); + + const auto & bucket_index_data = bucket_index_col.getData(); + std::vector bucket_positions(map_buckets.size()); + size_t bucket_index_offset = 0; + + for (size_t i = 0; i != num_rows; ++i) + { + size_t total_size = 0; + for (size_t bucket = 0; bucket < map_buckets.size(); ++bucket) + { + size_t offset_start = (*map_offsets_buckets[bucket])[ssize_t(i) - 1]; + size_t offset_end = (*map_offsets_buckets[bucket])[ssize_t(i)]; + bucket_positions[bucket] = offset_start; + total_size += offset_end - offset_start; + } + + for (size_t j = 0; j < total_size; ++j) + { + size_t bucket_idx = bucket_index_data[bucket_index_offset++]; + if (bucket_idx >= map_buckets.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Bucket index {} is out of range, total buckets: {}", bucket_idx, map_buckets.size()); + size_t pos = bucket_positions[bucket_idx]++; + map_keys_column.insertFrom(*map_keys_buckets[bucket_idx], pos); + map_values_column.insertFrom(*map_values_buckets[bucket_idx], pos); + } + + map_offsets.push_back(map_keys_column.size()); + } +} + +/// Dispatch wrapper for collectMapFromBucketsWithOrderImpl — dispatches on the index column type. +void collectMapFromBucketsWithOrderDispatch( + const VectorWithMemoryTracking & map_buckets, + const IColumn & bucket_index_column, + IColumn & map_column) +{ +// NOLINTBEGIN(bugprone-macro-parentheses) -- IndexColumn is a type used in static_cast<> +#define CALL_COLLECT(IndexColumn) collectMapFromBucketsWithOrderImpl( \ + map_buckets, static_cast(bucket_index_column), map_column) +// NOLINTEND(bugprone-macro-parentheses) + DISPATCH_BUCKET_INDEX_COLUMN_TYPE(bucket_index_column.getDataType(), CALL_COLLECT) +#undef CALL_COLLECT +} + +#undef DISPATCH_BUCKET_INDEX_COLUMN_TYPE #undef DISPATCH_MAP_COLUMN_TYPE } @@ -1030,7 +1206,7 @@ void splitMapToBucketsDispatch( /// one per bucket. Each key-value pair is assigned to a bucket by hashing the key via /// `getBucketForKeyImpl`. Uses two-level type dispatch (`splitMapToBucketsDispatch`) to /// devirtualize `insertFrom` and hash computation for common key/value column types. -VectorWithMemoryTracking SerializationMap::splitMapToBuckets(const IColumn & map_column, size_t start, size_t end, size_t buckets) const +VectorWithMemoryTracking SerializationMap::splitMapToBuckets(const IColumn & map_column, size_t start, size_t end, size_t buckets, IColumn & bucket_index_column) const { VectorWithMemoryTracking map_buckets(buckets); std::vector map_keys_buckets(buckets); @@ -1056,6 +1232,7 @@ VectorWithMemoryTracking SerializationMap::splitMapToBuckets(const IC splitMapToBucketsDispatch( *map_keys_column, *map_values_column, map_offsets, map_keys_buckets, map_values_buckets, map_offsets_buckets, + bucket_index_column, start, end, buckets); return map_buckets; @@ -1105,6 +1282,18 @@ void SerializationMap::collectMapFromBuckets(const VectorWithMemoryTracking & map_buckets, + const IColumn & bucket_index_column, + IColumn & map_column) const +{ + collectMapFromBucketsWithOrderDispatch(map_buckets, bucket_index_column, map_column); +} + void SerializationMap::serializeBinaryBulkWithMultipleStreams( const IColumn & column, size_t offset, @@ -1131,9 +1320,11 @@ void SerializationMap::serializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); } /// Multiple buckets. Split the Map column by key hash, then serialize each bucket independently. + /// Also write the bucket index stream to preserve original key order during deserialization. else { - auto map_buckets = splitMapToBuckets(column, offset, end, map_state->buckets); + auto bucket_index_column = map_state->bucket_index_type->createColumn(); + auto map_buckets = splitMapToBuckets(column, offset, end, map_state->buckets, *bucket_index_column); for (size_t bucket = 0; bucket < map_state->buckets; ++bucket) { settings.path.push_back(SubstreamType::Bucket); @@ -1141,6 +1332,12 @@ void SerializationMap::serializeBinaryBulkWithMultipleStreams( nested_serialization->serializeBinaryBulkWithMultipleStreams(extractNestedColumn(*map_buckets[bucket]), 0, map_buckets[bucket]->size(), settings, map_state->bucket_nested_states[bucket]); settings.path.pop_back(); } + + /// Write the bucket index stream. + settings.path.push_back(Substream::MapBucketIndexes); + map_state->bucket_index_serialization->serializeBinaryBulkWithMultipleStreams( + *bucket_index_column, 0, bucket_index_column->size(), settings, map_state->bucket_index_state); + settings.path.pop_back(); } /// Accumulate statistics from each serialized range. @@ -1207,7 +1404,9 @@ void SerializationMap::deserializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); } /// Multiple buckets. Deserialize each bucket into a separate Map column, - /// then reassemble them into a single column via `collectMapFromBuckets`. + /// then reassemble them into a single column. + /// If bucket index data is available, use it to restore original insertion order; + /// otherwise fall back to bucket-ascending order (old parts without the index stream). else { VectorWithMemoryTracking map_buckets(buckets_info_state->buckets); @@ -1221,7 +1420,31 @@ void SerializationMap::deserializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); } - collectMapFromBuckets(map_buckets, column_map); + if (map_state->has_bucket_index) + { + /// Compute total key-value pairs from per-bucket offsets. + size_t total_kv_pairs = 0; + for (size_t bucket = 0; bucket != buckets_info_state->buckets; ++bucket) + { + const auto & bucket_nested = assert_cast(*map_buckets[bucket]).getNestedColumn(); + const auto & bucket_offsets = bucket_nested.getOffsets(); + if (!bucket_offsets.empty()) + total_kv_pairs += bucket_offsets.back(); + } + + /// Read bucket indexes (flat array, one per key-value pair). + ColumnPtr bucket_index_column = map_state->bucket_index_type->createColumn(); + settings.path.push_back(Substream::MapBucketIndexes); + map_state->bucket_index_serialization->deserializeBinaryBulkWithMultipleStreams( + bucket_index_column, 0, total_kv_pairs, settings, map_state->bucket_index_state, cache); + settings.path.pop_back(); + + collectMapFromBucketsWithOrder(map_buckets, *bucket_index_column, column_map); + } + else + { + collectMapFromBuckets(map_buckets, column_map); + } } } diff --git a/src/DataTypes/Serializations/SerializationMap.h b/src/DataTypes/Serializations/SerializationMap.h index deae1b76644f..5f9a30ef0be1 100644 --- a/src/DataTypes/Serializations/SerializationMap.h +++ b/src/DataTypes/Serializations/SerializationMap.h @@ -157,8 +157,12 @@ class SerializationMap final : public SimpleTextSerialization template ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const; - VectorWithMemoryTracking splitMapToBuckets(const IColumn & map_column, size_t start, size_t end, size_t buckets) const; + VectorWithMemoryTracking splitMapToBuckets(const IColumn & map_column, size_t start, size_t end, size_t buckets, IColumn & bucket_index_column) const; void collectMapFromBuckets(const VectorWithMemoryTracking & map_buckets, IColumn & map_column) const; + void collectMapFromBucketsWithOrder( + const VectorWithMemoryTracking & map_buckets, + const IColumn & bucket_index_column, + IColumn & map_column) const; }; } diff --git a/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp b/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp index 6a1f6dfddca0..1b3958e0c2de 100644 --- a/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp +++ b/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp @@ -1,6 +1,8 @@ #include #include +#include #include +#include #include namespace DB @@ -10,6 +12,7 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } SerializationMapKeysOrValues::SerializationMapKeysOrValues( @@ -45,12 +48,19 @@ struct DeserializeBinaryBulkStateMapKeysOrValuesWithBuckets : public ISerializat /// Per-bucket deserialization state for the keys or values sub-stream. std::vector bucket_keys_or_values_states; + /// Bucket index stream state (used to preserve original key order). + DataTypePtr bucket_index_type; + SerializationPtr bucket_index_serialization; + ISerialization::DeserializeBinaryBulkStatePtr bucket_index_state; + bool has_bucket_index = false; + ISerialization::DeserializeBinaryBulkStatePtr clone() const override { auto new_state = std::make_shared(*this); new_state->buckets_info_state = buckets_info_state ? buckets_info_state->clone() : nullptr; for (size_t bucket = 0; bucket != bucket_keys_or_values_states.size(); ++bucket) new_state->bucket_keys_or_values_states[bucket] = bucket_keys_or_values_states[bucket] ? bucket_keys_or_values_states[bucket]->clone() : nullptr; + new_state->bucket_index_state = bucket_index_state ? bucket_index_state->clone() : nullptr; return new_state; } }; @@ -84,6 +94,24 @@ void SerializationMapKeysOrValues::enumerateStreams( const auto * map_keys_or_values_with_buckets_deserialize_state = checkAndGetState(data.deserialize_state) ; const auto * buckets_info_state = checkAndGetState(map_keys_or_values_with_buckets_deserialize_state->buckets_info_state); + /// Enumerate the bucket index stream (used to preserve original key order). + /// Only needed when there are multiple buckets. When a check_stream_exists_callback + /// is set, skip the stream if it does not exist in the part — old parts written + /// before the bucket index fix lack this stream. + if (buckets_info_state->buckets > 1) + { + settings.path.push_back(Substream::MapBucketIndexes); + bool enumerate_bucket_index = !settings.check_stream_exists_callback || settings.check_stream_exists_callback(settings.path); + if (enumerate_bucket_index) + { + auto bucket_index_serialization = getSmallestIndexesType(buckets_info_state->buckets)->getDefaultSerialization(); + auto bucket_index_data = SubstreamData(bucket_index_serialization) + .withDeserializeState(map_keys_or_values_with_buckets_deserialize_state->bucket_index_state); + bucket_index_serialization->enumerateStreams(settings, callback, bucket_index_data); + } + settings.path.pop_back(); + } + /// Enumerate a keys/values sub-stream for each bucket. for (size_t bucket = 0; bucket < buckets_info_state->buckets; ++bucket) { @@ -134,6 +162,20 @@ void SerializationMapKeysOrValues::deserializeBinaryBulkStatePrefix( map_keys_or_values_with_buckets_state->buckets_info_state = SerializationMap::deserializeBucketsInfoStatePrefix(settings, cache); const auto * buckets_info_state_concrete = checkAndGetState(map_keys_or_values_with_buckets_state->buckets_info_state); + /// Initialize bucket index deserialization state. + /// Only needed for multi-bucket parts; single-bucket parts preserve order trivially. + if (buckets_info_state_concrete->buckets > 1) + { + map_keys_or_values_with_buckets_state->bucket_index_type = getSmallestIndexesType(buckets_info_state_concrete->buckets); + map_keys_or_values_with_buckets_state->bucket_index_serialization = map_keys_or_values_with_buckets_state->bucket_index_type->getDefaultSerialization(); + + settings.path.push_back(Substream::MapBucketIndexes); + map_keys_or_values_with_buckets_state->has_bucket_index = settings.check_stream_exists_callback && settings.check_stream_exists_callback(settings.path); + if (map_keys_or_values_with_buckets_state->has_bucket_index) + map_keys_or_values_with_buckets_state->bucket_index_serialization->deserializeBinaryBulkStatePrefix(settings, map_keys_or_values_with_buckets_state->bucket_index_state, cache); + settings.path.pop_back(); + } + /// Initialize nested deserialization state for keys/values in each bucket. map_keys_or_values_with_buckets_state->bucket_keys_or_values_states.resize(buckets_info_state_concrete->buckets); for (size_t bucket = 0; bucket < buckets_info_state_concrete->buckets; ++bucket) @@ -186,6 +228,85 @@ void collectMapKeysOrValuesFromBuckets(const VectorWithMemoryTracking } } +/// Reassembles a single Array(key_type) or Array(value_type) column from per-bucket Array columns, +/// restoring original insertion order using the bucket index array. +template +void collectMapKeysOrValuesFromBucketsWithOrderImpl( + const VectorWithMemoryTracking & keys_or_values_buckets, + const IndexColumn & bucket_index_col, + IColumn & keys_or_values_column) +{ + if (keys_or_values_buckets.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty list of buckets provided"); + + VectorWithMemoryTracking data_buckets(keys_or_values_buckets.size()); + std::vector offsets_buckets(keys_or_values_buckets.size()); + for (size_t bucket = 0; bucket != keys_or_values_buckets.size(); ++bucket) + { + const auto & array_column = assert_cast(*keys_or_values_buckets[bucket]); + data_buckets[bucket] = array_column.getDataPtr(); + offsets_buckets[bucket] = &array_column.getOffsets(); + } + + auto & array_column = assert_cast(keys_or_values_column); + auto & data = array_column.getData(); + auto & offsets = array_column.getOffsets(); + size_t num_rows = keys_or_values_buckets[0]->size(); + offsets.reserve(offsets.size() + num_rows); + + const auto & bucket_index_data = bucket_index_col.getData(); + std::vector bucket_positions(keys_or_values_buckets.size()); + size_t bucket_index_offset = 0; + + for (size_t i = 0; i != num_rows; ++i) + { + size_t total_size = 0; + for (size_t bucket = 0; bucket < keys_or_values_buckets.size(); ++bucket) + { + size_t offset_start = (*offsets_buckets[bucket])[ssize_t(i) - 1]; + size_t offset_end = (*offsets_buckets[bucket])[ssize_t(i)]; + bucket_positions[bucket] = offset_start; + total_size += offset_end - offset_start; + } + + for (size_t j = 0; j < total_size; ++j) + { + size_t bucket_idx = bucket_index_data[bucket_index_offset++]; + if (bucket_idx >= keys_or_values_buckets.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Bucket index {} is out of range, total buckets: {}", bucket_idx, keys_or_values_buckets.size()); + size_t pos = bucket_positions[bucket_idx]++; + data.insertFrom(*data_buckets[bucket_idx], pos); + } + + offsets.push_back(data.size()); + } +} + +/// Dispatch on the index column type for collectMapKeysOrValuesFromBucketsWithOrderImpl. +void collectMapKeysOrValuesFromBucketsWithOrder( + const VectorWithMemoryTracking & keys_or_values_buckets, + const IColumn & bucket_index_column, + IColumn & keys_or_values_column) +{ + switch (bucket_index_column.getDataType()) + { + case TypeIndex::UInt8: + collectMapKeysOrValuesFromBucketsWithOrderImpl(keys_or_values_buckets, static_cast &>(bucket_index_column), keys_or_values_column); + break; + case TypeIndex::UInt16: + collectMapKeysOrValuesFromBucketsWithOrderImpl(keys_or_values_buckets, static_cast &>(bucket_index_column), keys_or_values_column); + break; + case TypeIndex::UInt32: + collectMapKeysOrValuesFromBucketsWithOrderImpl(keys_or_values_buckets, static_cast &>(bucket_index_column), keys_or_values_column); + break; + case TypeIndex::UInt64: + collectMapKeysOrValuesFromBucketsWithOrderImpl(keys_or_values_buckets, static_cast &>(bucket_index_column), keys_or_values_column); + break; + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected bucket index column type: {}", bucket_index_column.getName()); + } +} + } void SerializationMapKeysOrValues::deserializeBinaryBulkWithMultipleStreams( @@ -215,6 +336,8 @@ void SerializationMapKeysOrValues::deserializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); } /// Multiple buckets. Deserialize each bucket, then reassemble into a single Array column. + /// If bucket index data is available, use it to restore original insertion order; + /// otherwise fall back to bucket-ascending order (old parts without the index stream). else { VectorWithMemoryTracking keys_or_values_buckets(buckets_info_state_concrete->buckets); @@ -227,7 +350,30 @@ void SerializationMapKeysOrValues::deserializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); } - collectMapKeysOrValuesFromBuckets(keys_or_values_buckets, *column->assumeMutable()); + if (map_keys_or_values_with_buckets_state->has_bucket_index) + { + /// Compute total key-value pairs from per-bucket offsets. + size_t total_kv_pairs = 0; + for (size_t bucket = 0; bucket != buckets_info_state_concrete->buckets; ++bucket) + { + const auto & bucket_offsets = assert_cast(*keys_or_values_buckets[bucket]).getOffsets(); + if (!bucket_offsets.empty()) + total_kv_pairs += bucket_offsets.back(); + } + + /// Read bucket indexes (flat array, one per key-value pair). + ColumnPtr bucket_index_column = map_keys_or_values_with_buckets_state->bucket_index_type->createColumn(); + settings.path.push_back(Substream::MapBucketIndexes); + map_keys_or_values_with_buckets_state->bucket_index_serialization->deserializeBinaryBulkWithMultipleStreams( + bucket_index_column, 0, total_kv_pairs, settings, map_keys_or_values_with_buckets_state->bucket_index_state, cache); + settings.path.pop_back(); + + collectMapKeysOrValuesFromBucketsWithOrder(keys_or_values_buckets, *bucket_index_column, *column->assumeMutable()); + } + else + { + collectMapKeysOrValuesFromBuckets(keys_or_values_buckets, *column->assumeMutable()); + } } } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index e393549935b5..35f00883e668 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -399,6 +399,15 @@ void MergeTreeReaderCompact::initSubcolumnsDeserializationOrder() } } + /// Set check_stream_exists_callback so that enumerateStreams can skip substreams + /// that do not exist in this part (e.g. MapBucketIndexes in old bucketed Map parts). + enumerate_settings.check_stream_exists_callback = [&, column_pos = *pos](const ISerialization::SubstreamPath & substream_path) -> bool + { + auto substream = ISerialization::getFileNameForStream( + column, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); + return columns_substreams.tryGetSubstreamPosition(column_pos, substream).has_value(); + }; + auto order = getSubcolumnsDeserializationOrder(column, subcolumns_data, columns_substreams.getColumnSubstreams(*pos), enumerate_settings, ISerialization::StreamFileNameSettings(*storage_settings)); deserialization_order.reserve(subcolumns_indexes.size()); for (size_t i : order) @@ -441,13 +450,25 @@ void MergeTreeReaderCompact::readPrefix(size_t column_idx, size_t from_mark, Mer return stream.getDataBuffer(); }; + /// Build check_stream_exists_callback for this column if we have a column position. + ISerialization::DeserializeBinaryBulkSettings::CheckStreamExistsCallback check_stream_exists_callback; + if (column_positions[column_idx]) + { + check_stream_exists_callback = [&](const ISerialization::SubstreamPath & substream_path) -> bool + { + auto substream = ISerialization::getFileNameForStream( + column, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); + return columns_substreams.tryGetSubstreamPosition(*column_positions[column_idx], substream).has_value(); + }; + } + if (column.isSubcolumn()) { if (has_substream_marks) { const auto & serialization = serializations[column_idx]; auto & state = deserialize_binary_bulk_state_map_for_subcolumns[column.name]; - readPrefix(column, serialization, state, buffer_getter, seek_to_substream_mark ? cache : nullptr); + readPrefix(column, serialization, state, buffer_getter, seek_to_substream_mark ? cache : nullptr, check_stream_exists_callback); } else { @@ -457,14 +478,14 @@ void MergeTreeReaderCompact::readPrefix(size_t column_idx, size_t from_mark, Mer const auto & serialization = serializations_of_full_columns.at(name_in_storage); auto & state = deserialize_binary_bulk_state_map_for_subcolumns[name_in_storage]; - readPrefix(column, serialization, state, buffer_getter, nullptr); + readPrefix(column, serialization, state, buffer_getter, nullptr, check_stream_exists_callback); } } else { const auto & serialization = serializations[column_idx]; auto & state = deserialize_binary_bulk_state_map[column.name]; - readPrefix(column, serialization, state, buffer_getter, seek_to_substream_mark ? cache : nullptr); + readPrefix(column, serialization, state, buffer_getter, seek_to_substream_mark ? cache : nullptr, check_stream_exists_callback); } } @@ -473,7 +494,8 @@ void MergeTreeReaderCompact::readPrefix( const SerializationPtr & serialization, ISerialization::DeserializeBinaryBulkStatePtr & state, const InputStreamGetter & buffer_getter, - ISerialization::SubstreamsDeserializeStatesCache * cache) + ISerialization::SubstreamsDeserializeStatesCache * cache, + ISerialization::DeserializeBinaryBulkSettings::CheckStreamExistsCallback check_stream_exists_callback) { try { @@ -482,6 +504,7 @@ void MergeTreeReaderCompact::readPrefix( deserialize_settings.object_and_dynamic_read_statistics = true; deserialize_settings.use_specialized_prefixes_and_suffixes_substreams = true; deserialize_settings.data_part_type = MergeTreeDataPartType::Compact; + deserialize_settings.check_stream_exists_callback = std::move(check_stream_exists_callback); serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, state, cache); } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 8bec0f72a2bc..091564b49eb6 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -107,7 +107,8 @@ class MergeTreeReaderCompact : public IMergeTreeReader const SerializationPtr & serialization, ISerialization::DeserializeBinaryBulkStatePtr & state, const InputStreamGetter & buffer_getter, - ISerialization::SubstreamsDeserializeStatesCache * cache); + ISerialization::SubstreamsDeserializeStatesCache * cache, + ISerialization::DeserializeBinaryBulkSettings::CheckStreamExistsCallback check_stream_exists_callback = {}); NameAndTypePair getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column); void findPositionForMissedNested(size_t pos); diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 6e9a0f80cc14..254043c6ee22 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -447,6 +447,13 @@ void MergeTreeReaderWide::deserializePrefix( if (stream_name) streams.erase(*stream_name); }; + deserialize_settings.check_stream_exists_callback = [&](const ISerialization::SubstreamPath & substream_path) -> bool + { + auto stream_name = IMergeTreeDataPart::getStreamNameForColumn( + name_and_type, substream_path, ".bin", + data_part_info_for_read->getChecksums(), storage_settings); + return stream_name.has_value(); + }; deserialize_settings.release_all_prefixes_streams = settings.read_only_column_sample; deserialize_settings.has_uniform_marks_callback = [&](const ISerialization::SubstreamPath & substream_path, diff --git a/tests/integration/test_backward_compatibility/test_bucketed_map_order.py b/tests/integration/test_backward_compatibility/test_bucketed_map_order.py new file mode 100644 index 000000000000..11a0c7364bd4 --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_bucketed_map_order.py @@ -0,0 +1,259 @@ +""" +Test backward compatibility for bucketed Map serialization with MapBucketIndexes fix. + +Parts written by older ClickHouse versions (with bucketed Map from PR #99200 but without +the MapBucketIndexes fix) have no bucket_indexes stream. The new code must detect this +via check_stream_exists_callback and fall back to the unordered collectMapFromBuckets path. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster + +# First stable release with bucketed Map (PR #99200), before the bucket index fix. +# Pinned to exact patch tag because the fix will be backported; a floating minor +# tag like "26.4" could resolve to a patched release that already includes it. +OLD_VERSION = "26.4.1.1141" + +TABLE_SETTINGS_WIDE = """ + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + serialization_info_version = 'with_types' +""" + +TABLE_SETTINGS_COMPACT = """ + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = '200G', + min_rows_for_wide_part = 1000000, + serialization_info_version = 'with_types' +""" + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + with_zookeeper=False, + image="clickhouse/clickhouse-server", + tag=OLD_VERSION, + stay_alive=True, + with_installed_binary=True, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_bucketed_map_backward_compatibility(start_cluster): + """ + All backward compatibility scenarios in a single test to avoid redundant restarts. + Creates all tables and inserts data on the old version, upgrades once, then verifies. + """ + + # --- Phase 1: Create tables and insert data on old version --- + + # Wide parts table + node.query( + f""" + CREATE TABLE t_wide (id UInt64, m Map(String, UInt64)) + ENGINE = MergeTree ORDER BY id + SETTINGS {TABLE_SETTINGS_WIDE} + """ + ) + node.query( + """ + INSERT INTO t_wide VALUES + (1, {'z':1, 'a':2, 'm':3}), + (2, {'dog':10, 'ant':20, 'cat':30}) + """ + ) + + # Compact parts table + node.query( + f""" + CREATE TABLE t_compact (id UInt64, m Map(String, UInt64)) + ENGINE = MergeTree ORDER BY id + SETTINGS {TABLE_SETTINGS_COMPACT} + """ + ) + node.query( + """ + INSERT INTO t_compact VALUES + (1, {'z':1, 'a':2, 'm':3}), + (2, {'dog':10, 'ant':20, 'cat':30}) + """ + ) + + # Table for merge test (old part will be merged with a new part after upgrade) + node.query( + f""" + CREATE TABLE t_merge (id UInt64, m Map(String, UInt64)) + ENGINE = MergeTree ORDER BY id + SETTINGS {TABLE_SETTINGS_WIDE} + """ + ) + node.query("INSERT INTO t_merge VALUES (1, {'z':1, 'a':2})") + + # Table for subcolumn test (wide parts) + node.query( + f""" + CREATE TABLE t_sub (id UInt64, m Map(String, UInt64)) + ENGINE = MergeTree ORDER BY id + SETTINGS {TABLE_SETTINGS_WIDE} + """ + ) + node.query("INSERT INTO t_sub VALUES (1, {'z':1, 'a':2, 'm':3})") + + # Table for subcolumn test (compact parts) + node.query( + f""" + CREATE TABLE t_sub_compact (id UInt64, m Map(String, UInt64)) + ENGINE = MergeTree ORDER BY id + SETTINGS {TABLE_SETTINGS_COMPACT} + """ + ) + node.query("INSERT INTO t_sub_compact VALUES (1, {'z':1, 'a':2, 'm':3})") + + # --- Phase 2: Upgrade to latest version (single restart) --- + + node.restart_with_latest_version() + + # --- Phase 3: Verify old parts have NO bucket_indexes stream --- + + # Old parts were written before the fix, so they must not have the bucket_indexes stream. + # This confirms we're actually exercising the check_stream_exists_callback fallback path. + for table in ["t_wide", "t_compact", "t_merge", "t_sub", "t_sub_compact"]: + result = node.query( + f""" + SELECT has(substreams, 'm.bucket_indexes') + FROM system.parts_columns + WHERE database = currentDatabase() AND table = '{table}' + AND column = 'm' AND active = 1 + LIMIT 1 + """ + ).strip() + assert result == "0", f"Old part in {table} should not have bucket_indexes stream" + + # --- Phase 4a: Verify old wide parts are readable --- + + assert node.query("SELECT count() FROM t_wide").strip() == "2" + assert ( + node.query("SELECT id, length(m) FROM t_wide ORDER BY id").strip() + == "1\t3\n2\t3" + ) + # Subcolumn access on old wide parts + assert ( + node.query("SELECT id, m['z'], m['dog'] FROM t_wide ORDER BY id").strip() + == "1\t1\t0\n2\t0\t10" + ) + + # --- Phase 4b: Verify old compact parts are readable --- + + assert node.query("SELECT count() FROM t_compact").strip() == "2" + assert ( + node.query("SELECT id, length(m) FROM t_compact ORDER BY id").strip() + == "1\t3\n2\t3" + ) + assert ( + node.query("SELECT id, m['z'], m['dog'] FROM t_compact ORDER BY id").strip() + == "1\t1\t0\n2\t0\t10" + ) + + # --- Phase 5: Insert new data and merge with old parts --- + + # New part has bucket_indexes stream + node.query("INSERT INTO t_merge VALUES (2, {'x':10, 'b':20})") + + # Verify the new part has bucket_indexes + assert ( + node.query( + """ + SELECT has(substreams, 'm.bucket_indexes') + FROM system.parts_columns + WHERE database = currentDatabase() AND table = 't_merge' + AND column = 'm' AND active = 1 AND level = 0 + ORDER BY name DESC + LIMIT 1 + """ + ).strip() + == "1" + ) + + # Merge old (no bucket_indexes) + new (has bucket_indexes) parts + node.query("OPTIMIZE TABLE t_merge FINAL") + + # Merged part has bucket_indexes + assert ( + node.query( + """ + SELECT has(substreams, 'm.bucket_indexes') + FROM system.parts_columns + WHERE database = currentDatabase() AND table = 't_merge' + AND column = 'm' AND active = 1 + LIMIT 1 + """ + ).strip() + == "1" + ) + + assert node.query("SELECT count() FROM t_merge").strip() == "2" + + # New data has correct key order after merge + assert ( + node.query("SELECT id, mapKeys(m) FROM t_merge WHERE id = 2").strip() + == "2\t['x','b']" + ) + + # ORDER BY on merged data produces a deterministic result + result = node.query("SELECT id FROM t_merge ORDER BY m").strip() + assert len(result.split("\n")) == 2 + + # --- Phase 6: Verify subcolumns on old wide parts --- + + # map.keys subcolumn (uses SerializationMapKeysOrValues path) + result = node.query("SELECT m.keys FROM t_sub").strip() + assert len(result) > 0 + + # map.values subcolumn + result = node.query("SELECT m.values FROM t_sub").strip() + assert len(result) > 0 + + # map.size0 subcolumn + assert node.query("SELECT m.size0 FROM t_sub").strip() == "3" + + # --- Phase 7: Verify subcolumns on old compact parts --- + # This exercises the enumerateStreams check_stream_exists_callback path in + # initSubcolumnsDeserializationOrder — old compact parts lack the bucket_indexes + # stream and enumerating it unconditionally would cause "Unexpected substream" errors. + + # map.keys subcolumn on compact part + result = node.query("SELECT m.keys FROM t_sub_compact").strip() + assert len(result) > 0 + + # map.values subcolumn on compact part + result = node.query("SELECT m.values FROM t_sub_compact").strip() + assert len(result) > 0 + + # map.size0 subcolumn on compact part + assert node.query("SELECT m.size0 FROM t_sub_compact").strip() == "3" + + # --- Cleanup --- + + node.query("DROP TABLE t_wide") + node.query("DROP TABLE t_compact") + node.query("DROP TABLE t_merge") + node.query("DROP TABLE t_sub") + node.query("DROP TABLE t_sub_compact") diff --git a/tests/queries/0_stateless/03993_map_subcolumns_small_compact.reference b/tests/queries/0_stateless/03993_map_subcolumns_small_compact.reference index b2923b023a33..4d8e1687dca1 100644 --- a/tests/queries/0_stateless/03993_map_subcolumns_small_compact.reference +++ b/tests/queries/0_stateless/03993_map_subcolumns_small_compact.reference @@ -193,35 +193,35 @@ with_buckets compact: m {'a':1,'b':2,'c':3} {'a':10,'b':20} {} -{'d':200,'c':100} +{'c':100,'d':200} {'a':5} -{'a':1,'b':2,'d':4,'c':3,'e':5} +{'a':1,'b':2,'c':3,'d':4,'e':5} {'b':42} {'a':0,'d':0} {} -{'a':99,'b':98,'d':96,'c':97,'e':95} +{'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets compact: m.keys ['a','b','c'] ['a','b'] [] -['d','c'] +['c','d'] ['a'] -['a','b','d','c','e'] +['a','b','c','d','e'] ['b'] ['a','d'] [] -['a','b','d','c','e'] +['a','b','c','d','e'] with_buckets compact: m.values [1,2,3] [10,20] [] -[200,100] +[100,200] [5] -[1,2,4,3,5] +[1,2,3,4,5] [42] [0,0] [] -[99,98,96,97,95] +[99,98,97,96,95] with_buckets compact: m.size0 3 2 @@ -270,13 +270,13 @@ with_buckets compact: m.keys, m.values ['a','b','c'] [1,2,3] ['a','b'] [10,20] [] [] -['d','c'] [200,100] +['c','d'] [100,200] ['a'] [5] -['a','b','d','c','e'] [1,2,4,3,5] +['a','b','c','d','e'] [1,2,3,4,5] ['b'] [42] ['a','d'] [0,0] [] [] -['a','b','d','c','e'] [99,98,96,97,95] +['a','b','c','d','e'] [99,98,97,96,95] with_buckets compact: m.size0, m.key_a 3 1 2 10 @@ -292,46 +292,46 @@ with_buckets compact: m, m.keys {'a':1,'b':2,'c':3} ['a','b','c'] {'a':10,'b':20} ['a','b'] {} [] -{'d':200,'c':100} ['d','c'] +{'c':100,'d':200} ['c','d'] {'a':5} ['a'] -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] {'b':42} ['b'] {'a':0,'d':0} ['a','d'] {} [] -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] with_buckets compact: m, m.key_a {'a':1,'b':2,'c':3} 1 {'a':10,'b':20} 10 {} 0 -{'d':200,'c':100} 0 +{'c':100,'d':200} 0 {'a':5} 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} 1 {'b':42} 0 {'a':0,'d':0} 0 {} 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} 99 with_buckets compact: m.keys, m ['a','b','c'] {'a':1,'b':2,'c':3} ['a','b'] {'a':10,'b':20} [] {} -['d','c'] {'d':200,'c':100} +['c','d'] {'c':100,'d':200} ['a'] {'a':5} -['a','b','d','c','e'] {'a':1,'b':2,'d':4,'c':3,'e':5} +['a','b','c','d','e'] {'a':1,'b':2,'c':3,'d':4,'e':5} ['b'] {'b':42} ['a','d'] {'a':0,'d':0} [] {} -['a','b','d','c','e'] {'a':99,'b':98,'d':96,'c':97,'e':95} +['a','b','c','d','e'] {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets compact: m.key_a, m.size0, m 1 3 {'a':1,'b':2,'c':3} 10 2 {'a':10,'b':20} 0 0 {} -0 2 {'d':200,'c':100} +0 2 {'c':100,'d':200} 5 1 {'a':5} -1 5 {'a':1,'b':2,'d':4,'c':3,'e':5} +1 5 {'a':1,'b':2,'c':3,'d':4,'e':5} 0 1 {'b':42} 0 2 {'a':0,'d':0} 0 0 {} -99 5 {'a':99,'b':98,'d':96,'c':97,'e':95} +99 5 {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets compact: m.key_a, m.key_b, m.key_c 1 2 3 10 20 0 @@ -358,13 +358,13 @@ with_buckets compact: m, m.keys, m.values, m.size0, m.key_a {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 with_buckets compact: m, m.keys, m.values, m.size0, m.key_a limit 3 {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 @@ -373,13 +373,13 @@ with_buckets compact: m, m.keys, m.values, m.size0, m.key_a max_block_size=3 {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 basic compact tuple: data.m {'a':1,'b':2,'c':3} {'a':10,'b':20} @@ -483,35 +483,35 @@ with_buckets compact tuple: data.m {'a':1,'b':2,'c':3} {'a':10,'b':20} {} -{'d':200,'c':100} +{'c':100,'d':200} {'a':5} -{'a':1,'b':2,'d':4,'c':3,'e':5} +{'a':1,'b':2,'c':3,'d':4,'e':5} {'b':42} {'a':0,'d':0} {} -{'a':99,'b':98,'d':96,'c':97,'e':95} +{'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets compact tuple: data.m.keys ['a','b','c'] ['a','b'] [] -['d','c'] +['c','d'] ['a'] -['a','b','d','c','e'] +['a','b','c','d','e'] ['b'] ['a','d'] [] -['a','b','d','c','e'] +['a','b','c','d','e'] with_buckets compact tuple: data.m.values [1,2,3] [10,20] [] -[200,100] +[100,200] [5] -[1,2,4,3,5] +[1,2,3,4,5] [42] [0,0] [] -[99,98,96,97,95] +[99,98,97,96,95] with_buckets compact tuple: data.m.size0 3 2 @@ -549,32 +549,32 @@ with_buckets compact tuple: data.m, data.m.keys {'a':1,'b':2,'c':3} ['a','b','c'] {'a':10,'b':20} ['a','b'] {} [] -{'d':200,'c':100} ['d','c'] +{'c':100,'d':200} ['c','d'] {'a':5} ['a'] -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] {'b':42} ['b'] {'a':0,'d':0} ['a','d'] {} [] -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] with_buckets compact tuple: data.m.keys, data.m ['a','b','c'] {'a':1,'b':2,'c':3} ['a','b'] {'a':10,'b':20} [] {} -['d','c'] {'d':200,'c':100} +['c','d'] {'c':100,'d':200} ['a'] {'a':5} -['a','b','d','c','e'] {'a':1,'b':2,'d':4,'c':3,'e':5} +['a','b','c','d','e'] {'a':1,'b':2,'c':3,'d':4,'e':5} ['b'] {'b':42} ['a','d'] {'a':0,'d':0} [] {} -['a','b','d','c','e'] {'a':99,'b':98,'d':96,'c':97,'e':95} +['a','b','c','d','e'] {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets compact tuple: data.m, data.m.keys, data.m.values, data.m.size0, data.m.key_a {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 diff --git a/tests/queries/0_stateless/03994_map_subcolumns_small_wide.reference b/tests/queries/0_stateless/03994_map_subcolumns_small_wide.reference index 3f6ff418a503..b7c1ad683511 100644 --- a/tests/queries/0_stateless/03994_map_subcolumns_small_wide.reference +++ b/tests/queries/0_stateless/03994_map_subcolumns_small_wide.reference @@ -193,35 +193,35 @@ with_buckets wide: m {'a':1,'b':2,'c':3} {'a':10,'b':20} {} -{'d':200,'c':100} +{'c':100,'d':200} {'a':5} -{'a':1,'b':2,'d':4,'c':3,'e':5} +{'a':1,'b':2,'c':3,'d':4,'e':5} {'b':42} {'a':0,'d':0} {} -{'a':99,'b':98,'d':96,'c':97,'e':95} +{'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets wide: m.keys ['a','b','c'] ['a','b'] [] -['d','c'] +['c','d'] ['a'] -['a','b','d','c','e'] +['a','b','c','d','e'] ['b'] ['a','d'] [] -['a','b','d','c','e'] +['a','b','c','d','e'] with_buckets wide: m.values [1,2,3] [10,20] [] -[200,100] +[100,200] [5] -[1,2,4,3,5] +[1,2,3,4,5] [42] [0,0] [] -[99,98,96,97,95] +[99,98,97,96,95] with_buckets wide: m.size0 3 2 @@ -270,13 +270,13 @@ with_buckets wide: m.keys, m.values ['a','b','c'] [1,2,3] ['a','b'] [10,20] [] [] -['d','c'] [200,100] +['c','d'] [100,200] ['a'] [5] -['a','b','d','c','e'] [1,2,4,3,5] +['a','b','c','d','e'] [1,2,3,4,5] ['b'] [42] ['a','d'] [0,0] [] [] -['a','b','d','c','e'] [99,98,96,97,95] +['a','b','c','d','e'] [99,98,97,96,95] with_buckets wide: m.size0, m.key_a 3 1 2 10 @@ -292,46 +292,46 @@ with_buckets wide: m, m.keys {'a':1,'b':2,'c':3} ['a','b','c'] {'a':10,'b':20} ['a','b'] {} [] -{'d':200,'c':100} ['d','c'] +{'c':100,'d':200} ['c','d'] {'a':5} ['a'] -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] {'b':42} ['b'] {'a':0,'d':0} ['a','d'] {} [] -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] with_buckets wide: m, m.key_a {'a':1,'b':2,'c':3} 1 {'a':10,'b':20} 10 {} 0 -{'d':200,'c':100} 0 +{'c':100,'d':200} 0 {'a':5} 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} 1 {'b':42} 0 {'a':0,'d':0} 0 {} 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} 99 with_buckets wide: m.keys, m ['a','b','c'] {'a':1,'b':2,'c':3} ['a','b'] {'a':10,'b':20} [] {} -['d','c'] {'d':200,'c':100} +['c','d'] {'c':100,'d':200} ['a'] {'a':5} -['a','b','d','c','e'] {'a':1,'b':2,'d':4,'c':3,'e':5} +['a','b','c','d','e'] {'a':1,'b':2,'c':3,'d':4,'e':5} ['b'] {'b':42} ['a','d'] {'a':0,'d':0} [] {} -['a','b','d','c','e'] {'a':99,'b':98,'d':96,'c':97,'e':95} +['a','b','c','d','e'] {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets wide: m.key_a, m.size0, m 1 3 {'a':1,'b':2,'c':3} 10 2 {'a':10,'b':20} 0 0 {} -0 2 {'d':200,'c':100} +0 2 {'c':100,'d':200} 5 1 {'a':5} -1 5 {'a':1,'b':2,'d':4,'c':3,'e':5} +1 5 {'a':1,'b':2,'c':3,'d':4,'e':5} 0 1 {'b':42} 0 2 {'a':0,'d':0} 0 0 {} -99 5 {'a':99,'b':98,'d':96,'c':97,'e':95} +99 5 {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets wide: m.key_a, m.key_b, m.key_c 1 2 3 10 20 0 @@ -358,13 +358,13 @@ with_buckets wide: m, m.keys, m.values, m.size0, m.key_a {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 with_buckets wide: m, m.keys, m.values, m.size0, m.key_a limit 3 {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 @@ -373,13 +373,13 @@ with_buckets wide: m, m.keys, m.values, m.size0, m.key_a max_block_size=3 {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 basic wide tuple: data.m {'a':1,'b':2,'c':3} {'a':10,'b':20} @@ -483,35 +483,35 @@ with_buckets wide tuple: data.m {'a':1,'b':2,'c':3} {'a':10,'b':20} {} -{'d':200,'c':100} +{'c':100,'d':200} {'a':5} -{'a':1,'b':2,'d':4,'c':3,'e':5} +{'a':1,'b':2,'c':3,'d':4,'e':5} {'b':42} {'a':0,'d':0} {} -{'a':99,'b':98,'d':96,'c':97,'e':95} +{'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets wide tuple: data.m.keys ['a','b','c'] ['a','b'] [] -['d','c'] +['c','d'] ['a'] -['a','b','d','c','e'] +['a','b','c','d','e'] ['b'] ['a','d'] [] -['a','b','d','c','e'] +['a','b','c','d','e'] with_buckets wide tuple: data.m.values [1,2,3] [10,20] [] -[200,100] +[100,200] [5] -[1,2,4,3,5] +[1,2,3,4,5] [42] [0,0] [] -[99,98,96,97,95] +[99,98,97,96,95] with_buckets wide tuple: data.m.size0 3 2 @@ -549,32 +549,32 @@ with_buckets wide tuple: data.m, data.m.keys {'a':1,'b':2,'c':3} ['a','b','c'] {'a':10,'b':20} ['a','b'] {} [] -{'d':200,'c':100} ['d','c'] +{'c':100,'d':200} ['c','d'] {'a':5} ['a'] -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] {'b':42} ['b'] {'a':0,'d':0} ['a','d'] {} [] -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] with_buckets wide tuple: data.m.keys, data.m ['a','b','c'] {'a':1,'b':2,'c':3} ['a','b'] {'a':10,'b':20} [] {} -['d','c'] {'d':200,'c':100} +['c','d'] {'c':100,'d':200} ['a'] {'a':5} -['a','b','d','c','e'] {'a':1,'b':2,'d':4,'c':3,'e':5} +['a','b','c','d','e'] {'a':1,'b':2,'c':3,'d':4,'e':5} ['b'] {'b':42} ['a','d'] {'a':0,'d':0} [] {} -['a','b','d','c','e'] {'a':99,'b':98,'d':96,'c':97,'e':95} +['a','b','c','d','e'] {'a':99,'b':98,'c':97,'d':96,'e':95} with_buckets wide tuple: data.m, data.m.keys, data.m.values, data.m.size0, data.m.key_a {'a':1,'b':2,'c':3} ['a','b','c'] [1,2,3] 3 1 {'a':10,'b':20} ['a','b'] [10,20] 2 10 {} [] [] 0 0 -{'d':200,'c':100} ['d','c'] [200,100] 2 0 +{'c':100,'d':200} ['c','d'] [100,200] 2 0 {'a':5} ['a'] [5] 1 5 -{'a':1,'b':2,'d':4,'c':3,'e':5} ['a','b','d','c','e'] [1,2,4,3,5] 5 1 +{'a':1,'b':2,'c':3,'d':4,'e':5} ['a','b','c','d','e'] [1,2,3,4,5] 5 1 {'b':42} ['b'] [42] 1 0 {'a':0,'d':0} ['a','d'] [0,0] 2 0 {} [] [] 0 0 -{'a':99,'b':98,'d':96,'c':97,'e':95} ['a','b','d','c','e'] [99,98,96,97,95] 5 99 +{'a':99,'b':98,'c':97,'d':96,'e':95} ['a','b','c','d','e'] [99,98,97,96,95] 5 99 diff --git a/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.reference b/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.reference index 23529291b5ef..cb8a74635b76 100644 --- a/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.reference +++ b/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.reference @@ -29,9 +29,9 @@ 2: row count after merge 100 2: spot check data -0 {'y':0,'x':0} -25 {'y':250,'x':25} -49 {'y':490,'x':49} +0 {'x':0,'y':0} +25 {'x':25,'y':250} +49 {'x':49,'y':490} 50 {'x':50,'z':0} 75 {'x':75,'z':125} 99 {'x':99,'z':245} @@ -51,12 +51,12 @@ 3: row count after merge 100 3: spot check data -0 {'c':2,'a':0,'b':1} -25 {'c':27,'a':25,'b':26} -49 {'c':51,'a':49,'b':50} -50 {'d':52,'a':50,'b':51} -75 {'d':77,'a':75,'b':76} -99 {'d':101,'a':99,'b':100} +0 {'a':0,'b':1,'c':2} +25 {'a':25,'b':26,'c':27} +49 {'a':49,'b':50,'c':51} +50 {'a':50,'b':51,'d':52} +75 {'a':75,'b':76,'d':77} +99 {'a':99,'b':100,'d':101} 3: subcolumn correctness 0 0 1 2 0 25 25 26 27 0 @@ -86,12 +86,12 @@ all_3_3_0 1 6 0 {'a':0} 15 {'a':15} 29 {'a':29} -30 {'b':100,'a':30} -45 {'b':115,'a':45} -59 {'b':129,'a':59} -60 {'b':200,'c':300,'a':60} -75 {'b':215,'c':315,'a':75} -89 {'b':229,'c':329,'a':89} +30 {'a':30,'b':100} +45 {'a':45,'b':115} +59 {'a':59,'b':129} +60 {'a':60,'b':200,'c':300} +75 {'a':75,'b':215,'c':315} +89 {'a':89,'b':229,'c':329} 5: subcolumn correctness 0 0 0 0 15 15 0 0 diff --git a/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.sql b/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.sql index 87b07a482362..630af5120ab7 100644 --- a/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.sql +++ b/tests/queries/0_stateless/03999_map_change_serialization_version_and_merge.sql @@ -1,3 +1,5 @@ +-- Tags: long + -- Test: Changing map_serialization_version and bucket settings on existing table. -- Exercises merging parts with different serialization modes and bucket counts. diff --git a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference index ce47a112707c..89024e17dd7f 100644 --- a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference +++ b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference @@ -4,25 +4,25 @@ -- arrayElement should NOT appear in PREWHERE 1 -- Correctness: PREWHERE m[key1] > 90 -91 {'key2':910,'key1':91} 91 910 -92 {'key2':920,'key1':92} 92 920 -93 {'key2':930,'key1':93} 93 930 -94 {'key2':940,'key1':94} 94 940 -95 {'key2':950,'key1':95} 95 950 -96 {'key2':960,'key1':96} 96 960 -97 {'key2':970,'key1':97} 97 970 -98 {'key2':980,'key1':98} 98 980 -99 {'key2':990,'key1':99} 99 990 +91 {'key1':91,'key2':910} 91 910 +92 {'key1':92,'key2':920} 92 920 +93 {'key1':93,'key2':930} 93 930 +94 {'key1':94,'key2':940} 94 940 +95 {'key1':95,'key2':950} 95 950 +96 {'key1':96,'key2':960} 96 960 +97 {'key1':97,'key2':970} 97 970 +98 {'key1':98,'key2':980} 98 980 +99 {'key1':99,'key2':990} 99 990 -- Correctness: same without optimization -91 {'key2':910,'key1':91} 91 910 -92 {'key2':920,'key1':92} 92 920 -93 {'key2':930,'key1':93} 93 930 -94 {'key2':940,'key1':94} 94 940 -95 {'key2':950,'key1':95} 95 950 -96 {'key2':960,'key1':96} 96 960 -97 {'key2':970,'key1':97} 97 970 -98 {'key2':980,'key1':98} 98 980 -99 {'key2':990,'key1':99} 99 990 +91 {'key1':91,'key2':910} 91 910 +92 {'key1':92,'key2':920} 92 920 +93 {'key1':93,'key2':930} 93 930 +94 {'key1':94,'key2':940} 94 940 +95 {'key1':95,'key2':950} 95 950 +96 {'key1':96,'key2':960} 96 960 +97 {'key1':97,'key2':970} 97 970 +98 {'key1':98,'key2':980} 98 980 +99 {'key1':99,'key2':990} 99 990 -- Section 2: PREWHERE + WHERE combined -- PREWHERE on map subcolumn, WHERE on v 1 diff --git a/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.reference b/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.reference new file mode 100644 index 000000000000..361572639503 --- /dev/null +++ b/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.reference @@ -0,0 +1,32 @@ +S1: key order preserved in wide parts after insert +1 ['z','a','m','b'] +2 ['dog','ant','cat','bat'] +3 ['3','1','2'] +S2: key order preserved in compact parts after insert +1 ['z','a','m','b'] +2 ['dog','ant','cat','bat'] +3 ['3','1','2'] +S3: key order preserved after merge +1 ['z','a','m'] +2 ['x','b','w'] +S4: UInt64 key order preserved +1 [100,1,50,25] +2 [999,0,500] +S5: Int32 key order preserved +1 [-10,5,-100,0] +S6: bucket_indexes stream present +1 +S7: no bucket_indexes with 1 bucket +0 +S8: key order preserved after basic+with_buckets merge +1 ['z','a','m'] +2 ['x','b','w'] +S9: order after zero-level=basic merged=with_buckets +1 ['z','a'] +2 ['x','b'] +3 ['m','c'] +S10: order preserved after multiple merges +1 ['z','a','m'] +2 ['x','b','w'] +3 ['q','j','p'] +4 ['d','f','e'] diff --git a/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.sql b/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.sql new file mode 100644 index 000000000000..da7ee663ab29 --- /dev/null +++ b/tests/queries/0_stateless/04409_map_buckets_key_order_preservation.sql @@ -0,0 +1,244 @@ +-- Test: Bucketed Map serialization preserves original key insertion order. +-- The fix writes a MapBucketIndexes substream to restore key ordering during deserialization. + +-- Section 1: Wide parts, String keys — order preserved after insert +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'z':1, 'a':2, 'm':3, 'b':4}), + (2, {'dog':10, 'ant':20, 'cat':30, 'bat':40}), + (3, {'3':100, '1':200, '2':300}); + +SELECT 'S1: key order preserved in wide parts after insert'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 2: Compact parts, String keys — order preserved after insert +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = '200G', + min_rows_for_wide_part = 1000000, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'z':1, 'a':2, 'm':3, 'b':4}), + (2, {'dog':10, 'ant':20, 'cat':30, 'bat':40}), + (3, {'3':100, '1':200, '2':300}); + +SELECT 'S2: key order preserved in compact parts after insert'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 3: Wide parts — order preserved after OPTIMIZE FINAL +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'z':1, 'a':2, 'm':3}); +INSERT INTO t VALUES (2, {'x':10, 'b':20, 'w':30}); +OPTIMIZE TABLE t FINAL; + +SELECT 'S3: key order preserved after merge'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 4: UInt64 keys — order preserved +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(UInt64, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {100:'a', 1:'b', 50:'c', 25:'d'}), + (2, {999:'x', 0:'y', 500:'z'}); + +SELECT 'S4: UInt64 key order preserved'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 5: Int32 keys — order preserved (signed type) +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(Int32, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {-10:'a', 5:'b', -100:'c', 0:'d'}); + +SELECT 'S5: Int32 key order preserved'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 6: Bucket indexes stream presence in system.parts_columns +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'z':1, 'a':2, 'm':3, 'b':4}); + +SELECT 'S6: bucket_indexes stream present'; +SELECT has(substreams, 'm.bucket_indexes') AS has_bucket_indexes +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't' AND column = 'm' AND active = 1 +LIMIT 1; +DROP TABLE t; + +-- Section 7: No bucket_indexes when only 1 bucket (min_avg_size threshold) +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'sqrt', + map_buckets_min_avg_size = 32, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'a':1, 'b':2, 'c':3}); + +SELECT 'S7: no bucket_indexes with 1 bucket'; +SELECT has(substreams, 'm.bucket_indexes') AS has_bucket_indexes +FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't' AND column = 'm' AND active = 1 +LIMIT 1; +DROP TABLE t; + +-- Section 8: Order preserved across basic->with_buckets merge +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'basic', + map_serialization_version_for_zero_level_parts = 'basic', + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'z':1, 'a':2, 'm':3}); + +ALTER TABLE t MODIFY SETTING + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0; + +INSERT INTO t VALUES (2, {'x':10, 'b':20, 'w':30}); + +OPTIMIZE TABLE t FINAL; + +SELECT 'S8: key order preserved after basic+with_buckets merge'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 9: Order preserved with zero-level basic, merged with_buckets +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'basic', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'z':1, 'a':2}), (2, {'x':10, 'b':20}); +INSERT INTO t VALUES (3, {'m':100, 'c':200}); + +OPTIMIZE TABLE t FINAL; + +SELECT 'S9: order after zero-level=basic merged=with_buckets'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; + +-- Section 10: Multiple merges preserve order +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'z':1, 'a':2, 'm':3}); +INSERT INTO t VALUES (2, {'x':10, 'b':20, 'w':30}); +INSERT INTO t VALUES (3, {'q':100, 'j':200, 'p':300}); +INSERT INTO t VALUES (4, {'d':1000, 'f':2000, 'e':3000}); + +OPTIMIZE TABLE t FINAL; + +SELECT 'S10: order preserved after multiple merges'; +SELECT id, mapKeys(m) FROM t ORDER BY id; +DROP TABLE t; diff --git a/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.reference b/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.reference new file mode 100644 index 000000000000..855411a1dcfb --- /dev/null +++ b/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.reference @@ -0,0 +1,30 @@ +S1: ORDER BY map column +4 {'a':1,'b':1} +1 {'a':1,'b':2} +2 {'a':1,'c':3} +3 {'b':1,'a':2} +S2: equal maps compare equal +2 +S3: DISTINCT on map column +{'a':2,'z':1} +{'z':1,'a':2} +{'z':1,'a':3} +S4: GROUP BY map column +{'z':1,'a':2} 2 +{'x':10,'y':20} 3 +S5: min/max on map column +{'a':1,'b':2} {'c':1} +S6: ORDER BY after merge +4 {'a':1,'b':1} +1 {'a':1,'b':2} +2 {'a':1,'c':3} +3 {'b':1,'a':2} +S7: ORDER BY with UInt64 keys +2 {1:'x',100:'y'} +3 {50:'z'} +1 {100:'a',1:'b'} +S8: ORDER BY compact parts +4 {'a':1,'b':1} +1 {'a':1,'b':2} +2 {'a':1,'c':3} +3 {'b':1,'a':2} diff --git a/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.sql b/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.sql new file mode 100644 index 000000000000..21506bd102e3 --- /dev/null +++ b/tests/queries/0_stateless/04410_map_buckets_order_by_and_comparison.sql @@ -0,0 +1,201 @@ +-- Test: ORDER BY, equality, DISTINCT, GROUP BY, min/max on bucketed Map columns. +-- These operations depend on ColumnMap::compareAt (positional comparison), +-- which is broken without the key order preservation fix. + +-- Section 1: ORDER BY on Map column +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'a':1, 'b':2}), + (2, {'a':1, 'c':3}), + (3, {'b':1, 'a':2}), + (4, {'a':1, 'b':1}); + +SELECT 'S1: ORDER BY map column'; +SELECT id, m FROM t ORDER BY m, id; +DROP TABLE t; + +-- Section 2: Equality of semantically identical maps +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'a':1, 'b':2, 'c':3}), + (2, {'a':1, 'b':2, 'c':3}); + +SELECT 'S2: equal maps compare equal'; +SELECT count() FROM t WHERE m = (SELECT m FROM t WHERE id = 1 LIMIT 1); +DROP TABLE t; + +-- Section 3: DISTINCT on Map column +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'z':1, 'a':2}), + (2, {'z':1, 'a':2}), + (3, {'a':2, 'z':1}), + (4, {'z':1, 'a':3}); + +SELECT 'S3: DISTINCT on map column'; +SELECT DISTINCT m FROM t ORDER BY m; +DROP TABLE t; + +-- Section 4: GROUP BY on Map column +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'z':1, 'a':2}), + (2, {'z':1, 'a':2}), + (3, {'x':10, 'y':20}), + (4, {'x':10, 'y':20}), + (5, {'x':10, 'y':20}); + +SELECT 'S4: GROUP BY map column'; +SELECT m, count() AS cnt FROM t GROUP BY m ORDER BY cnt, m; +DROP TABLE t; + +-- Section 5: min/max on Map column +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'b':1, 'a':2}), + (2, {'a':1, 'b':2}), + (3, {'c':1}); + +SELECT 'S5: min/max on map column'; +SELECT min(m), max(m) FROM t; +DROP TABLE t; + +-- Section 6: ORDER BY after OPTIMIZE FINAL +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, {'a':1, 'b':2}); +INSERT INTO t VALUES (2, {'a':1, 'c':3}); +INSERT INTO t VALUES (3, {'b':1, 'a':2}); +INSERT INTO t VALUES (4, {'a':1, 'b':1}); + +OPTIMIZE TABLE t FINAL; + +SELECT 'S6: ORDER BY after merge'; +SELECT id, m FROM t ORDER BY m, id; +DROP TABLE t; + +-- Section 7: ORDER BY with UInt64 keys +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(UInt64, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = 1, + min_rows_for_wide_part = 1, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {100:'a', 1:'b'}), + (2, {1:'x', 100:'y'}), + (3, {50:'z'}); + +SELECT 'S7: ORDER BY with UInt64 keys'; +SELECT id, m FROM t ORDER BY m, id; +DROP TABLE t; + +-- Section 8: Compact parts — ORDER BY correctness +DROP TABLE IF EXISTS t; +CREATE TABLE t (id UInt64, m Map(String, UInt64)) +ENGINE = MergeTree ORDER BY id +SETTINGS + map_serialization_version = 'with_buckets', + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 4, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 0, + min_bytes_for_wide_part = '200G', + min_rows_for_wide_part = 1000000, + index_granularity = 8192, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES + (1, {'a':1, 'b':2}), + (2, {'a':1, 'c':3}), + (3, {'b':1, 'a':2}), + (4, {'a':1, 'b':1}); + +SELECT 'S8: ORDER BY compact parts'; +SELECT id, m FROM t ORDER BY m, id; +DROP TABLE t; diff --git a/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.reference b/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.reference new file mode 100644 index 000000000000..ebef93a50827 --- /dev/null +++ b/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.reference @@ -0,0 +1,18 @@ +S1: no injection +14 +S1: with injection +14 +S2: ReplacingMergeTree FINAL +14 +S3: composite PK with injection +14 +S4: composite PK + FINAL +14 +S5: Tuple(Map, UInt32) PK with injection +14 +S6: Tuple(Map, UInt32) PK + FINAL +14 +S7: mapKeys PK with injection +20 +S8: m.keys PK with injection +20 diff --git a/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.sql b/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.sql new file mode 100644 index 000000000000..f7bb8065083e --- /dev/null +++ b/tests/queries/0_stateless/04411_map_buckets_parts_splitter_row_drop.sql @@ -0,0 +1,205 @@ +-- Tags: no-object-storage +-- (no-object-storage: with_buckets writes many small per-bucket files; on S3 under ASan this +-- can time out. The splitter bug is storage-agnostic, so local-only coverage loses nothing.) + +-- Regression test for parallel reads of Map primary key with with_buckets serialization. +-- +-- When a Map column is the primary key and the part uses with_buckets serialization, +-- the primary key index stores Map values in insertion order, while without the key +-- order preservation fix, data files would store keys reordered by bucket index. +-- The PartsSplitter boundary calculation compares index values against actual data rows; +-- under positional ColumnMap::compareAt, mismatched key order causes +-- FilterSortedStreamByRange to drop rows. +-- +-- The MapBucketIndexes fix ensures data files preserve the original key insertion order, +-- so the index and data agree and PartsSplitter works correctly. + +-- Section 1: Basic Map primary key with PartsSplitter injection +DROP TABLE IF EXISTS t; + +CREATE TABLE t (a Map(String, Array(UInt8))) +ENGINE = MergeTree() ORDER BY a +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (map('k1', [1,2,3], 'k2', [4,5,6])), (map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k2', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S1: no injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0; + +SELECT 'S1: with injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 1, max_threads = 4; + +DROP TABLE t; + +-- Section 2: ReplacingMergeTree FINAL with Map primary key +DROP TABLE IF EXISTS t; + +CREATE TABLE t (a Map(String, Array(UInt8))) +ENGINE = ReplacingMergeTree() ORDER BY a +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (map('k1', [1,2,3], 'k2', [4,5,6])), (map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k2', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S2: ReplacingMergeTree FINAL'; +SELECT count() FROM t FINAL + SETTINGS max_threads = 4, split_parts_ranges_into_intersecting_and_non_intersecting_final = 0; + +DROP TABLE t; + +-- Section 3: Composite primary key (id, Map) +DROP TABLE IF EXISTS t; + +CREATE TABLE t (id UInt32, m Map(String, Array(UInt8))) +ENGINE = MergeTree() ORDER BY (id, m) +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, map('k1', [1,2,3], 'k2', [4,5,6])), (1, map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT 1, map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT 1, map('k2', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S3: composite PK with injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 1, max_threads = 4; + +DROP TABLE t; + +-- Section 4: Composite PK + ReplacingMergeTree FINAL +DROP TABLE IF EXISTS t; + +CREATE TABLE t (id UInt32, m Map(String, Array(UInt8))) +ENGINE = ReplacingMergeTree() ORDER BY (id, m) +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (1, map('k1', [1,2,3], 'k2', [4,5,6])), (1, map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT 1, map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT 1, map('k2', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S4: composite PK + FINAL'; +SELECT count() FROM t FINAL + SETTINGS max_threads = 4, split_parts_ranges_into_intersecting_and_non_intersecting_final = 0; + +DROP TABLE t; + +-- Section 5: Tuple(Map, UInt32) primary key +DROP TABLE IF EXISTS t; + +CREATE TABLE t (c Tuple(Map(String, Array(UInt8)), UInt32)) +ENGINE = MergeTree() ORDER BY c +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (tuple(map('k1', [1,2,3], 'k2', [4,5,6]), 1)), (tuple(map('k0', [], 'k1', [100,20,90]), 1)); +INSERT INTO t SELECT tuple(map('k1', [number, number + 2, number * 2]), 1) FROM numbers(6); +INSERT INTO t SELECT tuple(map('k2', [number, number + 2, number * 2]), 1) FROM numbers(6); + +SELECT 'S5: Tuple(Map, UInt32) PK with injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 1, max_threads = 4; + +DROP TABLE t; + +-- Section 6: Tuple(Map, UInt32) PK + ReplacingMergeTree FINAL +DROP TABLE IF EXISTS t; + +CREATE TABLE t (c Tuple(Map(String, Array(UInt8)), UInt32)) +ENGINE = ReplacingMergeTree() ORDER BY c +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (tuple(map('k1', [1,2,3], 'k2', [4,5,6]), 1)), (tuple(map('k0', [], 'k1', [100,20,90]), 1)); +INSERT INTO t SELECT tuple(map('k1', [number, number + 2, number * 2]), 1) FROM numbers(6); +INSERT INTO t SELECT tuple(map('k2', [number, number + 2, number * 2]), 1) FROM numbers(6); + +SELECT 'S6: Tuple(Map, UInt32) PK + FINAL'; +SELECT count() FROM t FINAL + SETTINGS max_threads = 4, split_parts_ranges_into_intersecting_and_non_intersecting_final = 0; + +DROP TABLE t; + +-- Section 7: mapKeys(m) as primary key +DROP TABLE IF EXISTS t; + +CREATE TABLE t (m Map(String, Array(UInt8))) +ENGINE = MergeTree() ORDER BY mapKeys(m) +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (map('k1', [1,2,3], 'k2', [4,5,6])), (map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k2', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k3', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S7: mapKeys PK with injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 1, max_threads = 4; + +DROP TABLE t; + +-- Section 8: m.keys as primary key +DROP TABLE IF EXISTS t; + +CREATE TABLE t (m Map(String, Array(UInt8))) +ENGINE = MergeTree() ORDER BY m.keys +SETTINGS + min_bytes_for_wide_part = 0, + map_serialization_version_for_zero_level_parts = 'with_buckets', + max_buckets_in_map = 11, + map_buckets_strategy = 'constant', + map_buckets_min_avg_size = 2, + serialization_info_version = 'with_types'; + +INSERT INTO t VALUES (map('k1', [1,2,3], 'k2', [4,5,6])), (map('k0', [], 'k1', [100,20,90])); +INSERT INTO t SELECT map('k1', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k2', [number, number + 2, number * 2]) FROM numbers(6); +INSERT INTO t SELECT map('k3', [number, number + 2, number * 2]) FROM numbers(6); + +SELECT 'S8: m.keys PK with injection'; +SELECT count() FROM t + SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 1, max_threads = 4; + +DROP TABLE t; From 3aebd8dcf061f06dbe9a21f0c619d7fc42b134bf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 2 Aug 2026 17:04:22 +0000 Subject: [PATCH 78/86] Backport #112943 to 26.6: Fix reading from an encrypted disk with O_DIRECT --- ...ynchronousReadBufferFromFileDescriptor.cpp | 7 +++- src/IO/ReadBufferFromFileDescriptor.cpp | 6 +++- ...69_encrypted_disk_direct_io_seek.reference | 1 + .../04669_encrypted_disk_direct_io_seek.sh | 36 +++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.reference create mode 100755 tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.sh diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index d83ea3a10006..e314521b83ba 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -278,7 +278,12 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) "Logical error in AsynchronousReadBufferFromFileDescriptor, bytes_to_ignore ({}" ") >= internal_buffer.size() ({})", bytes_to_ignore, internal_buffer.size()); - return seek_pos; + /// Return the position we are actually at, not `seek_pos`. With O_DIRECT (`required_alignment > 1`) + /// `seek_pos` is `new_pos` rounded down to the alignment, and the difference is accounted for by + /// `bytes_to_ignore` (which `getPosition` includes), so the buffer is positioned at `new_pos`. + /// Returning `seek_pos` would break callers that take the returned value as the new position + /// (see `ReadBufferFromEncryptedFile`). + return static_cast(new_pos); } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index 06dc994be490..92ff0596320e 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -260,7 +260,11 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) if (offset_after_seek_pos > 0) ignore(offset_after_seek_pos); - return seek_pos; + /// Return the position we are actually at, not `seek_pos`. With O_DIRECT (`required_alignment > 1`) + /// `seek_pos` is `new_pos` rounded down to the alignment, and the difference has just been skipped + /// by `ignore` above, so the buffer is positioned at `new_pos`. Returning `seek_pos` would break + /// callers that take the returned value as the new position (see `ReadBufferFromEncryptedFile`). + return static_cast(new_pos); } /// NOLINTEND(readability-else-after-return) } diff --git a/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.reference b/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.reference new file mode 100644 index 000000000000..c5009e411e54 --- /dev/null +++ b/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.reference @@ -0,0 +1 @@ +200000 19999900000 8215679822310692128 diff --git a/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.sh b/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.sh new file mode 100755 index 000000000000..b379df39fc95 --- /dev/null +++ b/tests/queries/0_stateless/04669_encrypted_disk_direct_io_seek.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-object-storage, no-replicated-database, no-shared-merge-tree +# Tag no-fasttest: depends on OpenSSL +# Tag no-object-storage: O_DIRECT applies to local disks only +# Tag no-replicated-database, no-shared-merge-tree: custom disk + +# Reading a Compact part from an `encrypted` disk with O_DIRECT used to fail with +# "ReadBufferFromEncryptedFile: Wrong file position ... in the inner buffer", because +# `ReadBufferFromFileDescriptor::seek` returned the offset rounded down to the O_DIRECT +# alignment instead of the position the buffer was left at. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiline -q """ +DROP TABLE IF EXISTS t_encrypted_direct_io SYNC; + +CREATE TABLE t_encrypted_direct_io (id UInt64, a UInt64, b UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS disk = disk( + type = encrypted, + disk = disk(type = local, path = '${CLICKHOUSE_DISKS_FILES}/${CLICKHOUSE_DATABASE}_encrypted_direct_io/'), + algorithm = 'AES_128_CTR', + key_hex = '00112233445566778899aabbccddeeff'), + min_bytes_for_wide_part = '1G'; + +-- A single Compact part large enough that reading the second column has to seek to an +-- offset that is not a multiple of the O_DIRECT alignment. +INSERT INTO t_encrypted_direct_io SELECT number, number, sipHash64(number) FROM numbers(200000); + +SELECT count(), sum(a), sum(b) FROM t_encrypted_direct_io +SETTINGS min_bytes_to_use_direct_io = 1, max_threads = 1; + +DROP TABLE t_encrypted_direct_io SYNC; +""" From 93f4c309b689d28bb86b0300c37747d0efea8b38 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 3 Aug 2026 09:07:40 +0000 Subject: [PATCH 79/86] Backport #112413 to 26.6: Bump aws-c-http library to v0.11.0 --- contrib/aws-c-http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/aws-c-http b/contrib/aws-c-http index a9745ea9998f..8aefd899fc32 160000 --- a/contrib/aws-c-http +++ b/contrib/aws-c-http @@ -1 +1 @@ -Subproject commit a9745ea9998f679cd7456e7d23cc8820e38c97d4 +Subproject commit 8aefd899fc3210bfd0e3fd414011a3cb708bf6e4 From 2e545a2ac5a782187ec10054ed4b89ba20cf9057 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 3 Aug 2026 14:39:58 +0000 Subject: [PATCH 80/86] Backport #108557 to 26.6: Increase restart timeout in test_keeper_snapshot_chunked_transfer recover tests --- ci/jobs/scripts/integration_tests_configs.py | 6 ++++++ .../test.py | 18 ++++++++++++------ .../test_concurrent.py | 8 +++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ci/jobs/scripts/integration_tests_configs.py b/ci/jobs/scripts/integration_tests_configs.py index ca5dbcf22483..b12669740a3a 100644 --- a/ci/jobs/scripts/integration_tests_configs.py +++ b/ci/jobs/scripts/integration_tests_configs.py @@ -32,6 +32,12 @@ class TC: ), TC("test_random_inserts/", False, "standard replicated inserts test; cluster is fully isolated"), TC("test_server_overload/", True, "uses taskset to pin ClickHouse to specific CPU cores; sensitive to concurrent CPU load"), + TC( + "test_keeper_snapshot_chunked_transfer/", + False, + "18-node Keeper+S3 cluster; concurrent --dist=each copies OOM the ASAN runner", + dist_each_sequential=True, + ), TC("test_storage_kafka/", False, "each cluster has its own Kafka container and Docker network"), TC("test_storage_rabbitmq/", False, "each cluster has its own RabbitMQ container; tests use unique exchange/db names"), TC("test_storage_kerberized_kafka/", False, "each cluster has its own Kafka container and Docker network"), diff --git a/tests/integration/test_keeper_snapshot_chunked_transfer/test.py b/tests/integration/test_keeper_snapshot_chunked_transfer/test.py index d2b795d2c06d..0255649fe2d3 100644 --- a/tests/integration/test_keeper_snapshot_chunked_transfer/test.py +++ b/tests/integration/test_keeper_snapshot_chunked_transfer/test.py @@ -82,6 +82,12 @@ def started_cluster(): CHUNK_SIZE = 4096 # matches snapshot_transfer_chunk_size in small-chunk configs +# Recovery here means replaying raft logs and installing a snapshot (over S3, with a +# 1 KiB read buffer) before the node accepts connections. Under msan/tsan that legitimately +# took ~95 s in CI, so the wait must be generous: start_clickhouse returns as soon as the +# server is ready, so a large upper bound costs nothing on fast runs. +RESTART_TIMEOUT_SECONDS = 180 + CHUNKED_TRANSFER_PARAMS = [ pytest.param({"leader": node1, "middle": node2, "lagging": node3, "disk_type": "local"}, id="local_disk"), pytest.param({"leader": node7, "middle": node8, "lagging": node9, "disk_type": "remote"}, id="remote_disk"), @@ -122,7 +128,7 @@ def test_recover_from_snapshot_with_chunked_transfer(started_cluster, nodes): node_lagging.stop_clickhouse(kill=True) fill_test_tree(leader_zk, prefix) - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node_lagging) received = get_received_snapshot_info(node_lagging, kill_time) assert received is not None @@ -179,7 +185,7 @@ def _drop_rule(): user="root", ) try: - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) node_lagging.query("SYSTEM ENABLE FAILPOINT keeper_save_snapshot_pause_mid_transfer") except Exception: _drop_rule() @@ -229,7 +235,7 @@ def _drop_rule(): ).strip() assert tmp_snapshot_path, "No tmp_snapshot file on disk after killing mid-transfer" - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node_lagging) lagging_zk = keeper_utils.get_fake_zk(cluster, node_lagging.name) lagging_zk.sync(prefix) # wait until all committed entries (including snapshot) are applied @@ -287,7 +293,7 @@ def test_recover_with_chunk_size_larger_than_snapshot(started_cluster, nodes): leader_zk = keeper_utils.get_fake_zk(cluster, node_leader.name) fill_test_tree(leader_zk, prefix) - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node_lagging) received = get_received_snapshot_info(node_lagging, kill_time) @@ -324,7 +330,7 @@ def test_recover_after_s3_read_error_during_transfer(started_cluster): node_leader.query("SYSTEM ENABLE FAILPOINT s3_read_buffer_throw_expired_token") try: - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node_lagging) received = get_received_snapshot_info(node_lagging, kill_time, timeout=30) @@ -360,7 +366,7 @@ def test_recover_from_snapshot_sent_by_old_leader(started_cluster, nodes): leader_zk = keeper_utils.get_fake_zk(cluster, node_old_leader.name) fill_test_tree(leader_zk, prefix) - node_lagging.start_clickhouse(20) + node_lagging.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node_lagging) received = get_received_snapshot_info(node_lagging, kill_time) diff --git a/tests/integration/test_keeper_snapshot_chunked_transfer/test_concurrent.py b/tests/integration/test_keeper_snapshot_chunked_transfer/test_concurrent.py index d83fe911a882..a2230265eab3 100644 --- a/tests/integration/test_keeper_snapshot_chunked_transfer/test_concurrent.py +++ b/tests/integration/test_keeper_snapshot_chunked_transfer/test_concurrent.py @@ -28,6 +28,12 @@ _small_buf_cfg = os.path.join(configs_dir, "small_remote_buf_user.xml") +# Recovery here means replaying raft logs and installing a snapshot (over S3, with a +# 1 KiB read buffer) before the node accepts connections. Under msan/tsan that legitimately +# took ~95 s in CI, so the wait must be generous: start_clickhouse returns as soon as the +# server is ready, so a large upper bound costs nothing on fast runs. +RESTART_TIMEOUT_SECONDS = 180 + cluster = ClickHouseCluster(__file__) node_conc1 = cluster.add_instance("node_conc1", main_configs=["configs/enable_keeper_conc1.xml"], stay_alive=True, with_remote_database_disk=False) @@ -78,7 +84,7 @@ def test_concurrent_followers_fetch_snapshot(started_cluster, nodes): fill_test_tree(leader_zk, prefix) def start_and_wait(node): - node.start_clickhouse(20) + node.start_clickhouse(RESTART_TIMEOUT_SECONDS) keeper_utils.wait_until_connected(cluster, node) with concurrent.futures.ThreadPoolExecutor(max_workers=len(lagging)) as pool: From 9115574f17833535bb8a327c913a8acd12f1353d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 3 Aug 2026 15:51:42 +0000 Subject: [PATCH 81/86] Backport #112594 to 26.6: Bump `simdjson` from v4.2.4 to v4.6.5 --- contrib/simdjson | 2 +- contrib/simdjson-cmake/CMakeLists.txt | 11 +++++++++++ src/Common/JSONParsers/SimdJSONParser.h | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/contrib/simdjson b/contrib/simdjson index 980f2ad3afb1..0a851a64cd98 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit 980f2ad3afb12729157b44ed33d2bac41b67b54b +Subproject commit 0a851a64cd984e9e1a6cab93b6e773aa3f4dc30d diff --git a/contrib/simdjson-cmake/CMakeLists.txt b/contrib/simdjson-cmake/CMakeLists.txt index e74f8a5b9a9e..e98fcdf23d99 100644 --- a/contrib/simdjson-cmake/CMakeLists.txt +++ b/contrib/simdjson-cmake/CMakeLists.txt @@ -29,6 +29,17 @@ if (NOT (ARCH_AMD64 AND X86_ARCH_LEVEL VERSION_GREATER_EQUAL 3)) target_compile_options(_simdjson PRIVATE -DSIMDJSON_IMPLEMENTATION_FALLBACK=1) endif() +# On LoongArch, simdjson's CPU dispatch reads the HWCAP_LOONGARCH_LSX / HWCAP_LOONGARCH_LASX +# bits (see contrib/simdjson/src/internal/isadetection.h). These are kernel UAPI macros defined +# in , but simdjson only includes , which pulls in glibc's . +# The glibc in our LoongArch sysroot is old enough that its defines no bits for this +# architecture ("No bits defined for this architecture."), so the macros are unreachable and the +# build fails. Force-include the kernel header so the macros are always available. The header has an +# include guard and its values match glibc's, so this is a benign no-op once the sysroot is updated. +if(ARCH_LOONGARCH64) + target_compile_options(_simdjson PRIVATE -include asm/hwcap.h) +endif() + # On ppc64le, ClickHouse globally defines __SSE2__ to enable SSE-to-AltiVec emulation. # This causes simdjson's experimental json_string_builder to activate its SSE2 code path, # which fails to compile due to __m128i vs __m128i_u type mismatch in the PPC SSE wrappers. diff --git a/src/Common/JSONParsers/SimdJSONParser.h b/src/Common/JSONParsers/SimdJSONParser.h index 5f2a8faacb28..53d30b841a86 100644 --- a/src/Common/JSONParsers/SimdJSONParser.h +++ b/src/Common/JSONParsers/SimdJSONParser.h @@ -208,6 +208,14 @@ class SimdJSONElementFormatter format.nullAtom(); break; } + /// `simdjson` only produces `BIGINT` when the parser is configured to store integers + /// that do not fit in 64 bits as raw digit strings. We do not enable that option, so + /// `simdjson` reports `BIGINT_ERROR` for such numbers instead and this case is + /// unreachable; it is handled to keep the switch exhaustive. + case simdjson::dom::element_type::BIGINT: { + format.string(value.get_bigint().value_unsafe()); + break; + } case simdjson::dom::element_type::ARRAY: { append(value.get_array().value_unsafe()); break; @@ -289,6 +297,10 @@ struct SimdJSONParser case simdjson::dom::element_type::OBJECT: return ElementType::OBJECT; case simdjson::dom::element_type::BOOL: return ElementType::BOOL; case simdjson::dom::element_type::NULL_VALUE: return ElementType::NULL_VALUE; + /// Unreachable unless the parser is told to store big integers as raw digit + /// strings, which we do not do. Reported as a string because that is how + /// `simdjson` exposes the value (`element::get_bigint`). + case simdjson::dom::element_type::BIGINT: return ElementType::STRING; } } From 90f5a0ed4c4fa3e297aec1255ddf3813278e4a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 3 Aug 2026 15:53:10 +0000 Subject: [PATCH 82/86] Drop dist_each_sequential entry, unsupported in 26.6 The dist_each_sequential field and force_heavy_modules_sequential are master-only; 26.6's TC has no such field, so every integration job died with TypeError at import. The 26.6 workflows run no --dist=each flaky job, so the entry would be a no-op there anyway. --- ci/jobs/scripts/integration_tests_configs.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/jobs/scripts/integration_tests_configs.py b/ci/jobs/scripts/integration_tests_configs.py index b12669740a3a..ca5dbcf22483 100644 --- a/ci/jobs/scripts/integration_tests_configs.py +++ b/ci/jobs/scripts/integration_tests_configs.py @@ -32,12 +32,6 @@ class TC: ), TC("test_random_inserts/", False, "standard replicated inserts test; cluster is fully isolated"), TC("test_server_overload/", True, "uses taskset to pin ClickHouse to specific CPU cores; sensitive to concurrent CPU load"), - TC( - "test_keeper_snapshot_chunked_transfer/", - False, - "18-node Keeper+S3 cluster; concurrent --dist=each copies OOM the ASAN runner", - dist_each_sequential=True, - ), TC("test_storage_kafka/", False, "each cluster has its own Kafka container and Docker network"), TC("test_storage_rabbitmq/", False, "each cluster has its own RabbitMQ container; tests use unique exchange/db names"), TC("test_storage_kerberized_kafka/", False, "each cluster has its own Kafka container and Docker network"), From c0128ae3703d394e677d08129878c1e53bf57158 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 3 Aug 2026 17:33:50 +0000 Subject: [PATCH 83/86] Backport #109724 to 26.6: DatabaseDataLake: Restore onelake_bearer_token, lazy init for internal create --- docs/en/engines/database-engines/datalake.md | 3 +- src/Databases/DataLake/DataLakeConstants.h | 1 + src/Databases/DataLake/DatabaseDataLake.cpp | 34 +++++- .../DataLake/DatabaseDataLakeSettings.cpp | 1 + src/Databases/DataLake/RestCatalog.cpp | 50 +++++--- src/Databases/DataLake/RestCatalog.h | 7 ++ src/Databases/DatabaseFactory.cpp | 9 +- src/Databases/DatabaseFactory.h | 7 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- .../ObjectStorage/Azure/Configuration.cpp | 25 +++- .../ObjectStorage/Azure/Configuration.h | 6 +- .../helpers/catalog_manager_onelake.py | 46 ++++++- tests/integration/test_e2e_catalogs/test.py | 114 ++++++++++++++++++ 13 files changed, 271 insertions(+), 34 deletions(-) diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index b37fc38f790d..e21bbb91d8a3 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -81,4 +81,5 @@ SETTINGS onelake_client_secret = client_secret; SHOW TABLES IN database_name; SELECT count() from database_name.table_name; -``` \ No newline at end of file +``` +To authenticate without sharing a client secret, set `onelake_bearer_token` to a pre-obtained bearer token (scoped to `https://storage.azure.com`) instead of `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the database must be recreated after it expires. diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index 0b228bf310ec..b65827669342 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -29,6 +29,7 @@ static inline std::unordered_map SETTINGS_TO_HIDE = {"aws_secret_access_key", DEFAULT_MASKING_RULE}, /// OneLake credentials {"onelake_client_secret", DEFAULT_MASKING_RULE}, + {"onelake_bearer_token", DEFAULT_MASKING_RULE}, /// Google credentials {"google_adc_client_secret", DEFAULT_MASKING_RULE}, {"google_adc_refresh_token", DEFAULT_MASKING_RULE}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 12fbb051ba4a..ddddefbb3732 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -72,6 +72,7 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString onelake_tenant_id; extern const DatabaseDataLakeSettingsString onelake_client_id; extern const DatabaseDataLakeSettingsString onelake_client_secret; + extern const DatabaseDataLakeSettingsString onelake_bearer_token; extern const DatabaseDataLakeSettingsBool onelake_use_blob_endpoint; extern const DatabaseDataLakeSettingsString dlf_access_key_id; extern const DatabaseDataLakeSettingsString dlf_access_key_secret; @@ -139,9 +140,9 @@ DatabaseDataLake::DatabaseDataLake( , db_uuid(uuid) { validateSettings(); - /// On ATTACH (server startup) defer catalog construction to first use: building it can - /// perform network I/O or credential validation that must not block startup. On CREATE - /// build eagerly so misconfiguration is reported immediately. + /// On ATTACH (server startup / user `ATTACH DATABASE`) or internal creates (restore), + /// defer catalog construction to first use: building it can perform network I/O or credential validation + /// that must not block startup. On CREATE build eagerly so misconfiguration is reported immediately. if (!lazy_init) { std::lock_guard lock(catalog_mutex); @@ -206,6 +207,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::onelake_tenant_id].value, settings[DatabaseDataLakeSetting::onelake_client_id].value, settings[DatabaseDataLakeSetting::onelake_client_secret].value, + settings[DatabaseDataLakeSetting::onelake_bearer_token].value, settings[DatabaseDataLakeSetting::auth_scope].value, settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, @@ -654,6 +656,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con rest_catalog->getClientId(), rest_catalog->getClientSecret(), rest_catalog->getTenantId(), + rest_catalog->getBearerToken(), settings[DatabaseDataLakeSetting::onelake_use_blob_endpoint].value ); #else @@ -1083,6 +1086,23 @@ void registerDatabaseDataLake(DatabaseFactory & factory) "To allow its usage, enable setting allow_database_iceberg"); } + if (!args.create_query.attach && catalog_type == DatabaseDataLakeCatalogType::ICEBERG_ONELAKE) + { + /// Require exactly one auth method: a bearer token, or a client id + secret pair. + const bool has_bearer = !database_settings[DatabaseDataLakeSetting::onelake_bearer_token].value.empty(); + const bool has_client_id = !database_settings[DatabaseDataLakeSetting::onelake_client_id].value.empty(); + const bool has_client_secret = !database_settings[DatabaseDataLakeSetting::onelake_client_secret].value.empty(); + + const bool has_client_pair = has_client_id && has_client_secret; + bool has_exactly_one_method = has_bearer != has_client_pair; + bool has_conflicting_fields = has_client_id != has_client_secret; + + if (!has_exactly_one_method || has_conflicting_fields) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "OneLake catalog requires exactly one authentication method: either `onelake_bearer_token` " + "or both `onelake_client_id` and `onelake_client_secret`"); + } + engine_func->name = "Iceberg"; break; } @@ -1149,7 +1169,9 @@ void registerDatabaseDataLake(DatabaseFactory & factory) database_engine_define->clone(), std::move(engine_for_tables), args.uuid, - /*lazy_init=*/args.create_query.attach); + /// Internal creates (`RESTORE DATABASE`) shouldn't do network I/O. + /// We don't want an unreachable or unauthorized catalog to block replica startup. + /*lazy_init=*/args.create_query.attach || args.internal); }; /// TODO: DataLakeCatalog is polymorphic — underlying source (S3, Azure, HDFS, etc.) depends /// on the catalog type chosen at runtime. Consider adding source_access_type once a mechanism @@ -1236,6 +1258,10 @@ SETTINGS SHOW TABLES IN database_name; SELECT count() from database_name.table_name; ``` + To authenticate without sharing a client secret, set `onelake_bearer_token` to a pre-obtained + bearer token (scoped to https://storage.azure.com) instead of + `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the + database must be recreated after it expires. )DOCS_MD", .syntax = "ENGINE = DataLakeCatalog('catalog_url'[, 'user', 'password']) SETTINGS catalog_type = '...'", .related = {}}); diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 969b0769d13a..9e55a5f792f0 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -36,6 +36,7 @@ namespace ErrorCodes DECLARE(String, onelake_tenant_id, "", "Tenant id from azure", 0) \ DECLARE(String, onelake_client_id, "", "Client id from azure", 0) \ DECLARE(String, onelake_client_secret, "", "Client secret from azure", 0) \ + DECLARE(String, onelake_bearer_token, "", "Pre-obtained bearer token for OneLake, scoped to https://storage.azure.com. The token is static and not refreshed, so a long-lived database must be recreated once it expires", 0) \ DECLARE(Bool, onelake_use_blob_endpoint, true, "Use the Blob endpoint (.blob.fabric.microsoft.com) for OneLake. When disabled, the DFS endpoint (.dfs.fabric.microsoft.com) is used instead", 0) \ DECLARE(String, google_project_id, "", "Google Cloud project ID for BigLake. Required for BigLake catalog. Used in x-goog-user-project header. If not set and google_adc_quota_project_id is provided, it latter will be used", 0) \ DECLARE(String, google_service_account, "", "Google Cloud service account email for metadata service authentication. Default: 'default'. Only used when ADC credentials are not provided", 0) \ diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 28c1195082e4..b51ff068d9ae 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -187,14 +187,7 @@ RestCatalog::RestCatalog( else if (!auth_header_.empty()) { auth_header = parseAuthHeader(auth_header_); - /// `registerDatabaseDataLake` validates `auth_header` on CREATE only, so that a database - /// persisted with a forbidden or malformed header does not block server startup on ATTACH. - /// The catalog is built lazily on first use instead; this is where the user-provided - /// `auth_header` first becomes a header sent to the catalog, so enforce `http_forbid_headers` - /// here, before `loadConfig` issues any request. Mirrors the CREATE-path check: a copy is - /// validated and the original parsed header is kept. - DB::HTTPHeaderEntries header_to_check{auth_header.value()}; - getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); + validateAuthHeaders(auth_header.value()); } config = loadConfig(); } @@ -255,6 +248,18 @@ void RestCatalog::parseCatalogConfigurationSettings(const Poco::JSON::Object::Pt result.default_base_location = object->get("default-base-location").extract(); } +void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const +{ + /// `registerDatabaseDataLake` validates `auth_header` on CREATE only, so that a database + /// persisted with a forbidden or malformed header does not block server startup on ATTACH. + /// The catalog is built lazily on first use instead; this is where the user-provided + /// `auth_header` first becomes a header sent to the catalog, so enforce `http_forbid_headers` + /// here, before `loadConfig` issues any request. Mirrors the CREATE-path check: a copy is + /// validated and the original parsed header is kept. + DB::HTTPHeaderEntries header_to_check{header}; + getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); +} + DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(bool update_token) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, @@ -294,6 +299,7 @@ OneLakeCatalog::OneLakeCatalog( const std::string & onelake_tenant_id, const std::string & onelake_client_id, const std::string & onelake_client_secret, + const std::string & bearer_token_, const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, @@ -301,17 +307,33 @@ OneLakeCatalog::OneLakeCatalog( : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, context_) , tenant_id(onelake_tenant_id) { - client_id = onelake_client_id; - client_secret = onelake_client_secret; - update_token_if_expired = true; - // Get token before loading config so getAuthHeaders() can work - if (!client_id.empty() && !client_secret.empty()) + if (!bearer_token_.empty()) { - access_token.set(std::make_unique(retrieveAccessToken())); + /// Pre-obtained token scoped to https://storage.azure.com. Used for both catalog header + /// and Azure Blob access. Does not support refresh. + bearer_token = bearer_token_; + auth_header = DB::HTTPHeaderEntry("Authorization", "Bearer " + bearer_token); + validateAuthHeaders(auth_header.value()); + } + else + { + client_id = onelake_client_id; + client_secret = onelake_client_secret; + update_token_if_expired = true; + // Get token before loading config so getAuthHeaders() can work + if (!client_id.empty() && !client_secret.empty()) + { + access_token.set(std::make_unique(retrieveAccessToken())); + } } config = loadConfig(); } +String OneLakeCatalog::getBearerToken() const +{ + return bearer_token; +} + AccessToken RestCatalog::retrieveAccessToken() const { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 982475ee2c96..4d960da2f013 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -164,6 +164,8 @@ class RestCatalog : public ICatalog, public DB::WithContext Config loadConfig(); virtual DB::HTTPHeaderEntries getAuthHeaders(bool update_token) const; + + void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; static void parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result); void sendRequest( @@ -186,6 +188,7 @@ class OneLakeCatalog : public RestCatalog const std::string & onelake_tenant_id, const std::string & onelake_client_id, const std::string & onelake_client_secret, + const std::string & bearer_token_, const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, @@ -198,9 +201,13 @@ class OneLakeCatalog : public RestCatalog String getTenantId() const { return tenant_id; } + String getBearerToken() const; + protected: /// Parameters for OneLake OAuth. const std::string tenant_id; + /// Set from `onelake_bearer_token`. + String bearer_token; }; class BigLakeCatalog : public RestCatalog diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index a893d8f39d25..e48c2dd128b2 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -96,7 +96,7 @@ void DatabaseFactory::validate(const ASTCreateQuery & create_query) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name); } -DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode) +DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode, bool internal) { const auto engine_name = create.storage->engine->name; /// check if the database engine is a valid one before proceeding @@ -113,7 +113,7 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m validate(create); cckMetadataPathForOrdinary(create, metadata_path); - DatabasePtr impl = getImpl(create, metadata_path, context, mode); + DatabasePtr impl = getImpl(create, metadata_path, context, mode, internal); if (impl && context->hasQueryContext() && context->getSettingsRef()[Setting::log_queries]) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Database, impl->getEngineName()); @@ -145,7 +145,7 @@ bool DatabaseFactory::isDatabaseExternal(const String & engine_name) const return it->second.features.is_external; } -DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode) +DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode, bool internal) { auto * storage = create.storage; const String & database_name = create.getDatabase(); @@ -164,7 +164,8 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String .metadata_path = metadata_path, .uuid = create.uuid, .context = context, - .mode = mode}; + .mode = mode, + .internal = internal}; // creator_fn creates and returns a DatabasePtr with the supplied arguments auto creator_fn = database_engines.at(engine_name).creator_fn; diff --git a/src/Databases/DatabaseFactory.h b/src/Databases/DatabaseFactory.h index aee533d85fdd..1976cde52348 100644 --- a/src/Databases/DatabaseFactory.h +++ b/src/Databases/DatabaseFactory.h @@ -45,6 +45,9 @@ class DatabaseFactory : private boost::noncopyable, public IHints<> const UUID & uuid; ContextPtr & context; LoadingStrictnessLevel mode = LoadingStrictnessLevel::CREATE; + /// True when the database is created by the server itself (e.g. loading metadata on startup) rather + /// than by a user query. Lets an engine distinguish an internal reload from a user `ATTACH DATABASE`. + bool internal = false; }; struct EngineFeatures @@ -72,7 +75,7 @@ class DatabaseFactory : private boost::noncopyable, public IHints<> Documentation documentation; }; - DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode = LoadingStrictnessLevel::CREATE); + DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode = LoadingStrictnessLevel::CREATE, bool internal = false); using DatabaseEngines = std::unordered_map; @@ -100,7 +103,7 @@ class DatabaseFactory : private boost::noncopyable, public IHints<> private: DatabaseEngines database_engines; - DatabasePtr getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode); + DatabasePtr getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context, LoadingStrictnessLevel mode, bool internal); /// validate validates the database engine that's specified in the create query for /// engine arguments, settings and table overrides. diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ced702fa3be4..83b2fce3268d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -336,7 +336,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); - DatabasePtr database = DatabaseFactory::instance().get(create, metadata_path / "", getContext(), mode); + DatabasePtr database = DatabaseFactory::instance().get(create, metadata_path / "", getContext(), mode, internal); if (create.uuid != UUIDHelpers::Nil) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index a5930cc25015..a581e388c9b8 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -878,14 +878,27 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure) { AzureStorageParsedArguments parsed_arguments; - if (!onelake_client_id.empty()) + if (is_onelake) { parsed_arguments.initializeForOneLake(engine_args, context, onelake_use_blob_endpoint); - parsed_arguments.connection_params.auth_method = std::make_shared( - onelake_tenant_id, - onelake_client_id, - onelake_client_secret - ); + if (!onelake_access_token.empty()) + { + /// Pre-obtained bearer token from `onelake_bearer_token`. + /// Use epoch as the expiry time. There is no refresh -- the database must be + /// recreated with a new token once it expires. + parsed_arguments.connection_params.auth_method = std::make_shared( + onelake_access_token, + std::chrono::system_clock::time_point{} + ); + } + else + { + parsed_arguments.connection_params.auth_method = std::make_shared( + onelake_tenant_id, + onelake_client_id, + onelake_client_secret + ); + } } else { diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index e74279ae1682..53f6a97751a4 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -125,12 +125,14 @@ class StorageAzureConfiguration : public StorageObjectStorageConfiguration ContextPtr context, bool with_structure) override; - void setInitializationAsOneLake(const String & client_id_, const String & client_secret_, const String & tenant_id_, bool use_blob_endpoint_) + void setInitializationAsOneLake(const String & client_id_, const String & client_secret_, const String & tenant_id_, const String & access_token_, bool use_blob_endpoint_) { onelake_client_id = client_id_; onelake_client_secret = client_secret_; onelake_tenant_id = tenant_id_; + onelake_access_token = access_token_; onelake_use_blob_endpoint = use_blob_endpoint_; + is_onelake = true; } protected: @@ -148,7 +150,9 @@ class StorageAzureConfiguration : public StorageObjectStorageConfiguration String onelake_client_id; String onelake_client_secret; String onelake_tenant_id; + String onelake_access_token; bool onelake_use_blob_endpoint = true; + bool is_onelake = false; void initializeFromParsedArguments(const AzureStorageParsedArguments & parsed_arguments); }; diff --git a/tests/integration/helpers/catalog_manager_onelake.py b/tests/integration/helpers/catalog_manager_onelake.py index 1da7ecc356f9..de7be456f156 100644 --- a/tests/integration/helpers/catalog_manager_onelake.py +++ b/tests/integration/helpers/catalog_manager_onelake.py @@ -204,13 +204,51 @@ def warehouse(self) -> str: def make_database_name() -> str: return f"e2e_onelake_{uuid.uuid4().hex[:8]}" + def bearer_token(self) -> str: + """Mint a real bearer token scoped to ``https://storage.azure.com``. + """ + return self._credential.get_token( + "https://storage.azure.com/.default" + ).token + + def create_db_sql_bearer( + self, database_name: str, bearer_token: str, **overrides + ) -> str: + """Build a ``CREATE DATABASE`` SQL string authenticating with a + pre-obtained bearer token via ``onelake_bearer_token``. + """ + cfg = self.config + u = overrides.get("catalog_url", cfg.catalog_url) + w = overrides.get("warehouse", self.warehouse) + return ( + f"CREATE DATABASE {database_name} ENGINE = DataLakeCatalog('{u}')\n" + f"SETTINGS\n" + f" catalog_type='onelake',\n" + f" warehouse='{w}',\n" + f" onelake_bearer_token='{bearer_token}'" + ) + + def create_catalog_bearer( + self, node, database_name: str, bearer_token: str + ) -> None: + """Drop-and-create a DataLakeCatalog database authenticating with a + pre-obtained bearer token. + + Assumes ``allow_experimental_database_iceberg`` is enabled in the + server's user config.""" + node.query( + f"DROP DATABASE IF EXISTS {database_name};\n" + + self.create_db_sql_bearer(database_name, bearer_token) + ) + def create_db_sql(self, database_name: str, **overrides) -> str: """Build a ``CREATE DATABASE`` SQL string. Uses real credentials by default; pass keyword overrides (``tenant_id``, ``client_id``, ``client_secret``, ``catalog_url``, ``oauth_server_uri``, ``warehouse``, - ``auth_scope``) to substitute individual values. + ``auth_scope``) to substitute individual values. Pass + ``bearer_token`` to also emit an ``onelake_bearer_token`` setting. """ cfg = self.config t = overrides.get("tenant_id", cfg.tenant_id) @@ -223,11 +261,17 @@ def create_db_sql(self, database_name: str, **overrides) -> str: ) w = overrides.get("warehouse", self.warehouse) a = overrides.get("auth_scope", "https://storage.azure.com/.default") + # Optional; lets a test provide both auth methods at once. + bearer = overrides.get("bearer_token") + bearer_line = ( + f" onelake_bearer_token='{bearer}',\n" if bearer is not None else "" + ) return ( f"CREATE DATABASE {database_name} ENGINE = DataLakeCatalog('{u}')\n" f"SETTINGS\n" f" catalog_type='onelake',\n" f" warehouse='{w}',\n" + f"{bearer_line}" f" onelake_tenant_id='{t}',\n" f" onelake_client_id='{c}',\n" f" onelake_client_secret='{s}',\n" diff --git a/tests/integration/test_e2e_catalogs/test.py b/tests/integration/test_e2e_catalogs/test.py index 575abc271582..703c34b0ae16 100644 --- a/tests/integration/test_e2e_catalogs/test.py +++ b/tests/integration/test_e2e_catalogs/test.py @@ -1067,6 +1067,97 @@ def test_onelake_show_create_table_no_secret( ) +@only_onelake +def test_onelake_system_databases_no_bearer_token(node, catalog_manager): + """engine_full in system.databases must not expose onelake_bearer_token.""" + + token = catalog_manager.bearer_token() + db = catalog_manager.make_database_name() + catalog_manager.create_catalog_bearer(node, db, token) + engine_full = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db}' " + f"FORMAT TSV", + settings={"show_data_lake_catalogs_in_system_tables": 1}, + ).strip() + assert engine_full, f"Database {db} not found in system.databases" + assert token not in engine_full, ( + "onelake_bearer_token leaked in engine_full of system.databases" + ) + assert "[HIDDEN]" in engine_full, ( + f"onelake_bearer_token was not masked in engine_full:\n{engine_full}" + ) + + +@only_onelake +def test_onelake_show_create_no_bearer_token(node, catalog_manager): + """SHOW CREATE DATABASE must not expose onelake_bearer_token.""" + + token = catalog_manager.bearer_token() + db = catalog_manager.make_database_name() + catalog_manager.create_catalog_bearer(node, db, token) + result = node.query(f"SHOW CREATE DATABASE {db}").strip() + assert token not in result, ( + f"onelake_bearer_token leaked in SHOW CREATE DATABASE:\n{result}" + ) + assert "[HIDDEN]" in result, ( + f"onelake_bearer_token was not masked in SHOW CREATE DATABASE:\n{result}" + ) + + +@only_onelake +def test_onelake_show_create_table_no_bearer_token( + node, catalog_manager, sales_table, +): + """SHOW CREATE TABLE / system.tables must not expose onelake_bearer_token.""" + + token = catalog_manager.bearer_token() + db = catalog_manager.make_database_name() + catalog_manager.create_catalog_bearer(node, db, token) + try: + full = catalog_manager.resolve_table_name(node, db, sales_table) + result = node.query(f"SHOW CREATE TABLE {db}.`{full}`").strip() + assert result, "SHOW CREATE TABLE returned empty result" + assert token not in result, ( + f"onelake_bearer_token leaked in SHOW CREATE TABLE:\n{result}" + ) + system_row = node.query( + f"SELECT engine_full FROM system.tables " + f"WHERE database = '{db}' AND name = '{full}' " + f"FORMAT TSV" + ).strip() + assert token not in system_row, ( + f"onelake_bearer_token leaked in system.tables engine_full:\n{system_row}" + ) + finally: + node.query(f"DROP DATABASE IF EXISTS {db}") + + +@only_onelake +def test_onelake_read_with_bearer_token(node, catalog_manager, sales_table): + """Read table data authenticating only with onelake_bearer_token. + + Exercises both the catalog Authorization header and the Azure Blob + StaticCredential read path (unlike the masking tests, which never + query data).""" + token = catalog_manager.bearer_token() + db = catalog_manager.make_database_name() + catalog_manager.create_catalog_bearer(node, db, token) + try: + full = catalog_manager.resolve_table_name(node, db, sales_table) + # count() alone can be served from Iceberg metadata; sum() over a + # data column forces an actual data-file read from OneLake blob storage. + count = node.query( + f"SELECT count() FROM {db}.`{full}` FORMAT TSV" + ).strip() + assert int(count) == 20 + total_qty = node.query( + f"SELECT sum(quantity) FROM {db}.`{full}` FORMAT TSV" + ).strip() + assert int(total_qty) == 60 + finally: + node.query(f"DROP DATABASE IF EXISTS {db}") + + def test_insert_into_table(node, catalog_manager, request): """INSERT INTO a catalog table and verify the row count increases.""" backend = request.node.callspec.params.get("catalog_manager") @@ -1142,6 +1233,29 @@ def test_onelake_warehouse_wrong_format(node, catalog_manager): assert error, "Expected CREATE DATABASE to fail with malformed warehouse" +@only_onelake +def test_onelake_both_auth_methods_rejected(node, catalog_manager): + """Providing both a bearer token and client credentials is rejected.""" + + db = catalog_manager.make_database_name() + # A dummy token is enough: validation fires before any network call. + sql = catalog_manager.create_db_sql(db, bearer_token="dummy_token") + + error = node.query_and_get_error(sql) + assert "exactly one" in error, error + + +@only_onelake +def test_onelake_no_auth_method_rejected(node, catalog_manager): + """Providing neither a bearer token nor client credentials is rejected.""" + + db = catalog_manager.make_database_name() + sql = catalog_manager.create_db_sql(db, client_id="", client_secret="") + + error = node.query_and_get_error(sql) + assert "exactly one" in error, error + + # --------------------------------------------------------------------------- # Catalog list pagination (regression: list-tables / list-namespaces # silently truncated when the server paginates the response) From 0e671968359478f1956f150b72e595dd4bdeae7a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 4 Aug 2026 08:02:42 +0000 Subject: [PATCH 84/86] Backport #113046 to 26.6: Fix out-of-bounds write reading a Parquet DECIMAL wider than declared --- .../Formats/Impl/Parquet/SchemaConverter.cpp | 14 +- ..._physical_wider_than_destination.reference | 27 ++++ ...decimal_physical_wider_than_destination.sh | 149 ++++++++++++++++++ 3 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.reference create mode 100755 tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.sh diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index 62d9aba02ad9..dbf8d7e2eaa1 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -1200,6 +1200,9 @@ void SchemaConverter::processPrimitiveColumn( UInt32 scale = logical.__isset.DECIMAL ? logical.DECIMAL.scale : element.scale; precision = std::max(precision, scale); + /// Precision of the Decimal type exactly as wide as one decoded value. Legal parquet can + /// make it exceed `precision` (e.g. INT64 with precision 9), so it, not `precision`, + /// determines the width of the column we decode into. UInt32 max_precision = 0; if (type == parq::Type::INT32 || type == parq::Type::INT64) { @@ -1268,8 +1271,15 @@ void SchemaConverter::processPrimitiveColumn( throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet decimal type precision or scale is too big ({} digits) for physical type {}", precision, thriftToString(type)); out_inferred_type = createDecimal(precision, scale); - size_t output_size = out_inferred_type->getSizeOfValueInMemory(); - out_decoder.allow_stats = is_output_type_decimal(output_size, scale); + + /// Decode into a column as wide as the converter writes; castColumn then narrows it to the + /// declared precision, throwing DECIMAL_OVERFLOW for values that don't fit. + auto decoded_type = createDecimal(max_precision, scale); + size_t decoded_size = decoded_type->getSizeOfValueInMemory(); + if (decoded_size != out_inferred_type->getSizeOfValueInMemory()) + out_decoded_type = std::move(decoded_type); + + out_decoder.allow_stats = is_output_type_decimal(decoded_size, scale); return; } diff --git a/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.reference b/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.reference new file mode 100644 index 000000000000..05fdcdf89925 --- /dev/null +++ b/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.reference @@ -0,0 +1,27 @@ +-- dictionary-encoded INT64, declared precision 9 +300 14083299.5 +-- plain-encoded INT64, declared precision 9 +300 14083299.5 +-- FIXED_LEN_BYTE_ARRAY type_length 8, declared precision 9 +300 14083299.5 +-- FIXED_LEN_BYTE_ARRAY type_length 16, declared precision 9 (skips two width buckets) +300 14083299.5 +-- a WHERE over the narrowed column still returns the right rows +140 10224882.45 +-- physical wider than declared: statistics are not usable, so no row group is pruned +read=10 pruned=0 +99 54499.5 +-- same file well-formed (declared precision 18): pruning still works +read=2 pruned=8 +99 54499.5 +-- a hint of exactly the decoded width restores pruning +read=2 pruned=8 +99 54499.5 +-- schema inference still reports the declared precision +k Nullable(Decimal(9, 2)) +-- explicit wider type hint reads without narrowing +Decimal(18, 2) 300 14083299.5 +-- a value that exceeds the declared precision is an error, not a corrupted read +DECIMAL_OVERFLOW +-- ... and reads losslessly with a wide enough hint +300 10014083299.49 diff --git a/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.sh b/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.sh new file mode 100755 index 000000000000..ed9765b5170c --- /dev/null +++ b/tests/queries/0_stateless/04670_parquet_v3_decimal_physical_wider_than_destination.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs pyarrow to craft the fixtures, and Parquet is not built in fasttest. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DATA="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$DATA" +mkdir -p "$DATA" +trap 'rm -rf "$DATA"' EXIT + +# Every fixture declares a DECIMAL precision that maps to a narrower ClickHouse type than one +# encoded value occupies (INT64 or FIXED_LEN_BYTE_ARRAY with precision 9 -> Decimal32). Reading +# any of them used to write past the end of the destination column. +python3 - "$DATA" <<'PYEOF' +import decimal +import os +import struct +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +OUT = sys.argv[1] + +# pyarrow always declares a precision that matches the physical width, so write the wide precision +# and then rewrite the footer's compact-thrift DecimalType {1: i32 scale, 2: i32 precision} varint +# pair in place. The replacement is the same byte width, so every offset in the file stays valid. +SCALE2 = bytes([0x15, 0x04]) +PREC = {9: bytes([0x15, 0x12]), 18: bytes([0x15, 0x24]), 38: bytes([0x15, 0x4C])} + + +def patch_precision(path, was, now): + b = bytearray(open(path, "rb").read()) + assert b[:4] == b"PAR1" and b[-4:] == b"PAR1", "not a parquet file" + flen = struct.unpack("= 1, f"declared precision {was} absent in {path}" + b[start:start + flen] = foot.replace(SCALE2 + PREC[was], SCALE2 + PREC[now]) + open(path, "wb").write(bytes(b)) + + +def write(name, values, *, precision, use_dictionary, as_integer, physical, type_length, + row_group_size=None, row_groups=1, declare=None): + path = os.path.join(OUT, name + ".parquet") + kw = {} if row_group_size is None else {"row_group_size": row_group_size} + pq.write_table(pa.table({"k": pa.array(values, type=pa.decimal128(precision, 2))}), path, + use_dictionary=use_dictionary, compression="none", version="2.6", + store_decimal_as_integer=as_integer, data_page_size=1 << 20, + write_statistics=True, **kw) + if declare is not None: + patch_precision(path, precision, declare) + # Read every property the test relies on back out of the file: the write options only request + # them, and the precision patch is a blind byte replace. + f = pq.ParquetFile(path) + c = f.schema.column(0) + want = f"Decimal(precision={declare or precision}, scale=2)" + assert str(c.logical_type) == want, f"{name}: declared {c.logical_type}, wanted {want}" + assert c.physical_type == physical, f"{name}: physical {c.physical_type}, wanted {physical}" + assert c.length == type_length, f"{name}: type_length {c.length}, wanted {type_length}" + assert f.metadata.num_rows == len(values) + assert f.metadata.num_row_groups == row_groups, f"{name}: {f.metadata.num_row_groups} row groups" + for g in range(f.metadata.num_row_groups): + col_meta = f.metadata.row_group(g).column(0) + assert col_meta.statistics is not None, f"{name}: rg{g} has no statistics" + dict_encoded = "RLE_DICTIONARY" in list(col_meta.encodings) + assert dict_encoded == use_dictionary, f"{name}: rg{g} encodings {list(col_meta.encodings)}" + + +# 300 rows over 60 distinct values: enough to fill a dictionary page, all within 9 digits so they +# fit Decimal32(9, 2) losslessly. +vals = [decimal.Decimal(f"{(i * 7919) % 100000}.{(i * 37) % 100:02d}") for i in range(60)] +col = [vals[i % 60] for i in range(300)] + +# Physical INT64, declared precision 9, on dictionary and on plain pages. +write("int64_dict", col, precision=18, use_dictionary=True, as_integer=True, + physical="INT64", type_length=0, declare=9) +write("int64_plain", col, precision=18, use_dictionary=False, as_integer=True, + physical="INT64", type_length=0, declare=9) +# Physical FIXED_LEN_BYTE_ARRAY, declared precision 9, type_length 8 and 16. +write("flba8", col, precision=18, use_dictionary=True, as_integer=False, + physical="FIXED_LEN_BYTE_ARRAY", type_length=8, declare=9) +write("flba16", col, precision=38, use_dictionary=True, as_integer=False, + physical="FIXED_LEN_BYTE_ARRAY", type_length=16, declare=9) +# A value needing 12 digits, so it does not fit Decimal32(9, 2). +write("overflow", [decimal.Decimal("9999999999.99")] + col[1:], precision=18, + use_dictionary=True, as_integer=True, physical="INT64", type_length=0, declare=9) + +# Row-group-pruning pair: values increase monotonically, so `k > 500` rules out 8 of the 10 row +# groups. Both files hold identical data and differ only in the declared precision, so the +# mismatched one's inability to prune is a property of the shape, not of the layout. +rg = [decimal.Decimal(f"{i}.{i % 100:02d}") for i in range(600)] +write("rowgroups_mismatch", rg, precision=18, use_dictionary=True, as_integer=True, + physical="INT64", type_length=0, row_group_size=60, row_groups=10, declare=9) +write("rowgroups_control", rg, precision=18, use_dictionary=True, as_integer=True, + physical="INT64", type_length=0, row_group_size=60, row_groups=10) +PYEOF + +echo '-- dictionary-encoded INT64, declared precision 9' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/int64_dict.parquet', Parquet)" + +echo '-- plain-encoded INT64, declared precision 9' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/int64_plain.parquet', Parquet)" + +echo '-- FIXED_LEN_BYTE_ARRAY type_length 8, declared precision 9' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/flba8.parquet', Parquet)" + +echo '-- FIXED_LEN_BYTE_ARRAY type_length 16, declared precision 9 (skips two width buckets)' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/flba16.parquet', Parquet)" + +echo '-- a WHERE over the narrowed column still returns the right rows' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/int64_dict.parquet', Parquet) WHERE k > 50000 SETTINGS input_format_parquet_filter_push_down = 1" + +# Statistics decode to a value of the physical width, not the type the key range is built from, so +# they are unusable on this shape. All three fixtures hold 10 row groups of which `k > 500` matches +# the last 2, so the exact counts are the oracle: pruning a matching group changes the split. +prune_counts() { + $CLICKHOUSE_LOCAL --print-profile-events -q "$1" 2>&1 | awk ' + /ParquetReadRowGroups:/ { read += $(NF-1) } + /ParquetPrunedRowGroups:/ { pruned += $(NF-1) } + END { printf "read=%d pruned=%d\n", read, pruned }' +} + +echo '-- physical wider than declared: statistics are not usable, so no row group is pruned' +prune_counts "SELECT count() FROM file('$DATA/rowgroups_mismatch.parquet', Parquet) WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/rowgroups_mismatch.parquet', Parquet) WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" + +echo '-- same file well-formed (declared precision 18): pruning still works' +prune_counts "SELECT count() FROM file('$DATA/rowgroups_control.parquet', Parquet) WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/rowgroups_control.parquet', Parquet) WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" + +echo '-- a hint of exactly the decoded width restores pruning' +prune_counts "SELECT count() FROM file('$DATA/rowgroups_mismatch.parquet', Parquet, 'k Decimal(18, 2)') WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/rowgroups_mismatch.parquet', Parquet, 'k Decimal(18, 2)') WHERE k > 500 SETTINGS input_format_parquet_filter_push_down = 1" + +echo '-- schema inference still reports the declared precision' +$CLICKHOUSE_LOCAL -q "DESC file('$DATA/int64_dict.parquet', Parquet)" + +echo '-- explicit wider type hint reads without narrowing' +$CLICKHOUSE_LOCAL -q "SELECT toTypeName(k), count(), sum(k) FROM file('$DATA/int64_dict.parquet', Parquet, 'k Decimal(18, 2)') GROUP BY 1" + +echo '-- a value that exceeds the declared precision is an error, not a corrupted read' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/overflow.parquet', Parquet)" 2>&1 | grep -o -m1 'DECIMAL_OVERFLOW' + +echo '-- ... and reads losslessly with a wide enough hint' +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(k) FROM file('$DATA/overflow.parquet', Parquet, 'k Decimal(18, 2)')" From 615392524c75e55c02566200a2ea5124e548d033 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 4 Aug 2026 09:02:09 +0000 Subject: [PATCH 85/86] Backport #113089 to 26.6: A dummy PR to backport --- src/Storages/MergeTree/MutateTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 6c0d7c0585a1..f8cd588bd7dd 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1,4 +1,5 @@ #include + #include #include #include From 0a6eba774fd43af4f6eef545641647075e60c45e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 4 Aug 2026 16:34:58 +0000 Subject: [PATCH 86/86] Backport #112784 to 26.6: Fix startup and ATTACH of a view with a recursive CTE with the old analyzer --- src/Databases/DDLDependencyVisitor.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 22 -------- src/Interpreters/ApplyWithSubqueryVisitor.h | 22 +++----- src/Interpreters/InterpreterAlterQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Interpreters/InterpreterInsertQuery.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 8 ++- src/Storages/StorageDistributed.cpp | 2 +- ...ive_cte_view_attach_old_analyzer.reference | 5 ++ ...recursive_cte_view_attach_old_analyzer.sql | 55 +++++++++++++++++++ 12 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.reference create mode 100644 tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.sql diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index 1b75d46d5454..f692a80516f7 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -153,7 +153,7 @@ namespace if (create.is_materialized_view) { auto select_copy = create.select->clone(); - ApplyWithSubqueryVisitor(global_context).visit(select_copy); + ApplyWithSubqueryVisitor::visit(select_copy); /// Use the database where the materialized view is created to resolve nested views. /// The database name can be empty when the AST has been mutated by SharedDatabaseCatalog::serializeCreateQuery diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 8155ac1bf6db..89480f6bbece 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -90,7 +90,7 @@ std::pair createTableFromAST( ast_create_query.setDatabase(database_name); if (ast_create_query.select && ast_create_query.isView()) - ApplyWithSubqueryVisitor(context).visit(*ast_create_query.select); + ApplyWithSubqueryVisitor::visit(*ast_create_query.select); if (ast_create_query.as_table_function) { diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 0706f279e6eb..7c753d9c2d52 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -2023,7 +2023,7 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadata( create.attach = true; if (create.select && create.isView()) - ApplyWithSubqueryVisitor(context_).visit(*create.select); + ApplyWithSubqueryVisitor::visit(*create.select); return ast; } diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index e3efb7f48298..c3c922348a8d 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -1,6 +1,4 @@ -#include #include -#include #include #include #include @@ -18,21 +16,6 @@ namespace DB { -namespace Setting -{ -extern const SettingsBool allow_experimental_analyzer; -} - -namespace ErrorCodes -{ -extern const int UNSUPPORTED_METHOD; -} - -ApplyWithSubqueryVisitor::ApplyWithSubqueryVisitor(ContextPtr context_) - : use_analyzer(context_->getSettingsRef()[Setting::allow_experimental_analyzer]) -{ -} - void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) { checkStackSize(); @@ -52,11 +35,6 @@ void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) void ApplyWithSubqueryVisitor::visit(ASTSelectQuery & ast, const Data & data) { - /// This is probably not the best place to check this, but it's just to throw a proper error to the user - if (!use_analyzer && ast.recursive_with) - throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, "WITH RECURSIVE is not supported with the old analyzer. Please use `enable_analyzer=1`"); - std::optional new_data; if (auto with = ast.with()) { diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.h b/src/Interpreters/ApplyWithSubqueryVisitor.h index 00537b0575ee..72fc07c71483 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.h +++ b/src/Interpreters/ApplyWithSubqueryVisitor.h @@ -2,7 +2,7 @@ #include -#include +#include #include @@ -16,26 +16,22 @@ struct ASTTableExpression; class ApplyWithSubqueryVisitor { public: - explicit ApplyWithSubqueryVisitor(ContextPtr context_); - struct Data { std::map subqueries; std::map literals; }; - void visit(ASTPtr & ast) { visit(ast, {}); } - void visit(ASTSelectQuery & select) { visit(select, {}); } - void visit(ASTSelectWithUnionQuery & select) { visit(select, {}); } + static void visit(ASTPtr & ast) { visit(ast, {}); } + static void visit(ASTSelectQuery & select) { visit(select, {}); } + static void visit(ASTSelectWithUnionQuery & select) { visit(select, {}); } private: - void visit(ASTPtr & ast, const Data & data); - void visit(ASTSelectQuery & ast, const Data & data); - void visit(ASTSelectWithUnionQuery & ast, const Data & data); - void visit(ASTTableExpression & table, const Data & data); - void visit(ASTFunction & func, const Data & data); - - const bool use_analyzer; + static void visit(ASTPtr & ast, const Data & data); + static void visit(ASTSelectQuery & ast, const Data & data); + static void visit(ASTSelectWithUnionQuery & ast, const Data & data); + static void visit(ASTTableExpression & table, const Data & data); + static void visit(ASTFunction & func, const Data & data); }; } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 7a6975d726eb..c1212479f10b 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -468,7 +468,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (modify_query) { // Expand CTE before filling default database - ApplyWithSubqueryVisitor(getContext()).visit(*modify_query); + ApplyWithSubqueryVisitor::visit(*modify_query); } /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 6ba105d7ea64..d2eb7a42f3db 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1745,7 +1745,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.select && create.isView()) { // Expand CTE before filling default database - ApplyWithSubqueryVisitor(getContext()).visit(*create.select); + ApplyWithSubqueryVisitor::visit(*create.select); AddDefaultDatabaseVisitor visitor(getContext(), current_database); visitor.visit(*create.select); } diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 21bdbd90108c..1b92e8056918 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -871,7 +871,7 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica select_query = sq; if (local_context->getSettingsRef()[Setting::enable_global_with_statement]) ApplyWithAliasVisitor::visit(select.list_of_selects->children.at(0)); - ApplyWithSubqueryVisitor(local_context).visit(select.list_of_selects->children.at(0)); + ApplyWithSubqueryVisitor::visit(select.list_of_selects->children.at(0)); JoinedTables joined_tables(Context::createCopy(local_context), *sq); if (joined_tables.tablesCount() == 1) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 42a79abffda2..26937355446f 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -240,6 +240,7 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int BAD_ARGUMENTS; extern const int SUPPORT_IS_DISABLED; + extern const int UNSUPPORTED_METHOD; } /// Assumes `storage` is set and the table filter (row-level security) is not empty. @@ -577,6 +578,11 @@ InterpreterSelectQuery::InterpreterSelectQuery( } } + /// Only the analyzer can resolve recursive CTEs, and reaching this interpreter means the old analyzer. + if (getSelectQuery().recursive_with) + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, "WITH RECURSIVE is not supported with the old analyzer. Please use `enable_analyzer=1`"); + initSettings(); // Automatic parallel replicas aren't supported in the old analyzer, this code is needed only as a safe guard for @@ -610,7 +616,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( { if (context->getSettingsRef()[Setting::enable_global_with_statement]) ApplyWithAliasVisitor::visit(query_ptr); - ApplyWithSubqueryVisitor(context).visit(query_ptr); + ApplyWithSubqueryVisitor::visit(query_ptr); } query_info.query = query_ptr->clone(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f8ea0d59c3c3..3b4fe6d1578b 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1383,7 +1383,7 @@ std::optional StorageDistributed::distributedWrite(const ASTInser { if (local_context->getSettingsRef()[Setting::enable_global_with_statement]) ApplyWithAliasVisitor::visit(select.list_of_selects->children.at(0)); - ApplyWithSubqueryVisitor(local_context).visit(select.list_of_selects->children.at(0)); + ApplyWithSubqueryVisitor::visit(select.list_of_selects->children.at(0)); JoinedTables joined_tables(Context::createCopy(local_context), *select_query); diff --git a/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.reference b/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.reference new file mode 100644 index 000000000000..dee6357add81 --- /dev/null +++ b/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.reference @@ -0,0 +1,5 @@ +6 +recursive_cte_mv +recursive_cte_mv +1 +6 diff --git a/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.sql b/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.sql new file mode 100644 index 000000000000..932d7cced470 --- /dev/null +++ b/tests/queries/0_stateless/04660_recursive_cte_view_attach_old_analyzer.sql @@ -0,0 +1,55 @@ +-- Reading back the definition of a view with a recursive CTE must not depend on the analyzer setting: +-- only executing such a query requires the analyzer. + +DROP TABLE IF EXISTS recursive_cte_view; +DROP TABLE IF EXISTS recursive_cte_mv; +DROP TABLE IF EXISTS recursive_cte_source; + +SET enable_analyzer = 1; + +CREATE TABLE recursive_cte_source (n UInt64) ENGINE = MergeTree ORDER BY n; +INSERT INTO recursive_cte_source VALUES (1); + +CREATE VIEW recursive_cte_view AS +WITH RECURSIVE chain AS +( + SELECT n FROM recursive_cte_source + UNION ALL + SELECT n + 1 FROM chain WHERE n < 3 +) +SELECT * FROM chain; + +CREATE MATERIALIZED VIEW recursive_cte_mv ENGINE = MergeTree ORDER BY n AS +WITH RECURSIVE chain AS +( + SELECT n FROM recursive_cte_source + UNION ALL + SELECT n + 1 FROM chain WHERE n < 3 +) +SELECT * FROM chain; + +SELECT sum(n) FROM recursive_cte_view; +SELECT arrayStringConcat(dependencies_table, ',') FROM system.tables WHERE database = currentDatabase() AND name = 'recursive_cte_source'; + +DETACH TABLE recursive_cte_view; +DETACH TABLE recursive_cte_mv; + +SET enable_analyzer = 0; + +ATTACH TABLE recursive_cte_view; +ATTACH TABLE recursive_cte_mv; + +-- The dependency of the materialized view on its source table is computed from the stored definition +SELECT arrayStringConcat(dependencies_table, ',') FROM system.tables WHERE database = currentDatabase() AND name = 'recursive_cte_source'; +SELECT position(create_table_query, 'WITH RECURSIVE') > 0 FROM system.tables WHERE database = currentDatabase() AND name = 'recursive_cte_mv'; + +SELECT sum(n) FROM recursive_cte_view SETTINGS enable_analyzer = 1; + +WITH RECURSIVE chain AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM chain WHERE n < 3) +SELECT * FROM chain; -- { serverError UNSUPPORTED_METHOD } + +SELECT * FROM recursive_cte_view; -- { serverError UNSUPPORTED_METHOD } + +DROP TABLE recursive_cte_view; +DROP TABLE recursive_cte_mv; +DROP TABLE recursive_cte_source;