From 9209361362bdf9bedc094b10c018b592da939728 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 15:08:41 +0200 Subject: [PATCH 01/18] add tests Signed-off-by: Konstantin Morozov --- .../test.py | 24 +++++++++ .../test.py | 49 +++++++++++++++++++ .../test.py | 23 +++++++++ .../test.py | 32 ++++++++++++ 4 files changed, 128 insertions(+) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 486adf1f2b17..78a0b94d2414 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,6 +13,7 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously + test_export_part_same_partition_key_different_column_order – same partition key, different column order """ import logging @@ -462,6 +463,29 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_same_partition_key_different_column_order(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_reordered_{sfx}" + iceberg = f"iceberg_reordered_{sfx}" + + make_mt(node, mt, "a Int32, b Int32", "a") + make_iceberg_s3(node, iceberg, "b Int32, a Int32", "a") + + node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") + + part_1 = get_part(node, mt, "1") + export_part(node, mt, part_1, iceberg) + wait_for_export_part(node, mt, part_1) + + result = node.query(f"SELECT a, b FROM {iceberg} ORDER BY a, b").strip() + expected = "1\t1\n1\t2" + assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b8c15c26275f..10bd5959f3e1 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -312,3 +312,52 @@ def test_pending_patch_parts_skip_before_export(cluster): assert "1\n2\n3" in result, "Export should contain original data before patch" node.query(f"DROP TABLE {mt_table}") + + +def test_export_part_same_partition_key_different_column_order(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"reordered_part_mt_table_{postfix}" + s3_table = f"reordered_part_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32) + ENGINE = MergeTree() + PARTITION BY a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (b Int32, a Int32) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY a + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + deadline = time.time() + 30 + while True: + node.query("SYSTEM FLUSH LOGS") + count = node.query( + f"SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' " + f"AND database = currentDatabase() AND table = '{mt_table}' AND part_name = '{part_name}'" + ).strip() + if count != "0": + break + if time.time() > deadline: + raise TimeoutError(f"ExportPart event for part {part_name!r} did not appear within 30s") + time.sleep(0.5) + + dest_result = node.query(f"SELECT a, b FROM {s3_table} ORDER BY a, b") + expected = "1\t1\n1\t2\n" + assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index ad2deba8de19..844f79e85f7c 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1324,6 +1324,29 @@ def test_export_partition_with_renamed_destination_column(cluster): ) +def test_export_partition_same_partition_key_different_column_order(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_reordered_{uid}" + iceberg_table = f"iceberg_reordered_{uid}" + + make_rmt(node, mt_table, "a Int32, b Int32", "a", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "b Int32, a Int32", partition_by="a") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "1", "COMPLETED") + + result = node.query(f"SELECT a, b FROM {iceberg_table} ORDER BY a, b").strip() + expected = "1\t1\n1\t2" + assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + + def test_export_partition_with_castable_widening(cluster): """A lossless widening of both a data column (id Int32 -> Int64) and the partition column (year Int32 -> Int64) round-trips.""" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 8d4589292e3c..6a6e63e36bc1 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1747,3 +1747,35 @@ def test_export_partition_all_failure_modes(cluster): f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" f" SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" ) + + +def test_export_partition_same_partition_key_different_column_order(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"reordered_mt_table_{postfix}" + s3_table = f"reordered_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY a + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (b Int32, a Int32) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY a + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") + + dest_result = node.query(f"SELECT * FROM {s3_table} ORDER BY a, b") + + expected = "1\t1\n2\t1\n" + assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" From 1a6d63c21c0376a60f858ac5e26b9786ea444814 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 15:09:35 +0200 Subject: [PATCH 02/18] refactoring Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 13 +++++++++++++ src/Storages/MergeTree/ExportPartitionUtils.h | 4 ++++ src/Storages/MergeTree/MergeTreeData.cpp | 12 +----------- src/Storages/StorageReplicatedMergeTree.cpp | 12 +----------- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 3f85ebc0e1fb..616a3384ba1c 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -634,6 +634,19 @@ namespace ExportPartitionUtils } #endif + void verifyMergeTreePartitionCompatibility( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata) + { + constexpr auto query_to_string = [] (const ASTPtr & ast) + { + return ast ? ast->formatWithSecretsOneLine() : ""; + }; + + if (query_to_string(source_metadata->getPartitionKeyAST()) != query_to_string(destination_metadata->getPartitionKeyAST())) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 0bb8acb9bda4..dd1ae4c18094 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -89,6 +89,10 @@ namespace ExportPartitionUtils const std::string & exception_message, const LoggerPtr & log); + void verifyMergeTreePartitionCompatibility( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata); + /// Validates that source columns can be exported into the destination with the /// same positional CAST matching as `INSERT INTO dest SELECT * FROM src`. Lossy /// casts are rejected unless `export_merge_tree_part_allow_lossy_cast` is set. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0ed451b59655..ac8ad03c3b63 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6726,11 +6726,6 @@ void MergeTreeData::exportPartToTable( if (!dest_storage->supportsImport(query_context)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Destination storage {} does not support MergeTree parts or uses unsupported partitioning", dest_storage->getName()); - auto query_to_string = [] (const ASTPtr & ast) - { - return ast ? ast->formatWithSecretsOneLine() : ""; - }; - auto source_metadata_ptr = getInMemoryMetadataPtr(); auto destination_metadata_ptr = dest_storage->getInMemoryMetadataPtr(); @@ -6791,13 +6786,8 @@ void MergeTreeData::exportPartToTable( ExportPartitionUtils::verifyExportSchemaCastable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); - /// Iceberg partition compatibility is checked above; here we only need the - /// partition-key ASTs to match (partition-column types follow the lossy-cast gate). if (!dest_storage->isDataLake()) - { - if (query_to_string(source_metadata_ptr->getPartitionKeyAST()) != query_to_string(destination_metadata_ptr->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); - } + ExportPartitionUtils::verifyMergeTreePartitionCompatibility(source_metadata_ptr, destination_metadata_ptr); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5677a5e111a8..759736dc06be 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8408,11 +8408,6 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & if (!dest_storage->supportsImport(query_context)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Destination storage {} does not support MergeTree parts or uses unsupported partitioning", dest_storage->getName()); - auto query_to_string = [] (const ASTPtr & ast) - { - return ast ? ast->formatWithSecretsOneLine() : ""; - }; - auto src_snapshot = getInMemoryMetadataPtr(); auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(); @@ -8420,13 +8415,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & ExportPartitionUtils::verifyExportSchemaCastable( src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); - /// Iceberg partition compatibility is checked below; here we only need the - /// partition-key ASTs to match (partition-column types follow the lossy-cast gate). if (!dest_storage->isDataLake()) - { - if (query_to_string(src_snapshot->getPartitionKeyAST()) != query_to_string(destination_snapshot->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); - } + ExportPartitionUtils::verifyMergeTreePartitionCompatibility(src_snapshot, destination_snapshot); zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); From 420f2e8ebf5cef375930bfca3307581a05a226c6 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 16:55:57 +0200 Subject: [PATCH 03/18] add check names and position Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 29 ++++++++++++++++--- .../test.py | 15 ++++++---- .../test.py | 24 ++++----------- .../test.py | 10 +++---- .../test.py | 11 ++++--- 5 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 616a3384ba1c..39256852b002 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -644,7 +645,8 @@ namespace ExportPartitionUtils }; if (query_to_string(source_metadata->getPartitionKeyAST()) != query_to_string(destination_metadata->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export partition: source and destination tables have different `PARTITION BY` expressions"); } void verifyExportSchemaCastable( @@ -670,15 +672,34 @@ namespace ExportPartitionUtils ActionsDAG::MatchColumnsMode::Position, context); - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]) - return; + auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); + const std::unordered_set partition_key_column_set( + std::make_move_iterator(partition_key_columns.begin()), + std::make_move_iterator(partition_key_columns.end())); + + const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) { const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; + + if (partition_key_column_set.contains(source_column.name) && source_column.name != destination_column.name) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + i, + destination_column.name); + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (allow_lossy_cast) + continue; + if (!canBeSafelyCast(source_column.type, destination_column.type)) throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 78a0b94d2414..cefd25cc0796 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -475,12 +475,17 @@ def test_export_part_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") part_1 = get_part(node, mt, "1") - export_part(node, mt, part_1, iceberg) - wait_for_export_part(node, mt, part_1) - result = node.query(f"SELECT a, b FROM {iceberg} ORDER BY a, b").strip() - expected = "1\t1\n1\t2" - assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part_1}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 10bd5959f3e1..e9b43c0141bb 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -343,21 +343,9 @@ def test_export_part_same_partition_key_different_column_order(cluster): f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - deadline = time.time() + 30 - while True: - node.query("SYSTEM FLUSH LOGS") - count = node.query( - f"SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' " - f"AND database = currentDatabase() AND table = '{mt_table}' AND part_name = '{part_name}'" - ).strip() - if count != "0": - break - if time.time() > deadline: - raise TimeoutError(f"ExportPart event for part {part_name!r} did not appear within 30s") - time.sleep(0.5) - - dest_result = node.query(f"SELECT a, b FROM {s3_table} ORDER BY a, b") - expected = "1\t1\n1\t2\n" - assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 844f79e85f7c..f4fa3f71da90 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1336,15 +1336,15 @@ def test_export_partition_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - node.query( + error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", settings={"allow_insert_into_iceberg": 1}, ) - wait_for_export_status(node, mt_table, iceberg_table, "1", "COMPLETED") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - result = node.query(f"SELECT a, b FROM {iceberg_table} ORDER BY a, b").strip() - expected = "1\t1\n1\t2" - assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" def test_export_partition_with_castable_widening(cluster): diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 6a6e63e36bc1..7cb9b71202fa 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1772,10 +1772,9 @@ def test_export_partition_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") - wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") - - dest_result = node.query(f"SELECT * FROM {s3_table} ORDER BY a, b") + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - expected = "1\t1\n2\t1\n" - assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" From 5322e48aae583cbaea6a6f427f0beb24c7248b53 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 31 Jul 2026 17:10:22 +0200 Subject: [PATCH 04/18] add tests Signed-off-by: Konstantin Morozov --- .../test.py | 183 +++++++++++++- .../test.py | 187 +++++++++++++- .../test.py | 144 ++++++++++- .../test.py | 236 +++++++++++++++++- 4 files changed, 744 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index cefd25cc0796..5494846b4433 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,7 +13,12 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously - test_export_part_same_partition_key_different_column_order – same partition key, different column order + test_export_part_same_partition_key_different_column_order_single_column – same 1-column partition key, different column order + test_export_part_same_partition_key_different_column_order_multi_column – same 2-column partition key, different column order + test_export_part_multi_column_partition_key_success – composite (a, b) partition key round-trips + test_export_part_multi_column_partition_key_order_mismatch_is_rejected – composite key fields swapped between src/dst + test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected – dst has fewer partition fields than src + test_export_part_multi_column_partition_key_more_in_destination_is_rejected – dst has more partition fields than src """ import logging @@ -463,7 +468,7 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_same_partition_key_different_column_order(cluster): +def test_export_part_same_partition_key_different_column_order_single_column(cluster): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_reordered_{sfx}" @@ -491,6 +496,180 @@ def test_export_part_same_partition_key_different_column_order(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_same_partition_key_different_column_order_multi_column(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_reordered_{sfx}" + iceberg = f"iceberg_multi_reordered_{sfx}" + + make_mt(node, mt, "a Int32, b Int32, c Int32, val String", "(a, b, c)") + make_iceberg_s3(node, iceberg, "c Int32, b Int32, a Int32, val String", "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_success(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_pkey_ok_{sfx}" + iceberg = f"iceberg_multi_pkey_ok_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + export_part(node, mt, part, iceberg) + wait_for_export_part(node, mt, part) + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {iceberg} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + assert_part_log(node, mt, part) + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_order_{sfx}" + iceberg = f"iceberg_multi_order_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(c, b, a)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_fewer_{sfx}" + iceberg = f"iceberg_multi_fewer_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(a, b)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_more_{sfx}" + iceberg = f"iceberg_multi_more_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b)") + make_iceberg_s3(node, iceberg, cols, "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_transform_partition_key_different_column_order_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_transform_reordered_{sfx}" + iceberg = f"iceberg_transform_reordered_{sfx}" + + make_mt(node, mt, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)") + make_iceberg_s3(node, iceberg, "user_id Int64, other_id Int64", "icebergBucket(8, user_id)") + + node.query(f"INSERT INTO {mt} VALUES (1, 42)") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index e9b43c0141bb..132d13ecc303 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -314,7 +314,7 @@ def test_pending_patch_parts_skip_before_export(cluster): node.query(f"DROP TABLE {mt_table}") -def test_export_part_same_partition_key_different_column_order(cluster): +def test_export_part_same_partition_key_different_column_order_single_column(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -349,3 +349,188 @@ def test_export_part_same_partition_key_different_column_order(cluster): count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_same_partition_key_different_column_order_multi_column(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_reordered_part_mt_table_{postfix}" + s3_table = f"multi_reordered_part_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_success(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(5) + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + +def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_order_mt_table_{postfix}" + s3_table = f"multi_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (c, b, a) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_fewer_mt_table_{postfix}" + s3_table = f"multi_fewer_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_more_mt_table_{postfix}" + s3_table = f"multi_more_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index f4fa3f71da90..e3aa21baeb33 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -754,6 +754,7 @@ def check_accepted(mt, iceberg, description): f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", settings={"allow_insert_into_iceberg": 1}, ) + return pid # 1. Compound identity: (year, region) cols = "id Int64, year Int32, region String" @@ -761,7 +762,12 @@ def check_accepted(mt, iceberg, description): make_rmt(node, t, cols, "(year, region)") node.query(f"INSERT INTO {t} VALUES (1, 2023, 'EU')") make_iceberg_s3(node, i, cols, "(year, region)") - check_accepted(t, i, "compound identity (year, region)") + pid = check_accepted(t, i, "compound identity (year, region)") + wait_for_export_status(node, t, i, pid, "COMPLETED") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 1, f"[compound identity (year, region)] Expected 1 row in Iceberg table, got {count}" + result = node.query(f"SELECT id, year, region FROM {i}").strip() + assert result == "1\t2023\tEU", f"[compound identity (year, region)] Unexpected exported data:\n{result}" # 2. Year transform cols = "id Int64, event_date Date" @@ -837,6 +843,8 @@ def assert_rejected(mt, iceberg, description): node.query(f"INSERT INTO {t} VALUES (1, 2020, 'EU')") make_iceberg_s3(node, i, cols, "(region, year)") assert_rejected(t, i, "compound field order reversed") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 0, f"[compound field order reversed] Expected 0 rows in destination, got {count}" # 2. Transform mismatch: MergeTree year-transform, Iceberg identity on same Date col cols = "id Int64, event_date Date" @@ -869,6 +877,8 @@ def assert_rejected(mt, iceberg, description): node.query(f"INSERT INTO {t} VALUES (1, 2020, 'EU')") make_iceberg_s3(node, i, cols, "year") assert_rejected(t, i, "2-field MergeTree vs 1-field Iceberg") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 0, f"[2-field MergeTree vs 1-field Iceberg] Expected 0 rows in destination, got {count}" # 6. Unsupported MergeTree expression: intDiv(year, 100) is not an Iceberg transform cols = "id Int64, year Int32" @@ -1324,7 +1334,7 @@ def test_export_partition_with_renamed_destination_column(cluster): ) -def test_export_partition_same_partition_key_different_column_order(cluster): +def test_export_partition_same_partition_key_different_column_order_single_column(cluster): node = cluster.instances["replica1"] uid = unique_suffix() @@ -1343,6 +1353,136 @@ def test_export_partition_same_partition_key_different_column_order(cluster): assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_reordered_{uid}" + iceberg_table = f"iceberg_multi_reordered_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "c Int32, b Int32, a Int32, val String", partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_more_{uid}" + iceberg_table = f"iceberg_multi_more_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_pkey_ok_all_{uid}" + iceberg_table = f"iceberg_multi_pkey_ok_all_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + for pid in partition_ids: + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {iceberg_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" + + +def test_export_partition_transform_partition_key_different_column_order_is_rejected(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_transform_reordered_{uid}" + iceberg_table = f"iceberg_transform_reordered_{uid}" + + make_rmt(node, mt_table, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "user_id Int64, other_id Int64", partition_by="icebergBucket(8, user_id)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 42)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 7cb9b71202fa..cb73cc926f4e 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1749,7 +1749,7 @@ def test_export_partition_all_failure_modes(cluster): ) -def test_export_partition_same_partition_key_different_column_order(cluster): +def test_export_partition_same_partition_key_different_column_order_single_column(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] @@ -1776,5 +1776,239 @@ def test_export_partition_same_partition_key_different_column_order(cluster): assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_reordered_mt_table_{postfix}" + s3_table = f"multi_reordered_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + +def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_order_mt_table_{postfix}" + s3_table = f"multi_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (c, b, a) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_fewer_mt_table_{postfix}" + s3_table = f"multi_fewer_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_more_mt_table_{postfix}" + s3_table = f"multi_more_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_all_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_all_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + + for pid in partition_ids: + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" From cb859ebdff1940c5f38b2879913f06479bc8c9f0 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 14:37:30 +0200 Subject: [PATCH 05/18] update tests, add hive part check Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 3 + docs/en/antalya/partition_export.md | 8 + .../MergeTree/ExportPartitionUtils.cpp | 32 +++ .../test.py | 253 +++++++----------- .../test.py | 214 ++++++++------- .../test.py | 168 +++++------- .../test.py | 227 +++++++--------- 7 files changed, 415 insertions(+), 490 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 73d467c5d9b1..2ff4a7301674 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,6 +51,9 @@ Source and destination tables must be 100% compatible: 1. **Identical schemas** - same columns, types, and order 2. **Matching partition keys** - partition expressions must be identical +3. **Partition key columns at the same position** - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`. It is not enough for the `PARTITION BY` expressions to be textually identical: every column that is part of the source table's partition key must also sit at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`, so the export is rejected with `BAD_ARGUMENTS: partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. + + This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 687029b9adc6..ccd18914845b 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -43,6 +43,14 @@ TO TABLE [destination_database.]destination_table - **`partition_id`**: The partition identifier to export (e.g., `'2020'`, `'2021'`) - **`destination_table`**: The target table for the export (typically an S3, Azure, or other object storage table) +## Requirements + +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: + +1. **Identical schemas** - same columns, types, and order +2. **Matching partition keys** - partition expressions must be identical +3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements) for a worked example and the exact error message. + ## Settings ### Server Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 39256852b002..d11b74aff13f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "Storages/ExportReplicatedMergeTreePartitionManifest.h" #include "Storages/ExportReplicatedMergeTreePartitionTaskEntry.h" #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -635,6 +638,18 @@ namespace ExportPartitionUtils } #endif + namespace + { + std::optional getDateTimeTimeZoneName(const DataTypePtr & type) + { + if (const auto * datetime_type = typeid_cast(type.get())) + return datetime_type->getTimeZone().getTimeZone(); + if (const auto * datetime64_type = typeid_cast(type.get())) + return datetime64_type->getTimeZone().getTimeZone(); + return {}; + } + } + void verifyMergeTreePartitionCompatibility( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -696,6 +711,23 @@ namespace ExportPartitionUtils i, destination_column.name); + if (partition_key_column_set.contains(source_column.name)) + { + const auto source_time_zone = getDateTimeTimeZoneName(source_column.type); + const auto destination_time_zone = getDateTimeTimeZoneName(destination_column.type); + if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is {} in the source table " + "but {} in the destination. The destination's hive-style partition path is " + "rendered from the source value without converting the timezone, so this " + "would silently shift the exported value by the timezone offset. Use the " + "same timezone in both tables' partition key column.", + destination_storage_id.getFullTableName(), + destination_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } + /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) continue; diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 5494846b4433..e1dd26ea3820 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,16 +13,14 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously - test_export_part_same_partition_key_different_column_order_single_column – same 1-column partition key, different column order - test_export_part_same_partition_key_different_column_order_multi_column – same 2-column partition key, different column order - test_export_part_multi_column_partition_key_success – composite (a, b) partition key round-trips - test_export_part_multi_column_partition_key_order_mismatch_is_rejected – composite key fields swapped between src/dst - test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected – dst has fewer partition fields than src - test_export_part_multi_column_partition_key_more_in_destination_is_rejected – dst has more partition fields than src + test_export_part_multi_column_partition_key_success – composite (a, b, c) partition key round-trips + test_export_part_partition_key_mismatch_variants_are_rejected (parametrized) – partition key column reordering, + cardinality mismatches, and transform-expression reordering between src/dst are all rejected synchronously """ import logging import time +from typing import NamedTuple import pytest @@ -468,44 +466,96 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PART_EXPORT_CASES = [ + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition field 0 mismatch",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_more_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="other_id Int64, user_id Int64", + src_partition_by="icebergBucket(8, user_id)", + dst_columns="user_id Int64, other_id Int64", + dst_partition_by="icebergBucket(8, user_id)", + insert_values="(1, 42)", + error_substrings=("partition key column",), + ), + id="transform_partition_key_different_column_order", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PART_EXPORT_CASES) +def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case): node = cluster.instances["node1"] sfx = unique_suffix() - mt = f"mt_reordered_{sfx}" - iceberg = f"iceberg_reordered_{sfx}" + mt = f"mt_rejected_{sfx}" + iceberg = f"iceberg_rejected_{sfx}" - make_mt(node, mt, "a Int32, b Int32", "a") - make_iceberg_s3(node, iceberg, "b Int32, a Int32", "a") + make_mt(node, mt, case.src_columns, case.src_partition_by) + make_iceberg_s3(node, iceberg, case.dst_columns, case.dst_partition_by) - node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") - - part_1 = get_part(node, mt, "1") - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part_1}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_same_partition_key_different_column_order_multi_column(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_reordered_{sfx}" - iceberg = f"iceberg_multi_reordered_{sfx}" - - make_mt(node, mt, "a Int32, b Int32, c Int32, val String", "(a, b, c)") - make_iceberg_s3(node, iceberg, "c Int32, b Int32, a Int32, val String", "(a, b, c)") - - node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt} VALUES {case.insert_values}") pid = first_partition_id(node, mt) part = get_part(node, mt, pid) @@ -516,7 +566,8 @@ def test_export_part_same_partition_key_different_column_order_multi_column(clus f"allow_experimental_insert_into_iceberg = 1" ) assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error!r}" count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" @@ -535,7 +586,7 @@ def test_export_part_multi_column_partition_key_success(cluster): make_mt(node, mt, cols, "(a, b, c)") make_iceberg_s3(node, iceberg, cols, "(a, b, c)") - node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") pid = first_partition_id(node, mt) part = get_part(node, mt, pid) @@ -546,7 +597,7 @@ def test_export_part_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {iceberg} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" assert_part_log(node, mt, part) @@ -554,122 +605,6 @@ def test_export_part_multi_column_partition_key_success(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_order_{sfx}" - iceberg = f"iceberg_multi_order_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b, c)") - make_iceberg_s3(node, iceberg, cols, "(c, b, a)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_fewer_{sfx}" - iceberg = f"iceberg_multi_fewer_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b, c)") - make_iceberg_s3(node, iceberg, cols, "(a, b)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_more_{sfx}" - iceberg = f"iceberg_multi_more_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b)") - make_iceberg_s3(node, iceberg, cols, "(a, b, c)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_transform_partition_key_different_column_order_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_transform_reordered_{sfx}" - iceberg = f"iceberg_transform_reordered_{sfx}" - - make_mt(node, mt, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)") - make_iceberg_s3(node, iceberg, "user_id Int64, other_id Int64", "icebergBucket(8, user_id)") - - node.query(f"INSERT INTO {mt} VALUES (1, 42)") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 132d13ecc303..f946b01e726f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1,6 +1,7 @@ import logging import time import uuid +from typing import NamedTuple import pytest @@ -314,29 +315,98 @@ def test_pending_patch_parts_skip_before_export(cluster): node.query(f"DROP TABLE {mt_table}") -def test_export_part_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PART_EXPORT_CASES = [ + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_more_in_destination", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PART_EXPORT_CASES) +def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"reordered_part_mt_table_{postfix}" - s3_table = f"reordered_part_s3_table_{postfix}" + mt_table = f"rejected_mt_table_{postfix}" + s3_table = f"rejected_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32) + CREATE TABLE {mt_table} ({case.src_columns}) ENGINE = MergeTree() - PARTITION BY a + PARTITION BY {case.src_partition_by} ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (b Int32, a Int32) + CREATE TABLE {s3_table} ({case.dst_columns}) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY a + PARTITION BY {case.dst_partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -345,44 +415,8 @@ def test_export_part_same_partition_key_different_column_order_single_column(clu error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_part_same_partition_key_different_column_order_multi_column(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_reordered_part_mt_table_{postfix}" - s3_table = f"multi_reordered_part_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = MergeTree() - PARTITION BY (a, b, c) - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" @@ -410,7 +444,7 @@ def test_export_part_multi_column_partition_key_success(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -425,32 +459,32 @@ def test_export_part_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): +def test_export_part_partition_key_timezone_mismatch_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_order_mt_table_{postfix}" - s3_table = f"multi_order_s3_table_{postfix}" + mt_table = f"tz_mismatch_mt_table_{postfix}" + s3_table = f"tz_mismatch_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = MergeTree() - PARTITION BY (a, b, c) + PARTITION BY ts ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (c, b, a) + PARTITION BY ts """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -459,78 +493,62 @@ def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(clust error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" -def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): +def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_fewer_mt_table_{postfix}" - s3_table = f"multi_fewer_s3_table_{postfix}" + mt_table = f"tz_ok_mt_table_{postfix}" + s3_table = f"tz_ok_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = MergeTree() - PARTITION BY (a, b, c) + PARTITION BY id ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b) + PARTITION BY id """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_more_mt_table_{postfix}" - s3_table = f"multi_more_s3_table_{postfix}" + time.sleep(5) - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = MergeTree() - PARTITION BY (a, b) - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 1, f"Expected 1 row in destination after export, got {count}" - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) + source_ts = node.query(f"SELECT ts FROM {mt_table}").strip() + assert source_ts == "2024-03-05 15:00:00", f"Unexpected source value: {source_ts}" - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + dest_ts = node.query(f"SELECT ts FROM {s3_table}").strip() + assert dest_ts == "2024-03-06 00:00:00", ( + f"Expected the exported value to be the same instant displayed in the " + f"destination's Asia/Tokyo timezone ('2024-03-06 00:00:00'), got: {dest_ts}" + ) - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() + source_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip()) + dest_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_table}").strip()) + assert source_unix_ts == dest_unix_ts, ( + f"Expected exported DateTime value to be preserved regardless of the " + f"destination column's timezone, got source={source_unix_ts}, dest={dest_unix_ts}" + ) - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index e3aa21baeb33..53e18c3df66b 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -3,6 +3,7 @@ import logging import re import time +from typing import NamedTuple import pytest from avro.datafile import DataFileReader @@ -1334,78 +1335,75 @@ def test_export_partition_with_renamed_destination_column(cluster): ) -def test_export_partition_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartitionExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PARTITION_EXPORT_CASES = [ + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_more_in_destination", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="other_id Int64, user_id Int64", + src_partition_by="icebergBucket(8, user_id)", + dst_columns="user_id Int64, other_id Int64", + dst_partition_by="icebergBucket(8, user_id)", + insert_values="(1, 42)", + error_substrings=("partition key column",), + ), + id="transform_partition_key_different_column_order", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case): node = cluster.instances["replica1"] uid = unique_suffix() - mt_table = f"mt_reordered_{uid}" - iceberg_table = f"iceberg_reordered_{uid}" + mt_table = f"mt_rejected_{uid}" + iceberg_table = f"iceberg_rejected_{uid}" - make_rmt(node, mt_table, "a Int32, b Int32", "a", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "b Int32, a Int32", partition_by="a") + make_rmt(node, mt_table, case.src_columns, case.src_partition_by, replica_name="replica1") + make_iceberg_s3(node, iceberg_table, case.dst_columns, partition_by=case.dst_partition_by) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_multi_reordered_{uid}" - iceberg_table = f"iceberg_multi_reordered_{uid}" - - cols = "a Int32, b Int32, c Int32, val String" - make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "c Int32, b Int32, a Int32, val String", partition_by="(a, b, c)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_multi_more_{uid}" - iceberg_table = f"iceberg_multi_more_{uid}" - - cols = "a Int32, b Int32, c Int32, val String" - make_rmt(node, mt_table, cols, "(a, b)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") pid = first_partition_id(node, mt_table) error = node.query_and_get_error( @@ -1413,6 +1411,8 @@ def test_export_partition_multi_column_partition_key_more_in_destination_is_reje settings={"allow_insert_into_iceberg": 1}, ) assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" error_all = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", @@ -1435,7 +1435,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") partition_ids = node.query( f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1454,37 +1454,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {iceberg_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" - - -def test_export_partition_transform_partition_key_different_column_order_is_rejected(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_transform_reordered_{uid}" - iceberg_table = f"iceberg_transform_reordered_{uid}" - - make_rmt(node, mt_table, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "user_id Int64, other_id Int64", partition_by="icebergBucket(8, user_id)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 42)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" def test_export_partition_with_castable_widening(cluster): diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index cb73cc926f4e..f73b35e29b1a 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1,6 +1,7 @@ import logging import time import uuid +from typing import NamedTuple import pytest @@ -1749,62 +1750,97 @@ def test_export_partition_all_failure_modes(cluster): ) -def test_export_partition_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartitionExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PARTITION_EXPORT_CASES = [ + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_more_in_destination", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"reordered_mt_table_{postfix}" - s3_table = f"reordered_s3_table_{postfix}" + mt_table = f"rejected_mt_table_{postfix}" + s3_table = f"rejected_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32) + CREATE TABLE {mt_table} ({case.src_columns}) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY a + PARTITION BY {case.src_partition_by} ORDER BY tuple() """) node.query(f""" - CREATE TABLE {s3_table} (b Int32, a Int32) + CREATE TABLE {s3_table} ({case.dst_columns}) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY a + PARTITION BY {case.dst_partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_reordered_mt_table_{postfix}" - s3_table = f"multi_reordered_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1813,7 +1849,8 @@ def test_export_partition_same_partition_key_different_column_order_multi_column error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" @@ -1843,7 +1880,7 @@ def test_export_partition_multi_column_partition_key_success(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1857,31 +1894,31 @@ def test_export_partition_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected(cluster): +def test_export_partition_partition_key_timezone_mismatch_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_order_mt_table_{postfix}" - s3_table = f"multi_order_s3_table_{postfix}" + mt_table = f"tz_mismatch_mt_table_{postfix}" + s3_table = f"tz_mismatch_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) + PARTITION BY ts ORDER BY tuple() """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (c, b, a) + PARTITION BY ts """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1890,85 +1927,7 @@ def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected( error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_fewer_mt_table_{postfix}" - s3_table = f"multi_fewer_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_more_mt_table_{postfix}" - s3_table = f"multi_more_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" @@ -1995,7 +1954,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") partition_ids = node.query( f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -2011,4 +1970,4 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" From faef4636174e10cad68de2da31a2816a53024aa9 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 14:46:38 +0200 Subject: [PATCH 06/18] up tests Signed-off-by: Konstantin Morozov --- .../test.py | 48 +++++-------------- .../test.py | 47 +++++------------- 2 files changed, 22 insertions(+), 73 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index f946b01e726f..b4a1160313aa 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -380,6 +380,17 @@ class RejectedPartExportCase(NamedTuple): ), id="multi_column_partition_key_more_in_destination", ), + pytest.param( + RejectedPartExportCase( + src_columns="id Int64, ts DateTime('UTC')", + src_partition_by="ts", + dst_columns="id Int64, ts DateTime('Asia/Tokyo')", + dst_partition_by="ts", + insert_values="(1, '2024-03-05 15:00:00')", + error_substrings=("timezone",), + ), + id="partition_key_timezone_mismatch", + ), ] @@ -462,43 +473,6 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_mismatch_mt_table_{postfix}" - s3_table = f"tz_mismatch_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index f73b35e29b1a..0b38f97df4a5 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1815,6 +1815,17 @@ class RejectedPartitionExportCase(NamedTuple): ), id="multi_column_partition_key_more_in_destination", ), + pytest.param( + RejectedPartitionExportCase( + src_columns="id Int64, ts DateTime('UTC')", + src_partition_by="ts", + dst_columns="id Int64, ts DateTime('Asia/Tokyo')", + dst_partition_by="ts", + insert_values="(1, '2024-03-05 15:00:00')", + error_substrings=("timezone",), + ), + id="partition_key_timezone_mismatch", + ), ] @@ -1897,42 +1908,6 @@ def test_export_partition_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_partition_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_mismatch_mt_table_{postfix}" - s3_table = f"tz_mismatch_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY ts - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - def test_export_partition_multi_column_partition_key_success_all(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] From 31116d664c6a7d9caf1fd0174f2a10d0d49f176b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 18:28:26 +0200 Subject: [PATCH 07/18] refactoring Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index d11b74aff13f..433c663a445b 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include "Storages/ExportReplicatedMergeTreePartitionManifest.h" #include "Storages/ExportReplicatedMergeTreePartitionTaskEntry.h" #include @@ -17,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -642,9 +642,9 @@ namespace ExportPartitionUtils { std::optional getDateTimeTimeZoneName(const DataTypePtr & type) { - if (const auto * datetime_type = typeid_cast(type.get())) + if (const auto * datetime_type = checkAndGetDataType(type.get())) return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = typeid_cast(type.get())) + if (const auto * datetime64_type = checkAndGetDataType(type.get())) return datetime64_type->getTimeZone().getTimeZone(); return {}; } From 9c4ee5ade4ce15235b895572104fb71cb756745c Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 15:34:50 +0200 Subject: [PATCH 08/18] verify subcolumns Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 101 ++- src/Storages/MergeTree/ExportPartitionUtils.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../test.py | 44 ++ .../test.py | 584 ++++++++++++++++++ 5 files changed, 701 insertions(+), 32 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 433c663a445b..17497b8ad056 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #if USE_AVRO #include @@ -648,9 +649,55 @@ namespace ExportPartitionUtils return datetime64_type->getTimeZone().getTimeZone(); return {}; } + + void verifyPartitionKeyColumn( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + size_t position, + const std::vector & required_columns, + const ColumnsDescription & source_columns_description, + const ColumnsDescription & destination_columns_description, + const StorageID & destination_storage_id) + { + if (source_column.name != destination_column.name) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + position, + destination_column.name); + + for (const auto & column_name : required_columns) + { + const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + const auto destination_resolved + = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + if (!source_resolved || !destination_resolved) + continue; + + const auto source_time_zone = getDateTimeTimeZoneName(source_resolved->type); + const auto destination_time_zone = getDateTimeTimeZoneName(destination_resolved->type); + if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is {} in the source table " + "but {} in the destination. The destination's hive-style partition path is " + "rendered from the source value without converting the timezone, so this " + "would silently shift the exported value by the timezone offset. Use the " + "same timezone in both tables' partition key column.", + destination_storage_id.getFullTableName(), + column_name, + source_resolved->type->getName(), + destination_resolved->type->getName()); + } + } } - void verifyMergeTreePartitionCompatibility( + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) { @@ -687,10 +734,21 @@ namespace ExportPartitionUtils ActionsDAG::MatchColumnsMode::Position, context); + const auto & source_columns_description = source_metadata->getColumns(); + const auto & destination_columns_description = destination_metadata->getColumns(); auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); - const std::unordered_set partition_key_column_set( - std::make_move_iterator(partition_key_columns.begin()), - std::make_move_iterator(partition_key_columns.end())); + + /// Owning top-level column name -> required PARTITION BY names it owns. Two cases: + /// - Flat key `PARTITION BY a` -> {"a": ["a"]}: "a" is not a subcolumn, it owns itself. + /// - Composite key `PARTITION BY t.ts` -> {"t": ["t.ts"]}: "t.ts" is a subcolumn of "t". + /// - Multiple subcolumns of one owner, `PARTITION BY (t.a, t.b)` -> {"t": ["t.a", "t.b"]}. + std::unordered_map> owner_to_partition_key_columns; + for (const auto & column_name : partition_key_columns) + { + auto resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + const auto & owner_column_name = resolved ? resolved->getNameInStorage() : column_name; + owner_to_partition_key_columns[owner_column_name].push_back(column_name); + } const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; @@ -700,33 +758,16 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (partition_key_column_set.contains(source_column.name) && source_column.name != destination_column.name) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, + if (const auto owned_it = owner_to_partition_key_columns.find(source_column.name); + owned_it != owner_to_partition_key_columns.end()) + verifyPartitionKeyColumn( + source_column, + destination_column, i, - destination_column.name); - - if (partition_key_column_set.contains(source_column.name)) - { - const auto source_time_zone = getDateTimeTimeZoneName(source_column.type); - const auto destination_time_zone = getDateTimeTimeZoneName(destination_column.type); - if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is {} in the source table " - "but {} in the destination. The destination's hive-style partition path is " - "rendered from the source value without converting the timezone, so this " - "would silently shift the exported value by the timezone offset. Use the " - "same timezone in both tables' partition key column.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); - } + owned_it->second, + source_columns_description, + destination_columns_description, + destination_storage_id); /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index dd1ae4c18094..b58ecad9e7bb 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -89,7 +89,7 @@ namespace ExportPartitionUtils const std::string & exception_message, const LoggerPtr & log); - void verifyMergeTreePartitionCompatibility( + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 759736dc06be..7b7533d26d9b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8416,7 +8416,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); if (!dest_storage->isDataLake()) - ExportPartitionUtils::verifyMergeTreePartitionCompatibility(src_snapshot, destination_snapshot); + ExportPartitionUtils::assertPartitionKeyASTAreEqual(src_snapshot, destination_snapshot); zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index e1dd26ea3820..ceafc0a120fd 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -896,3 +896,47 @@ def test_export_part_runtime_cast_failure_propagates_async(cluster): node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_tuple_subcolumn_partition_key_iceberg_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_tuple_subcol_{sfx}" + iceberg = f"iceberg_tuple_subcol_{sfx}" + iceberg_partitioned = f"iceberg_tuple_subcol_part_{sfx}" + + create_error = node.query_and_get_error( + f"CREATE TABLE {iceberg_partitioned} (t Tuple(b Int32, a Int32), val String) " + f"ENGINE = IcebergS3('http://minio1:9001/root/data/{iceberg_partitioned}/', 'minio', 'ClickHouse_Minio_P@ssw0rd') " + f"PARTITION BY t.a" + ) + assert "Unknown field to partition" in create_error, ( + f"Expected Iceberg to reject the tuple subcolumn partition key at CREATE time, " + f"got: {create_error!r}" + ) + + make_mt(node, mt, "t Tuple(a Int32, b Int32), val String", "t.a") + make_iceberg_s3(node, iceberg, "t Tuple(b Int32, a Int32), val String", "val") + + node.query(f"INSERT INTO {mt} VALUES ((1, 99), 'x')") + + part = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt}' AND active ORDER BY name LIMIT 1" + ).strip() + + export_error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "Unknown field to partition" in export_error, ( + f"Expected export validation to reject the tuple subcolumn partition key of {mt}, " + f"got: {export_error!r}" + ) + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b4a1160313aa..a8cd30d50b2c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -526,3 +526,587 @@ def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): ) +def test_export_part_tuple_subcolumn_partition_key_hive_destination_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + s3_table = f"tuple_subcol_s3_table_{postfix}" + + error = node.query_and_get_error(f""" + CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY t.a + """) + assert "BAD_ARGUMENTS" in error and "part of the storage columns" in error, ( + f"Expected hive partition strategy to reject the tuple subcolumn " + f"partition expression at CREATE time, got: {error!r}" + ) + + +def read_exported_files(node, s3_table): + return node.query( + f"SELECT t.a, t.b, val FROM " + f"s3('http://minio1:9001/root/data/{s3_table}/**', 'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'t Tuple(b Int32, a Int32), val String') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + + +def get_export_part_log(node, mt_table): + node.query("SYSTEM FLUSH LOGS") + return node.query( + f"SELECT part_name, error, exception FROM system.part_log " + f"WHERE event_type = 'ExportPart' AND database = currentDatabase() " + f"AND table = '{mt_table}' ORDER BY event_time" + ).strip() + + +def test_export_part_tuple_subcolumn_partition_key_wildcard_destination(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_subcol_wc_mt_table_{postfix}" + s3_table = f"tuple_subcol_wc_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(5) + + part_log = get_export_part_log(node, mt_table) + result = read_exported_files(node, s3_table) + assert result == "1\t99\tx", ( + f"Tuple element values were remapped: source t = (a=1, b=99), " + f"exported files read back (t.a, t.b) as: {result!r}; part_log: {part_log!r}" + ) + + +def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_diff_subcol_mt_table_{postfix}" + s3_table = f"subcol_diff_subcol_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Tuple(b Int32, c Int32), val String) + ENGINE = MergeTree() + PARTITION BY a.b + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Tuple(b Int32, c Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY a.c + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 2), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "different `PARTITION BY` expressions" in error, ( + f"Both tables declare `a` as the same Tuple(b Int32, c Int32) (so the column-cast " + f"check passes and the owner-name-only partition_key_column_set = {{'a'}} in " + f"verifyExportSchemaCastable cannot distinguish `a.b` from `a.c`), but the source " + f"partitions by `a.b` and the destination by `a.c` — a genuinely different " + f"partition key that must be caught by the `PARTITION BY` AST comparison; " + f"got: {error!r}" + ) + + +def test_export_part_tuple_subcolumn_partition_key_owner_column_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_subcol_owner_mt_table_{postfix}" + s3_table = f"tuple_subcol_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), decoy Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Tuple(a Int32, b Int32), t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 100), (2, 200), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"Expected export to reject `t` and `decoy` swapping positions around the " + f"partition key column `t.a`, the same way a plain (non-tuple) partition key " + f"column position swap is rejected; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_tz_mt_table_{postfix}" + s3_table = f"subcol_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY t.ts + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('Asia/Tokyo')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.ts + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Expected export to reject the nested partition key column `t.ts` changing " + f"timezone from UTC to Asia/Tokyo between source and destination, the same way " + f"a top-level DateTime partition key column with mismatched timezones is " + f"rejected (see test_export_part_partition_key_mismatch_variants_are_rejected's " + f"'partition_key_timezone_mismatch' case). getDateTimeTimeZoneName() is called " + f"on the owning column's type (Tuple(...)), not on the actual DateTime " + f"subcolumn's type, so it can never recognize a timezone at all here; " + f"got: {error!r}" + ) + + +def test_export_part_multi_level_subcolumn_partition_key_owner_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nested_subcol_owner_mt_table_{postfix}" + s3_table = f"nested_subcol_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ( + t Tuple(x Tuple(a Int32, b Int32), c Int32), + decoy Tuple(x Tuple(a Int32, b Int32), c Int32), + val String + ) + ENGINE = MergeTree() + PARTITION BY t.x.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} ( + decoy Tuple(x Tuple(a Int32, b Int32), c Int32), + t Tuple(x Tuple(a Int32, b Int32), c Int32), + val String + ) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.x.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((((1, 100), 1000)), (((2, 200), 2000)), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"Expected export to reject `t` and `decoy` swapping positions around the " + f"two-level-deep partition key column `t.x.a`. This only works if " + f"getNameInStorage() resolves all the way to the top-level column `t`, not to " + f"the intermediate level `t.x`; got: {error!r}" + ) + + +def test_export_part_multiple_subcolumn_partition_keys_owner_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_subcol_key_mt_table_{postfix}" + s3_table = f"multi_subcol_key_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ( + t Tuple(a Int32, x Int32), + u Tuple(b Int32, y Int32), + decoy Tuple(b Int32, y Int32), + val String + ) + ENGINE = MergeTree() + PARTITION BY (t.a, u.b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} ( + t Tuple(a Int32, x Int32), + decoy Tuple(b Int32, y Int32), + u Tuple(b Int32, y Int32), + val String + ) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY (t.a, u.b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 10), (2, 20), (3, 30), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column 'u'" in error, ( + f"`t` (owner of key part `t.a`) stays at position 0 on both sides, so the guard " + f"must independently catch `u` (owner of key part `u.b`) swapping positions " + f"with `decoy` — a partition key with two subcolumn-owning columns must have " + f"both validated, not just the first one encountered; got: {error!r}" + ) + + +def test_export_part_mixed_flat_and_subcolumn_partition_key_flat_part_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"mixed_key_mt_table_{postfix}" + s3_table = f"mixed_key_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, t Tuple(b Int32, c Int32), decoy Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, t.b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Int32, t Tuple(b Int32, c Int32), a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY (a, t.b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, (2, 3), 4, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column 'a'" in error, ( + f"`t` (owner of key part `t.b`) stays at position 1 on both sides, so the guard " + f"must independently catch the plain, non-tuple key part `a` swapping positions " + f"with `decoy` — the pre-existing flat-column check and the new subcolumn-owner " + f"resolution must both keep working when combined in one `PARTITION BY` " + f"expression; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_owner_reordered_rejected_even_with_allow_lossy_cast(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"lossy_owner_mt_table_{postfix}" + s3_table = f"lossy_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), decoy Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Tuple(a Int32, b Int32), t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 100), (2, 200), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_allow_lossy_cast = 1" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"The partition-key position/name guard is checked before the " + f"`allow_lossy_cast` early-continue in verifyExportSchemaCastable, so setting " + f"`export_merge_tree_part_allow_lossy_cast = 1` must not suppress the rejection " + f"of `t`/`decoy` swapping positions around the partition key column `t.a`; " + f"got: {error!r}" + ) + + +def test_export_part_tuple_column_real_narrowing_same_order_is_rejected_diagnostic(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_narrow_same_order_mt_table_{postfix}" + s3_table = f"tuple_narrow_same_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int64), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((100, 5000000000), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + print(f"DIAGNOSTIC SAME-ORDER-NARROWING ERROR: {error!r}") + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Expected the genuinely narrowing `b` field (Int64 -> Int32, same field order " + f"on both sides) to be rejected as a lossy cast; got: {error!r}" + ) + + +def test_export_part_tuple_column_fewer_fields_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_fewer_fields_mt_table_{postfix}" + s3_table = f"tuple_fewer_fields_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"`t` is not a partition key column here. The destination's `t` (Tuple(a Int32)) " + f"has fewer fields than the source's `t` (Tuple(a Int32, ts DateTime('UTC'))), " + f"which canBeSafelyCast's tuple arity check (lhs_type_elements_size != " + f"to_tuple_type_elements.size()) must reject; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_key_fewer_fields_mt_table_{postfix}" + s3_table = f"subcol_key_fewer_fields_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY t.ts + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"The partition key `t.ts` requires a field the destination's `t` doesn't have " + f"at all, so verifyPartitionKeyColumn's timezone lookup finds " + f"destination_resolved == nullopt for 't.ts' and defers (continue); the arity " + f"mismatch must still be caught right after by canBeSafelyCast on the whole " + f"`t` column; got: {error!r}" + ) + + +def test_export_part_tuple_column_fewer_fields_in_source_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_fewer_fields_src_mt_table_{postfix}" + s3_table = f"tuple_fewer_fields_src_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Reverse direction of the fewer-fields case: the source's `t` (Tuple(a Int32)) " + f"has fewer fields than the destination's `t` (Tuple(a Int32, ts " + f"DateTime('UTC'))). canBeSafelyCast's arity check is a plain size " + f"inequality, so it must reject this direction too; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_key_fewer_fields_src_mt_table_{postfix}" + s3_table = f"subcol_key_fewer_fields_src_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Here `t.a` (the partition key) exists and resolves fine on both sides, so " + f"verifyPartitionKeyColumn's timezone check passes without throwing; the extra " + f"`ts` field the destination has beyond the source's `t` must still be caught " + f"by canBeSafelyCast's arity check, independent of the partition key guard " + f"succeeding; got: {error!r}" + ) + + From a536161595dec9ef615abbf176e365188f996ad1 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 16:12:28 +0200 Subject: [PATCH 09/18] lowcardinality and nullable Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 7 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../test.py | 81 +++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 17497b8ad056..ca954e022beb 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -643,9 +645,10 @@ namespace ExportPartitionUtils { std::optional getDateTimeTimeZoneName(const DataTypePtr & type) { - if (const auto * datetime_type = checkAndGetDataType(type.get())) + const auto unwrapped_type = removeNullable(removeLowCardinality(type)); + if (const auto * datetime_type = checkAndGetDataType(unwrapped_type.get())) return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = checkAndGetDataType(type.get())) + if (const auto * datetime64_type = checkAndGetDataType(unwrapped_type.get())) return datetime64_type->getTimeZone().getTimeZone(); return {}; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ac8ad03c3b63..0519730a5ee8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6787,7 +6787,7 @@ void MergeTreeData::exportPartToTable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); if (!dest_storage->isDataLake()) - ExportPartitionUtils::verifyMergeTreePartitionCompatibility(source_metadata_ptr, destination_metadata_ptr); + ExportPartitionUtils::assertPartitionKeyASTAreEqual(source_metadata_ptr, destination_metadata_ptr); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index a8cd30d50b2c..4bab7fceba98 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1110,3 +1110,84 @@ def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rej ) +def test_export_part_nullable_datetime_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nullable_tz_mt_table_{postfix}" + s3_table = f"nullable_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts Nullable(DateTime('UTC'))) + ENGINE = MergeTree() + PARTITION BY ts + ORDER BY id + SETTINGS allow_nullable_key = 1, enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts Nullable(DateTime('Asia/Tokyo'))) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY ts + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"getDateTimeTimeZoneName() only recognizes bare DateTime/DateTime64, so wrapping " + f"the partition key in Nullable(...) hides the timezone from it entirely; " + f"canBeSafelyCast() provides no fallback here either, since " + f"DataTypeDateTime::equals() treats any two DateTime types as equal regardless " + f"of timezone by design, so this needs its own dedicated rejection - even " + f"without export_merge_tree_part_allow_lossy_cast being set; got: {error!r}" + ) + + +def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"lc_tz_mt_table_{postfix}" + s3_table = f"lc_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts LowCardinality(DateTime('UTC'))) + ENGINE = MergeTree() + PARTITION BY ts + ORDER BY id + SETTINGS allow_suspicious_low_cardinality_types = 1, enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts LowCardinality(DateTime('Asia/Tokyo'))) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY ts + SETTINGS allow_suspicious_low_cardinality_types = 1 + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Same gap as the Nullable case, but for LowCardinality(DateTime(...)); " + f"got: {error!r}" + ) + + From 9ce7987f46a2c3c5e33b29b48e6cf62b87500b76 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 16:51:05 +0200 Subject: [PATCH 10/18] sensetive column Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 47 ++++++++++ .../test.py | 86 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index ca954e022beb..f7591b628e2a 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #if USE_AVRO @@ -653,11 +655,49 @@ namespace ExportPartitionUtils return {}; } + /// toUnixTimestamp-family functions return the same value regardless of the argument's + /// declared timezone, so a column used only inside them doesn't need the check below. + void collectColumnsRequiringTimezoneCheck( + const ASTPtr & node, + bool inside_timezone_invariant_function, + std::unordered_set & columns_requiring_timezone_check) + { + if (!node) + return; + + if (const auto * identifier = node->as()) + { + if (!inside_timezone_invariant_function) + columns_requiring_timezone_check.insert(identifier->name()); + return; + } + + if (const auto * function = node->as()) + { + static const std::unordered_set timezone_invariant_functions = { + "toUnixTimestamp", + "toUnixTimestamp64Milli", + "toUnixTimestamp64Micro", + "toUnixTimestamp64Nano", + }; + const bool wraps_in_invariant_function + = inside_timezone_invariant_function || timezone_invariant_functions.contains(function->name); + if (function->arguments) + for (const auto & argument : function->arguments->children) + collectColumnsRequiringTimezoneCheck(argument, wraps_in_invariant_function, columns_requiring_timezone_check); + return; + } + + for (const auto & child : node->children) + collectColumnsRequiringTimezoneCheck(child, inside_timezone_invariant_function, columns_requiring_timezone_check); + } + void verifyPartitionKeyColumn( const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, size_t position, const std::vector & required_columns, + const std::unordered_set & columns_requiring_timezone_check, const ColumnsDescription & source_columns_description, const ColumnsDescription & destination_columns_description, const StorageID & destination_storage_id) @@ -676,6 +716,9 @@ namespace ExportPartitionUtils for (const auto & column_name : required_columns) { + if (!columns_requiring_timezone_check.contains(column_name)) + continue; + const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); const auto destination_resolved = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); @@ -753,6 +796,9 @@ namespace ExportPartitionUtils owner_to_partition_key_columns[owner_column_name].push_back(column_name); } + std::unordered_set columns_requiring_timezone_check; + collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), false, columns_requiring_timezone_check); + const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); @@ -768,6 +814,7 @@ namespace ExportPartitionUtils destination_column, i, owned_it->second, + columns_requiring_timezone_check, source_columns_description, destination_columns_description, destination_storage_id); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 4bab7fceba98..59aaa66eaf91 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1191,3 +1191,89 @@ def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_ ) +def test_export_part_timezone_invariant_expression_partition_key_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_invariant_mt_table_{postfix}" + s3_table = f"tz_invariant_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"Expected the export to succeed and produce 1 row: toUnixTimestamp(ts) does " + f"not depend on ts's declared timezone, so a mismatched timezone between " + f"source and destination must not block it; got count={result!r}" + ) + + +def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_sensitive_mt_table_{postfix}" + s3_table = f"tz_sensitive_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toDate(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toDate(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " + f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " + f"remain protected by the timezone guard - the allowlist in " + f"collectTimezoneSensitiveColumns() must not be so broad that it lets this " + f"through too; got: {error!r}" + ) + + From f6be56b2af392a3fe41451ea40d9763ae6080d74 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 17:38:37 +0200 Subject: [PATCH 11/18] resolve some cases Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 29 ++++++------ .../test.py | 44 ++++++++++++++++++- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index f7591b628e2a..47c3bb095e93 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -655,41 +655,40 @@ namespace ExportPartitionUtils return {}; } - /// toUnixTimestamp-family functions return the same value regardless of the argument's - /// declared timezone, so a column used only inside them doesn't need the check below. void collectColumnsRequiringTimezoneCheck( const ASTPtr & node, - bool inside_timezone_invariant_function, + const ASTFunction * immediate_parent_function, std::unordered_set & columns_requiring_timezone_check) { if (!node) return; if (const auto * identifier = node->as()) - { - if (!inside_timezone_invariant_function) - columns_requiring_timezone_check.insert(identifier->name()); - return; - } - - if (const auto * function = node->as()) { static const std::unordered_set timezone_invariant_functions = { "toUnixTimestamp", + "toUnixTimestamp64Second", "toUnixTimestamp64Milli", "toUnixTimestamp64Micro", "toUnixTimestamp64Nano", }; - const bool wraps_in_invariant_function - = inside_timezone_invariant_function || timezone_invariant_functions.contains(function->name); + const bool wrapped_by_invariant_function + = immediate_parent_function && timezone_invariant_functions.contains(immediate_parent_function->name); + if (!wrapped_by_invariant_function) + columns_requiring_timezone_check.insert(identifier->name()); + return; + } + + if (const auto * function = node->as()) + { if (function->arguments) for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, wraps_in_invariant_function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(argument, function, columns_requiring_timezone_check); return; } for (const auto & child : node->children) - collectColumnsRequiringTimezoneCheck(child, inside_timezone_invariant_function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(child, nullptr, columns_requiring_timezone_check); } void verifyPartitionKeyColumn( @@ -797,7 +796,7 @@ namespace ExportPartitionUtils } std::unordered_set columns_requiring_timezone_check; - collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), false, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), nullptr, columns_requiring_timezone_check); const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 59aaa66eaf91..58849e43864c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1272,8 +1272,48 @@ def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(clu f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " f"remain protected by the timezone guard - the allowlist in " - f"collectTimezoneSensitiveColumns() must not be so broad that it lets this " - f"through too; got: {error!r}" + f"collectColumnsRequiringTimezoneCheck() must not be so broad that it lets " + f"this through too; got: {error!r}" + ) + + +def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_nested_mt_table_{postfix}" + s3_table = f"tz_nested_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(toDate(ts)) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(toDate(ts)) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"toDate(ts) inside toUnixTimestamp(toDate(ts)) already produces a " + f"timezone-dependent value before toUnixTimestamp ever runs, so wrapping it " + f"in an outer invariant function must not exempt ts from the check - only " + f"ts's *immediate* parent function determines that; got: {error!r}" ) From 7bf6fa564dc8d0ae809796516ca7f0ea4a882795 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 18:13:17 +0200 Subject: [PATCH 12/18] fix problems with func Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 13 +- .../test.py | 134 ++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 47c3bb095e93..cec3fe0289d5 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -657,7 +657,7 @@ namespace ExportPartitionUtils void collectColumnsRequiringTimezoneCheck( const ASTPtr & node, - const ASTFunction * immediate_parent_function, + const ASTFunction * effective_parent_function, std::unordered_set & columns_requiring_timezone_check) { if (!node) @@ -673,7 +673,7 @@ namespace ExportPartitionUtils "toUnixTimestamp64Nano", }; const bool wrapped_by_invariant_function - = immediate_parent_function && timezone_invariant_functions.contains(immediate_parent_function->name); + = effective_parent_function && timezone_invariant_functions.contains(effective_parent_function->name); if (!wrapped_by_invariant_function) columns_requiring_timezone_check.insert(identifier->name()); return; @@ -681,9 +681,16 @@ namespace ExportPartitionUtils if (const auto * function = node->as()) { + static const std::unordered_set value_preserving_functions = { + "identity", + "assumeNotNull", + "materialize", + }; + const ASTFunction * next_parent_function + = value_preserving_functions.contains(function->name) ? effective_parent_function : function; if (function->arguments) for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(argument, next_parent_function, columns_requiring_timezone_check); return; } diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 58849e43864c..aff4d13bb15f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1317,3 +1317,137 @@ def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is ) +def test_export_part_unix_timestamp64_second_partition_key_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_u64s_mt_table_{postfix}" + s3_table = f"tz_u64s_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime64(3, 'UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp64Second(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime64(3, 'Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp64Second(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00.000')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime64(3, \\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"toUnixTimestamp64Second(ts) is documented as being relative to UTC, not the " + f"declared timezone of ts (see toUnixTimestamp64Second.cpp's own " + f"documentation), so a mismatched timezone between source and destination " + f"must not block this export; got count={result!r}" + ) + + +def test_export_part_value_preserving_function_wrapped_in_invariant_function_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_passthrough_mt_table_{postfix}" + s3_table = f"tz_passthrough_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(assumeNotNull(ts)) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(assumeNotNull(ts)) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"assumeNotNull(ts) is identity on the underlying value, so it must not break " + f"the chain between ts and the enclosing toUnixTimestamp - the check must look " + f"past value-preserving wrapper functions, not just the immediate parent; " + f"got count={result!r}" + ) + + +def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper_is_rejected( + cluster, +): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_nested_passthrough_mt_table_{postfix}" + s3_table = f"tz_nested_passthrough_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"toDate(ts) is still timezone-sensitive even when reached through the " + f"value-preserving assumeNotNull() and wrapped by the invariant " + f"toUnixTimestamp() two levels up - skipping past passthrough functions must " + f"not also skip past genuinely timezone-sensitive ones; got: {error!r}" + ) + + From 2d7f3b2e68d33bd003c54182d197b56c906e0b29 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 18:36:22 +0200 Subject: [PATCH 13/18] issue with func with tz Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 4 ++ docs/en/antalya/partition_export.md | 8 +-- .../test.py | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 2ff4a7301674..4eb9ca299ab9 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -55,6 +55,10 @@ Source and destination tables must be 100% compatible: This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. +4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - if a partition key column has type `DateTime`/`DateTime64` and its declared timezone differs between the source and destination tables, the export is rejected with `BAD_ARGUMENTS`, because the partition value the source computed at insert time may not match what the destination would compute for the same physical instant. This check is skipped only when the partition key wraps the column in a function that is known to return a timezone-independent result, such as `toUnixTimestamp`, `toUnixTimestamp64Second`, `toUnixTimestamp64Milli`, `toUnixTimestamp64Micro`, or `toUnixTimestamp64Nano`, optionally passed through value-preserving wrappers like `assumeNotNull`, `materialize`, or `identity` (e.g. `PARTITION BY toUnixTimestamp(assumeNotNull(ts))`). + + This is a fixed allowlist, not a general proof of timezone-independence, so some genuinely timezone-independent expressions are currently rejected too. For example, `PARTITION BY toUInt32(ts)` is also timezone-independent because converting `DateTime` to a numeric type uses the stored Unix timestamp and does not perform a calendar-time computation. However, `toUInt32` is not on the allowlist, so the export is rejected even though it would be safe. + In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. ## Settings diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index ccd18914845b..13e859a03241 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -45,11 +45,12 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: 1. **Identical schemas** - same columns, types, and order 2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements) for a worked example and the exact error message. +3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the exact error message. +4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - a mismatched declared timezone between source and destination is rejected unless the partition key column is wrapped in a known timezone-independent function such as `toUnixTimestamp`. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for the full list and its known limitations. ## Settings @@ -259,5 +260,4 @@ WHERE source_table = 'rmt_table' AND destination_table = 's3_table'; ## Related Features -- [ALTER TABLE EXPORT PART](/docs/en/engines/table-engines/mergetree-family/part_export.md) - Export individual parts (non-replicated) - +- [ALTER TABLE EXPORT PART](/docs/en/antalya/part_export.md) - Export individual parts (non-replicated) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index aff4d13bb15f..86cc1c0da8c1 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1451,3 +1451,64 @@ def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper ) +# `timezone_invariant_functions` in `collectColumnsRequiringTimezoneCheck` is a +# fixed allowlist, not a proof of timezone-independence. This test documents the +# conservative rejection of safe expressions that the allowlist does not recognize. +@pytest.mark.parametrize( + "function_name, source_type, destination_type, value", + [ + pytest.param( + "toUInt32", + "DateTime('UTC')", + "DateTime('Asia/Tokyo')", + "2024-03-05 15:00:00", + id="toUInt32-DateTime", + ), + pytest.param( + "toInt64", + "DateTime64(3, 'UTC')", + "DateTime64(3, 'Asia/Tokyo')", + "2024-03-05 15:00:00.000", + id="toInt64-DateTime64", + ), + ], +) +def test_export_part_unrecognized_timezone_invariant_function_is_rejected( + cluster, function_name, source_type, destination_type, value +): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_unrecognized_mt_table_{postfix}" + s3_table = f"tz_unrecognized_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts {source_type}) + ENGINE = MergeTree() + PARTITION BY {function_name}(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts {destination_type}) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY {function_name}(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '{value}')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"`{function_name}(ts)` uses the stored epoch value and does not perform a " + f"calendar-time conversion, but `{function_name}` is not on the allowlist. " + f"The export must be conservatively rejected; got: {error!r}" + ) From 16804520c21d1221099415c66c8b4fe23bdb67d3 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 11:53:21 +0200 Subject: [PATCH 14/18] refuse type checking Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 14 +- docs/en/antalya/partition_export.md | 7 +- .../MergeTree/ExportPartitionUtils.cpp | 160 +--- .../test.py | 805 ++---------------- .../test.py | 23 +- 5 files changed, 132 insertions(+), 877 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 4eb9ca299ab9..63ffdc8d561f 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -47,17 +47,17 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 ## Requirements -Source and destination tables must be 100% compatible: +Source and destination tables must support positional schema conversion: -1. **Identical schemas** - same columns, types, and order -2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`. It is not enough for the `PARTITION BY` expressions to be textually identical: every column that is part of the source table's partition key must also sit at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`, so the export is rejected with `BAD_ARGUMENTS: partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. +2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. +3. **Partition key columns at the same position** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. - This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. + This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. -4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - if a partition key column has type `DateTime`/`DateTime64` and its declared timezone differs between the source and destination tables, the export is rejected with `BAD_ARGUMENTS`, because the partition value the source computed at insert time may not match what the destination would compute for the same physical instant. This check is skipped only when the partition key wraps the column in a function that is known to return a timezone-independent result, such as `toUnixTimestamp`, `toUnixTimestamp64Second`, `toUnixTimestamp64Milli`, `toUnixTimestamp64Micro`, or `toUnixTimestamp64Nano`, optionally passed through value-preserving wrappers like `assumeNotNull`, `materialize`, or `identity` (e.g. `PARTITION BY toUnixTimestamp(assumeNotNull(ts))`). + For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. The position of `a` inside a named `Tuple` does not have to match because casts between named tuples match their elements by name. For example, exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is allowed when both tables use `PARTITION BY t.a`. - This is a fixed allowlist, not a general proof of timezone-independence, so some genuinely timezone-independent expressions are currently rejected too. For example, `PARTITION BY toUInt32(ts)` is also timezone-independent because converting `DateTime` to a numeric type uses the stored Unix timestamp and does not perform a calendar-time computation. However, `toUInt32` is not on the allowlist, so the export is rejected even though it would be safe. + For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 13e859a03241..1dc1b34df61e 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -47,10 +47,9 @@ TO TABLE [destination_database.]destination_table `EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: -1. **Identical schemas** - same columns, types, and order -2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the exact error message. -4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - a mismatched declared timezone between source and destination is rejected unless the partition key column is wrapped in a known timezone-independent function such as `toUnixTimestamp`. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for the full list and its known limitations. +1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. +2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. +3. **Partition key columns at the same position** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index cec3fe0289d5..f302090a4905 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,17 +13,10 @@ #include #include #include -#include -#include -#include -#include #include -#include #include #include #include -#include -#include #include #if USE_AVRO @@ -643,112 +636,6 @@ namespace ExportPartitionUtils } #endif - namespace - { - std::optional getDateTimeTimeZoneName(const DataTypePtr & type) - { - const auto unwrapped_type = removeNullable(removeLowCardinality(type)); - if (const auto * datetime_type = checkAndGetDataType(unwrapped_type.get())) - return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = checkAndGetDataType(unwrapped_type.get())) - return datetime64_type->getTimeZone().getTimeZone(); - return {}; - } - - void collectColumnsRequiringTimezoneCheck( - const ASTPtr & node, - const ASTFunction * effective_parent_function, - std::unordered_set & columns_requiring_timezone_check) - { - if (!node) - return; - - if (const auto * identifier = node->as()) - { - static const std::unordered_set timezone_invariant_functions = { - "toUnixTimestamp", - "toUnixTimestamp64Second", - "toUnixTimestamp64Milli", - "toUnixTimestamp64Micro", - "toUnixTimestamp64Nano", - }; - const bool wrapped_by_invariant_function - = effective_parent_function && timezone_invariant_functions.contains(effective_parent_function->name); - if (!wrapped_by_invariant_function) - columns_requiring_timezone_check.insert(identifier->name()); - return; - } - - if (const auto * function = node->as()) - { - static const std::unordered_set value_preserving_functions = { - "identity", - "assumeNotNull", - "materialize", - }; - const ASTFunction * next_parent_function - = value_preserving_functions.contains(function->name) ? effective_parent_function : function; - if (function->arguments) - for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, next_parent_function, columns_requiring_timezone_check); - return; - } - - for (const auto & child : node->children) - collectColumnsRequiringTimezoneCheck(child, nullptr, columns_requiring_timezone_check); - } - - void verifyPartitionKeyColumn( - const ColumnWithTypeAndName & source_column, - const ColumnWithTypeAndName & destination_column, - size_t position, - const std::vector & required_columns, - const std::unordered_set & columns_requiring_timezone_check, - const ColumnsDescription & source_columns_description, - const ColumnsDescription & destination_columns_description, - const StorageID & destination_storage_id) - { - if (source_column.name != destination_column.name) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, - position, - destination_column.name); - - for (const auto & column_name : required_columns) - { - if (!columns_requiring_timezone_check.contains(column_name)) - continue; - - const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - const auto destination_resolved - = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - if (!source_resolved || !destination_resolved) - continue; - - const auto source_time_zone = getDateTimeTimeZoneName(source_resolved->type); - const auto destination_time_zone = getDateTimeTimeZoneName(destination_resolved->type); - if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is {} in the source table " - "but {} in the destination. The destination's hive-style partition path is " - "rendered from the source value without converting the timezone, so this " - "would silently shift the exported value by the timezone offset. Use the " - "same timezone in both tables' partition key column.", - destination_storage_id.getFullTableName(), - column_name, - source_resolved->type->getName(), - destination_resolved->type->getName()); - } - } - } - void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -787,24 +674,17 @@ namespace ExportPartitionUtils context); const auto & source_columns_description = source_metadata->getColumns(); - const auto & destination_columns_description = destination_metadata->getColumns(); - auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); - - /// Owning top-level column name -> required PARTITION BY names it owns. Two cases: - /// - Flat key `PARTITION BY a` -> {"a": ["a"]}: "a" is not a subcolumn, it owns itself. - /// - Composite key `PARTITION BY t.ts` -> {"t": ["t.ts"]}: "t.ts" is a subcolumn of "t". - /// - Multiple subcolumns of one owner, `PARTITION BY (t.a, t.b)` -> {"t": ["t.a", "t.b"]}. - std::unordered_map> owner_to_partition_key_columns; - for (const auto & column_name : partition_key_columns) - { - auto resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - const auto & owner_column_name = resolved ? resolved->getNameInStorage() : column_name; - owner_to_partition_key_columns[owner_column_name].push_back(column_name); + /// Collect the top-level columns that own columns or subcolumns required by `PARTITION BY`. + /// For example, both `PARTITION BY t.a` and `PARTITION BY (t.a, t.b)` add `t`. + std::unordered_set partition_key_owner_columns; + for (const auto & column_or_subcolumn_name : source_metadata->getColumnsRequiredForPartitionKey()) + { + auto resolved = source_columns_description.tryGetColumnOrSubcolumn( + GetColumnsOptions::All, column_or_subcolumn_name); + const auto & column_name = resolved ? resolved->getNameInStorage() : column_or_subcolumn_name; + partition_key_owner_columns.insert(column_name); } - std::unordered_set columns_requiring_timezone_check; - collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), nullptr, columns_requiring_timezone_check); - const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); @@ -813,17 +693,19 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (const auto owned_it = owner_to_partition_key_columns.find(source_column.name); - owned_it != owner_to_partition_key_columns.end()) - verifyPartitionKeyColumn( - source_column, - destination_column, + if (partition_key_owner_columns.contains(source_column.name) && source_column.name != destination_column.name) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, i, - owned_it->second, - columns_requiring_timezone_check, - source_columns_description, - destination_columns_description, - destination_storage_id); + destination_column.name); + } /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 86cc1c0da8c1..b77c2f8159cb 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -322,6 +322,7 @@ class RejectedPartExportCase(NamedTuple): dst_partition_by: str insert_values: str error_substrings: tuple = () + partition_strategy: str = "hive" REJECTED_PART_EXPORT_CASES = [ @@ -332,7 +333,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="b Int32, a Int32", dst_partition_by="a", insert_values="(1, 1), (1, 2)", - error_substrings=("partition key column",), + error_substrings=( + "partition key column 'a' is at position 0 in the source table", + ), ), id="same_partition_key_different_column_order_single_column", ), @@ -343,7 +346,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="c Int32, b Int32, a Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", - error_substrings=("partition key column",), + error_substrings=( + "partition key column 'a' is at position 0 in the source table", + ), ), id="same_partition_key_different_column_order_multi_column", ), @@ -354,7 +359,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_order_mismatch", ), @@ -365,7 +372,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_fewer_in_destination", ), @@ -376,20 +385,51 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_more_in_destination", ), pytest.param( RejectedPartExportCase( - src_columns="id Int64, ts DateTime('UTC')", - src_partition_by="ts", - dst_columns="id Int64, ts DateTime('Asia/Tokyo')", - dst_partition_by="ts", - insert_values="(1, '2024-03-05 15:00:00')", - error_substrings=("timezone",), + src_columns="ts DateTime, category String, decoy DateTime, val String", + src_partition_by="(toYYYYMM(ts), category)", + dst_columns="decoy DateTime, category String, ts DateTime, val String", + dst_partition_by="(toYYYYMM(ts), category)", + insert_values=( + "('2024-03-05 15:00:00', 'category', " + "'2024-03-06 15:00:00', 'x')" + ), + error_substrings=( + "partition key column 'ts' is at position 0 in the source table", + ), + partition_strategy="wildcard", ), - id="partition_key_timezone_mismatch", + id="function_and_column_partition_key_owner_reordered", + ), + pytest.param( + RejectedPartExportCase( + src_columns=( + "t Tuple(ts DateTime, value Int32), category String, " + "decoy Tuple(ts DateTime, value Int32), val String" + ), + src_partition_by="(toYYYYMM(t.ts), category)", + dst_columns=( + "decoy Tuple(ts DateTime, value Int32), category String, " + "t Tuple(ts DateTime, value Int32), val String" + ), + dst_partition_by="(toYYYYMM(t.ts), category)", + insert_values=( + "(('2024-03-05 15:00:00', 1), 'category', " + "('2024-03-06 15:00:00', 2), 'x')" + ), + error_substrings=( + "partition key column 't' is at position 0 in the source table", + ), + partition_strategy="wildcard", + ), + id="function_over_subcolumn_partition_key_owner_reordered", ), ] @@ -411,9 +451,14 @@ def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case) SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) + filename = ( + f"{s3_table}/{{_partition_id}}/{{_file}}" + if case.partition_strategy == "wildcard" + else s3_table + ) node.query(f""" CREATE TABLE {s3_table} ({case.dst_columns}) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + ENGINE = S3(s3_conn, filename='{filename}', format=Parquet, partition_strategy='{case.partition_strategy}') PARTITION BY {case.dst_partition_by} """) @@ -429,8 +474,11 @@ def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case) for substring in case.error_substrings: assert substring in error, f"Expected {substring!r} in error, got: {error}" - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + if case.partition_strategy == "hive": + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, ( + f"Expected 0 rows in destination after rejected export, got {count}" + ) def test_export_part_multi_column_partition_key_success(cluster): @@ -473,77 +521,6 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_ok_mt_table_{postfix}" - s3_table = f"tz_ok_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY id - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY id - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(5) - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 1, f"Expected 1 row in destination after export, got {count}" - - source_ts = node.query(f"SELECT ts FROM {mt_table}").strip() - assert source_ts == "2024-03-05 15:00:00", f"Unexpected source value: {source_ts}" - - dest_ts = node.query(f"SELECT ts FROM {s3_table}").strip() - assert dest_ts == "2024-03-06 00:00:00", ( - f"Expected the exported value to be the same instant displayed in the " - f"destination's Asia/Tokyo timezone ('2024-03-06 00:00:00'), got: {dest_ts}" - ) - - source_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip()) - dest_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_table}").strip()) - assert source_unix_ts == dest_unix_ts, ( - f"Expected exported DateTime value to be preserved regardless of the " - f"destination column's timezone, got source={source_unix_ts}, dest={dest_unix_ts}" - ) - - -def test_export_part_tuple_subcolumn_partition_key_hive_destination_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - s3_table = f"tuple_subcol_s3_table_{postfix}" - - error = node.query_and_get_error(f""" - CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY t.a - """) - assert "BAD_ARGUMENTS" in error and "part of the storage columns" in error, ( - f"Expected hive partition strategy to reject the tuple subcolumn " - f"partition expression at CREATE time, got: {error!r}" - ) - - def read_exported_files(node, s3_table): return node.query( f"SELECT t.a, t.b, val FROM " @@ -562,7 +539,7 @@ def get_export_part_log(node, mt_table): ).strip() -def test_export_part_tuple_subcolumn_partition_key_wildcard_destination(cluster): +def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -635,10 +612,15 @@ def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(clu error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" ) - assert "BAD_ARGUMENTS" in error and "different `PARTITION BY` expressions" in error, ( + assert ( + "BAD_ARGUMENTS" in error + and "source and destination tables have different `PARTITION BY` expressions" + in error + ), ( f"Both tables declare `a` as the same Tuple(b Int32, c Int32) (so the column-cast " - f"check passes and the owner-name-only partition_key_column_set = {{'a'}} in " - f"verifyExportSchemaCastable cannot distinguish `a.b` from `a.c`), but the source " + f"check passes and the owner-name-only `partition_key_owner_columns` contains " + f"only `a`, so `verifyExportSchemaCastable` cannot distinguish `a.b` from " + f"`a.c`), but the source " f"partitions by `a.b` and the destination by `a.c` — a genuinely different " f"partition key that must be caught by the `PARTITION BY` AST comparison; " f"got: {error!r}" @@ -684,29 +666,37 @@ def test_export_part_tuple_subcolumn_partition_key_owner_column_reordered_is_rej ) -def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(cluster): +def test_export_part_multiple_partition_key_subcolumns_with_same_owner_reordered_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_tz_mt_table_{postfix}" - s3_table = f"subcol_tz_s3_table_{postfix}" + mt_table = f"same_owner_subcolumns_mt_table_{postfix}" + s3_table = f"same_owner_subcolumns_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + CREATE TABLE {mt_table} ( + t Tuple(a Int32, b Int32), + decoy Tuple(a Int32, b Int32), + val String + ) ENGINE = MergeTree() - PARTITION BY t.ts + PARTITION BY (t.a, t.b) ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('Asia/Tokyo')), val String) + CREATE TABLE {s3_table} ( + decoy Tuple(a Int32, b Int32), + t Tuple(a Int32, b Int32), + val String + ) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.ts + PARTITION BY (t.a, t.b) """) - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + node.query(f"INSERT INTO {mt_table} VALUES ((1, 10), (2, 20), 'x')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -716,15 +706,9 @@ def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(clust error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Expected export to reject the nested partition key column `t.ts` changing " - f"timezone from UTC to Asia/Tokyo between source and destination, the same way " - f"a top-level DateTime partition key column with mismatched timezones is " - f"rejected (see test_export_part_partition_key_mismatch_variants_are_rejected's " - f"'partition_key_timezone_mismatch' case). getDateTimeTimeZoneName() is called " - f"on the owning column's type (Tuple(...)), not on the actual DateTime " - f"subcolumn's type, so it can never recognize a timezone at all here; " - f"got: {error!r}" + assert "BAD_ARGUMENTS" in error and "partition key column 't'" in error, ( + f"Expected both `t.a` and `t.b` to resolve to the same top-level owner `t` " + f"and reject swapping `t` with `decoy`; got: {error!r}" ) @@ -771,7 +755,7 @@ def test_export_part_multi_level_subcolumn_partition_key_owner_reordered_is_reje assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( f"Expected export to reject `t` and `decoy` swapping positions around the " f"two-level-deep partition key column `t.x.a`. This only works if " - f"getNameInStorage() resolves all the way to the top-level column `t`, not to " + f"`getNameInStorage` resolves all the way to the top-level column `t`, not to " f"the intermediate level `t.x`; got: {error!r}" ) @@ -907,608 +891,3 @@ def test_export_part_subcolumn_partition_key_owner_reordered_rejected_even_with_ f"of `t`/`decoy` swapping positions around the partition key column `t.a`; " f"got: {error!r}" ) - - -def test_export_part_tuple_column_real_narrowing_same_order_is_rejected_diagnostic(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_narrow_same_order_mt_table_{postfix}" - s3_table = f"tuple_narrow_same_order_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, b Int64), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, b Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((100, 5000000000), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - print(f"DIAGNOSTIC SAME-ORDER-NARROWING ERROR: {error!r}") - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Expected the genuinely narrowing `b` field (Int64 -> Int32, same field order " - f"on both sides) to be rejected as a lossy cast; got: {error!r}" - ) - - -def test_export_part_tuple_column_fewer_fields_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_fewer_fields_mt_table_{postfix}" - s3_table = f"tuple_fewer_fields_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"`t` is not a partition key column here. The destination's `t` (Tuple(a Int32)) " - f"has fewer fields than the source's `t` (Tuple(a Int32, ts DateTime('UTC'))), " - f"which canBeSafelyCast's tuple arity check (lhs_type_elements_size != " - f"to_tuple_type_elements.size()) must reject; got: {error!r}" - ) - - -def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_key_fewer_fields_mt_table_{postfix}" - s3_table = f"subcol_key_fewer_fields_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = MergeTree() - PARTITION BY t.ts - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"The partition key `t.ts` requires a field the destination's `t` doesn't have " - f"at all, so verifyPartitionKeyColumn's timezone lookup finds " - f"destination_resolved == nullopt for 't.ts' and defers (continue); the arity " - f"mismatch must still be caught right after by canBeSafelyCast on the whole " - f"`t` column; got: {error!r}" - ) - - -def test_export_part_tuple_column_fewer_fields_in_source_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_fewer_fields_src_mt_table_{postfix}" - s3_table = f"tuple_fewer_fields_src_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Reverse direction of the fewer-fields case: the source's `t` (Tuple(a Int32)) " - f"has fewer fields than the destination's `t` (Tuple(a Int32, ts " - f"DateTime('UTC'))). canBeSafelyCast's arity check is a plain size " - f"inequality, so it must reject this direction too; got: {error!r}" - ) - - -def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_key_fewer_fields_src_mt_table_{postfix}" - s3_table = f"subcol_key_fewer_fields_src_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32), val String) - ENGINE = MergeTree() - PARTITION BY t.a - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Here `t.a` (the partition key) exists and resolves fine on both sides, so " - f"verifyPartitionKeyColumn's timezone check passes without throwing; the extra " - f"`ts` field the destination has beyond the source's `t` must still be caught " - f"by canBeSafelyCast's arity check, independent of the partition key guard " - f"succeeding; got: {error!r}" - ) - - -def test_export_part_nullable_datetime_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"nullable_tz_mt_table_{postfix}" - s3_table = f"nullable_tz_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts Nullable(DateTime('UTC'))) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY id - SETTINGS allow_nullable_key = 1, enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts Nullable(DateTime('Asia/Tokyo'))) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"getDateTimeTimeZoneName() only recognizes bare DateTime/DateTime64, so wrapping " - f"the partition key in Nullable(...) hides the timezone from it entirely; " - f"canBeSafelyCast() provides no fallback here either, since " - f"DataTypeDateTime::equals() treats any two DateTime types as equal regardless " - f"of timezone by design, so this needs its own dedicated rejection - even " - f"without export_merge_tree_part_allow_lossy_cast being set; got: {error!r}" - ) - - -def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"lc_tz_mt_table_{postfix}" - s3_table = f"lc_tz_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts LowCardinality(DateTime('UTC'))) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY id - SETTINGS allow_suspicious_low_cardinality_types = 1, enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts LowCardinality(DateTime('Asia/Tokyo'))) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY ts - SETTINGS allow_suspicious_low_cardinality_types = 1 - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Same gap as the Nullable case, but for LowCardinality(DateTime(...)); " - f"got: {error!r}" - ) - - -def test_export_part_timezone_invariant_expression_partition_key_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_invariant_mt_table_{postfix}" - s3_table = f"tz_invariant_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"Expected the export to succeed and produce 1 row: toUnixTimestamp(ts) does " - f"not depend on ts's declared timezone, so a mismatched timezone between " - f"source and destination must not block it; got count={result!r}" - ) - - -def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_sensitive_mt_table_{postfix}" - s3_table = f"tz_sensitive_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toDate(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toDate(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " - f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " - f"remain protected by the timezone guard - the allowlist in " - f"collectColumnsRequiringTimezoneCheck() must not be so broad that it lets " - f"this through too; got: {error!r}" - ) - - -def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_nested_mt_table_{postfix}" - s3_table = f"tz_nested_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(toDate(ts)) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(toDate(ts)) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"toDate(ts) inside toUnixTimestamp(toDate(ts)) already produces a " - f"timezone-dependent value before toUnixTimestamp ever runs, so wrapping it " - f"in an outer invariant function must not exempt ts from the check - only " - f"ts's *immediate* parent function determines that; got: {error!r}" - ) - - -def test_export_part_unix_timestamp64_second_partition_key_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_u64s_mt_table_{postfix}" - s3_table = f"tz_u64s_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime64(3, 'UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp64Second(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime64(3, 'Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp64Second(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00.000')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime64(3, \\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"toUnixTimestamp64Second(ts) is documented as being relative to UTC, not the " - f"declared timezone of ts (see toUnixTimestamp64Second.cpp's own " - f"documentation), so a mismatched timezone between source and destination " - f"must not block this export; got count={result!r}" - ) - - -def test_export_part_value_preserving_function_wrapped_in_invariant_function_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_passthrough_mt_table_{postfix}" - s3_table = f"tz_passthrough_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(assumeNotNull(ts)) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(assumeNotNull(ts)) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"assumeNotNull(ts) is identity on the underlying value, so it must not break " - f"the chain between ts and the enclosing toUnixTimestamp - the check must look " - f"past value-preserving wrapper functions, not just the immediate parent; " - f"got count={result!r}" - ) - - -def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper_is_rejected( - cluster, -): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_nested_passthrough_mt_table_{postfix}" - s3_table = f"tz_nested_passthrough_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"toDate(ts) is still timezone-sensitive even when reached through the " - f"value-preserving assumeNotNull() and wrapped by the invariant " - f"toUnixTimestamp() two levels up - skipping past passthrough functions must " - f"not also skip past genuinely timezone-sensitive ones; got: {error!r}" - ) - - -# `timezone_invariant_functions` in `collectColumnsRequiringTimezoneCheck` is a -# fixed allowlist, not a proof of timezone-independence. This test documents the -# conservative rejection of safe expressions that the allowlist does not recognize. -@pytest.mark.parametrize( - "function_name, source_type, destination_type, value", - [ - pytest.param( - "toUInt32", - "DateTime('UTC')", - "DateTime('Asia/Tokyo')", - "2024-03-05 15:00:00", - id="toUInt32-DateTime", - ), - pytest.param( - "toInt64", - "DateTime64(3, 'UTC')", - "DateTime64(3, 'Asia/Tokyo')", - "2024-03-05 15:00:00.000", - id="toInt64-DateTime64", - ), - ], -) -def test_export_part_unrecognized_timezone_invariant_function_is_rejected( - cluster, function_name, source_type, destination_type, value -): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_unrecognized_mt_table_{postfix}" - s3_table = f"tz_unrecognized_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts {source_type}) - ENGINE = MergeTree() - PARTITION BY {function_name}(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts {destination_type}) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY {function_name}(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '{value}')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"`{function_name}(ts)` uses the stored epoch value and does not perform a " - f"calendar-time conversion, but `{function_name}` is not on the allowlist. " - f"The export must be conservatively rejected; got: {error!r}" - ) diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 0b38f97df4a5..aeb738496b86 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1789,7 +1789,9 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_order_mismatch", ), @@ -1800,7 +1802,9 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_fewer_in_destination", ), @@ -1811,21 +1815,12 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_more_in_destination", ), - pytest.param( - RejectedPartitionExportCase( - src_columns="id Int64, ts DateTime('UTC')", - src_partition_by="ts", - dst_columns="id Int64, ts DateTime('Asia/Tokyo')", - dst_partition_by="ts", - insert_values="(1, '2024-03-05 15:00:00')", - error_substrings=("timezone",), - ), - id="partition_key_timezone_mismatch", - ), ] From acf634a799834a3f8c9f70ebaf324e3d14bdc0ad Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 12:25:54 +0200 Subject: [PATCH 15/18] fix tupleElement Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 6 +- docs/en/antalya/partition_export.md | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 74 +++++++++++++++---- .../test.py | 52 +++++-------- 4 files changed, 86 insertions(+), 48 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 63ffdc8d561f..24ce21db5b02 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,11 +51,13 @@ Source and destination tables must support positional schema conversion: 1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. -3. **Partition key columns at the same position** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column has a named `Tuple` type, its element names must also be declared in the same order, recursively for nested tuples. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. - For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. The position of `a` inside a named `Tuple` does not have to match because casts between named tuples match their elements by name. For example, exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is allowed when both tables use `PARTITION BY t.a`. + For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + + In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`. For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 1dc1b34df61e..90b979690c95 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -49,7 +49,7 @@ TO TABLE [destination_database.]destination_table 1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. -3. **Partition key columns at the same position** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. +3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index f302090a4905..467dfb4394fa 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,7 +13,11 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include @@ -636,6 +640,61 @@ namespace ExportPartitionUtils } #endif + namespace + { + bool haveSameTupleElementLayout(const DataTypePtr & source_type, const DataTypePtr & destination_type) + { + const auto source_type_unwrapped = removeNullable(removeLowCardinality(source_type)); + const auto destination_type_unwrapped = removeNullable(removeLowCardinality(destination_type)); + + const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); + if (!source_tuple || !destination_tuple) + return source_tuple == destination_tuple; + + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + + const auto & source_elements = source_tuple->getElements(); + const auto & destination_elements = destination_tuple->getElements(); + for (size_t i = 0; i < source_elements.size(); ++i) + if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + return false; + + return true; + } + + void verifyPartitionKeyColumn( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + size_t position, + const StorageID & destination_storage_id) + { + if (source_column.name != destination_column.name) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + position, + destination_column.name); + + if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' has a different Tuple element " + "layout in the source ({}) and destination ({}). Tuple element names must be " + "declared in the same order in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } + } + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -693,19 +752,8 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (partition_key_owner_columns.contains(source_column.name) && source_column.name != destination_column.name) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, - i, - destination_column.name); - } + if (partition_key_owner_columns.contains(source_column.name)) + verifyPartitionKeyColumn(source_column, destination_column, i, destination_storage_id); /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b77c2f8159cb..241c689c259c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -521,36 +521,27 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def read_exported_files(node, s3_table): - return node.query( - f"SELECT t.a, t.b, val FROM " - f"s3('http://minio1:9001/root/data/{s3_table}/**', 'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'t Tuple(b Int32, a Int32), val String') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - - -def get_export_part_log(node, mt_table): - node.query("SYSTEM FLUSH LOGS") - return node.query( - f"SELECT part_name, error, exception FROM system.part_log " - f"WHERE event_type = 'ExportPart' AND database = currentDatabase() " - f"AND table = '{mt_table}' ORDER BY event_time" - ).strip() - - -def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_is_allowed(cluster): +@pytest.mark.parametrize( + "partition_by", + [ + pytest.param("t.a", id="named_subcolumn"), + pytest.param("tupleElement(t, 1)", id="positional_tuple_element"), + ], +) +def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( + cluster, partition_by +): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_subcol_wc_mt_table_{postfix}" - s3_table = f"tuple_subcol_wc_s3_table_{postfix}" + mt_table = f"reordered_tuple_mt_table_{postfix}" + s3_table = f"reordered_tuple_s3_table_{postfix}" node.query(f""" CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) ENGINE = MergeTree() - PARTITION BY t.a + PARTITION BY {partition_by} ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) @@ -558,7 +549,7 @@ def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_i node.query(f""" CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a + PARTITION BY {partition_by} """) node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") @@ -568,15 +559,12 @@ def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_i f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(5) - - part_log = get_export_part_log(node, mt_table) - result = read_exported_files(node, s3_table) - assert result == "1\t99\tx", ( - f"Tuple element values were remapped: source t = (a=1, b=99), " - f"exported files read back (t.a, t.b) as: {result!r}; part_log: {part_log!r}" + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "different Tuple element layout" in error, ( + f"Expected export to reject reordered named `Tuple` fields used by " + f"`PARTITION BY {partition_by}`, got: {error!r}" ) From bce08c3b157e147496f29322ed397e91a94f0ce7 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 12:45:41 +0200 Subject: [PATCH 16/18] tuple in array/map Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 4 +- docs/en/antalya/partition_export.md | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 44 +++++++++++++--- .../test.py | 51 ++++++++++++++++--- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 24ce21db5b02..4ec36db5ea6f 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,12 +51,14 @@ Source and destination tables must support positional schema conversion: 1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column has a named `Tuple` type, its element names must also be declared in the same order, recursively for nested tuples. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order. This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + The same rule applies when the named tuple is nested inside a container. For example, `arr Array(Tuple(a Int32, b Int32))` and `arr Array(Tuple(b Int32, a Int32))` are incompatible when `arr` provides an input to the partition key. Likewise, tuple layouts in both the key and value types of `Map` are checked recursively. + In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`. For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 90b979690c95..7a9773ed293d 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -49,7 +49,7 @@ TO TABLE [destination_database.]destination_table 1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. +3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 467dfb4394fa..1140ae21d3f3 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -649,18 +651,44 @@ namespace ExportPartitionUtils const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); - if (!source_tuple || !destination_tuple) - return source_tuple == destination_tuple; + if (source_tuple || destination_tuple) + { + if (!source_tuple || !destination_tuple) + return false; + + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + + const auto & source_elements = source_tuple->getElements(); + const auto & destination_elements = destination_tuple->getElements(); + for (size_t i = 0; i < source_elements.size(); ++i) + if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + return false; + + return true; + } - if (source_tuple->getElementNames() != destination_tuple->getElementNames()) - return false; + const auto * source_array = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_array = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_array || destination_array) + { + if (!source_array || !destination_array) + return false; + + return haveSameTupleElementLayout(source_array->getNestedType(), destination_array->getNestedType()); + } - const auto & source_elements = source_tuple->getElements(); - const auto & destination_elements = destination_tuple->getElements(); - for (size_t i = 0; i < source_elements.size(); ++i) - if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + const auto * source_map = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_map = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_map || destination_map) + { + if (!source_map || !destination_map) return false; + return haveSameTupleElementLayout(source_map->getKeyType(), destination_map->getKeyType()) + && haveSameTupleElementLayout(source_map->getValueType(), destination_map->getValueType()); + } + return true; } diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 241c689c259c..ba806b8efc03 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -522,14 +522,49 @@ def test_export_part_multi_column_partition_key_success(cluster): @pytest.mark.parametrize( - "partition_by", + "owner_name, source_type, destination_type, partition_by, insert_value", [ - pytest.param("t.a", id="named_subcolumn"), - pytest.param("tupleElement(t, 1)", id="positional_tuple_element"), + pytest.param( + "t", + "Tuple(a Int32, b Int32)", + "Tuple(b Int32, a Int32)", + "t.a", + "(1, 99)", + id="named_subcolumn", + ), + pytest.param( + "t", + "Tuple(a Int32, b Int32)", + "Tuple(b Int32, a Int32)", + "tupleElement(t, 1)", + "(1, 99)", + id="positional_tuple_element", + ), + pytest.param( + "arr", + "Array(Tuple(a Int32, b Int32))", + "Array(Tuple(b Int32, a Int32))", + "tupleElement(arr[1], 'a')", + "[(1, 99)]", + id="tuple_nested_in_array", + ), + pytest.param( + "m", + "Map(String, Tuple(a Int32, b Int32))", + "Map(String, Tuple(b Int32, a Int32))", + "tupleElement(m['key'], 'a')", + "map('key', (1, 99))", + id="tuple_nested_in_map_value", + ), ], ) -def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( - cluster, partition_by +def test_export_part_tuple_fields_reordered_for_partition_key_is_rejected( + cluster, + owner_name, + source_type, + destination_type, + partition_by, + insert_value, ): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -539,7 +574,7 @@ def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( s3_table = f"reordered_tuple_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) + CREATE TABLE {mt_table} ({owner_name} {source_type}, val String) ENGINE = MergeTree() PARTITION BY {partition_by} ORDER BY tuple() @@ -547,12 +582,12 @@ def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( """) node.query(f""" - CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + CREATE TABLE {s3_table} ({owner_name} {destination_type}, val String) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') PARTITION BY {partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + node.query(f"INSERT INTO {mt_table} VALUES ({insert_value}, 'x')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " From c5bdb0259a6b1d84e5eb93b279d79eaa3f267382 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 14:59:50 +0200 Subject: [PATCH 17/18] support unnamed tuple Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 7 +++- .../test.py | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 1140ae21d3f3..f3c4aadfb31f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -656,7 +656,12 @@ namespace ExportPartitionUtils if (!source_tuple || !destination_tuple) return false; - if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + if (source_tuple->hasExplicitNames() && destination_tuple->hasExplicitNames()) + { + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + } + else if (source_tuple->getElements().size() != destination_tuple->getElements().size()) return false; const auto & source_elements = source_tuple->getElements(); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index ba806b8efc03..107fc755c269 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -603,6 +603,38 @@ def test_export_part_tuple_fields_reordered_for_partition_key_is_rejected( ) +def test_export_part_unnamed_tuple_partition_key_owner_matching_named_destination_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"unnamed_tuple_ok_mt_table_{postfix}" + s3_table = f"unnamed_tuple_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(Int32, Int32), val String) + ENGINE = MergeTree() + PARTITION BY tupleElement(t, 1) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(x Int32, y Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY tupleElement(t, 1) + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] From 9286316f9495a9fc12c18028733029a719b7b630 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 15:00:37 +0200 Subject: [PATCH 18/18] up doc Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 4ec36db5ea6f..35a6368b5ceb 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -57,6 +57,8 @@ Source and destination tables must support positional schema conversion: For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + The element-name check only applies when both the source and destination `Tuple` declare explicit names; an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is compared to the destination by element position and type only. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. + The same rule applies when the named tuple is nested inside a container. For example, `arr Array(Tuple(a Int32, b Int32))` and `arr Array(Tuple(b Int32, a Int32))` are incompatible when `arr` provides an input to the partition key. Likewise, tuple layouts in both the key and value types of `Map` are checked recursively. In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`.